From 7c21068180e3bae242ff4fb2a62db2ebd55d8c67 Mon Sep 17 00:00:00 2001 From: James Braid Date: Fri, 30 Oct 2026 08:03:24 +0000 Subject: [PATCH 01/31] cpu: keep every implemented VPN2 bit on the R10000 family EntryHi, the TLB match and the xuseg bounds check all stopped at VA bit 39, but the R10000 family implements 44 bits: VPN2 is 43:13 and the user segment is 16 TiB (R10000 UM pp. 219, 288). A 64-bit kernel can put an n64 user stack above 40 bits (0x0fff_ffff_aee4, for one). EntryHi dropped VA[43:40] and such an address took an Address Error. The width now comes from the part's VA_BITS in each place: the EntryHi write mask, the TLB's 64-bit compare (Tlb::set_va_bits, installed by the executor beside set_cache_attrs), the xuseg bounds check, the micro-TLB's cacheable segment test and the EntryHi value a TLB exception loads. The mask was spelled three times and is now one const fn, mips_tlb::vpn2_mask. All of it is a const of the CPU model or set once at construction; the TLB lookup is unchanged. This reverses a comment in the executor (b7d2a18b) that said widening the EntryHi mask breaks the IP28 and Octane kernels. Those two machines and an O2 set to the R10000 now see 44 bits. The R4400 and R5000 (Indy, Indigo2, O2 with an R5000) stay at 40. One test maps a page at 0x123_4000_0000 on the R10000 and the R4400: the R10000 keeps EntryHi and reads the page, the R4400 masks it and takes Address Error. --- CHANGELOG.md | 7 ++++++ src/cpu/mips_core.rs | 16 +++++++----- src/cpu/mips_exec.rs | 52 ++++++++++++++++++++++----------------- src/cpu/mips_exec_test.rs | 50 +++++++++++++++++++++++++++++++++++++ src/cpu/mips_tlb.rs | 17 +++++++++++-- 5 files changed, 112 insertions(+), 30 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e73a77b8d..76c6d4372 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -114,6 +114,13 @@ hashes are given where a change is easiest to understand by reading the commit. the cost follows what the guest wrote. Reflinking is unchanged. On Windows the copy is marked sparse first. +### R10000 TLB widths + +- **Full 44-bit user addresses.** On the R10000 (Octane, IP28, and an O2 set to + `r10000`) EntryHi, the TLB match and the user-segment check keep VA bits 43:40; + they stopped at bit 39 before, so an address above 1 TiB took an Address Error. + The R4400 and R5000 are unchanged. + ### `IRIS_GUI_GL` works in iris-gui - The GUI's GL capture path (`IRIS_GUI_GL=1`, or `[debug] gui_gl_capture`) never diff --git a/src/cpu/mips_core.rs b/src/cpu/mips_core.rs index 068f7dab0..f2efd6611 100644 --- a/src/cpu/mips_core.rs +++ b/src/cpu/mips_core.rs @@ -881,6 +881,9 @@ pub struct MipsCore { pub cp0_errorepc: u64, // 30: Error Exception PC pub tlb_entries: u32, // Total TLB entries + /// Writable EntryHi bits: region, ASID and as much VPN2 as the model keeps + /// (`CpuModel::VA_BITS`). Set once by the executor. + pub entryhi_mask: u64, /// CP0 PRId / CP1 FIR reset values, supplied by the CPU model at construction. pub reset_prid: u32, pub reset_fir: u32, @@ -1413,6 +1416,7 @@ impl MipsCore { cp0_taghi: 0, cp0_errorepc: 0, tlb_entries: 48, + entryhi_mask: 0xC000_00FF_FFFF_E0FF, reset_prid: 0x0000_0440, reset_fir: 0x0000_0500, cp0_random_cycle: 0, @@ -2035,16 +2039,16 @@ impl MipsCore { } 10 => { // always use 64bit mask because the entries need to be valid in 64 bit mode even when they were set from 32 bit mode // `log mips mask cp0` shows what the guest wrote against - // what survives the mask. The mask is R4400's 40-bit virtual - // address; the R10000 implements 44. - if cp0_log() && (value & !0xC000_00FF_FFFF_E0FF) != 0 { + // what survives the mask: the model's VPN2 width, 40 bits or the + // R10000 family's 44. + if cp0_log() && (value & !self.entryhi_mask) != 0 { crate::dlog!(crate::devlog::LogModule::Mips, "ip28ehi: wrote {value:#018x} -> kept {:#018x} (lost {:#018x}) pc={:#018x}", - value & 0xC000_00FF_FFFF_E0FF, - value & !0xC000_00FF_FFFF_E0FF, + value & self.entryhi_mask, + value & !self.entryhi_mask, self.pc); } - self.cp0_entryhi = value & 0xC000_00FF_FFFF_E0FF; + self.cp0_entryhi = value & self.entryhi_mask; }, 11 => { // Materialize the count *first* so both new_delta below and diff --git a/src/cpu/mips_exec.rs b/src/cpu/mips_exec.rs index 62aaee8de..76b099e2c 100644 --- a/src/cpu/mips_exec.rs +++ b/src/cpu/mips_exec.rs @@ -577,17 +577,23 @@ pub const TR_GLOBAL: u32 = 1 << 4; /// the privilege epoch that filled it. The only thing rejected here is a VA /// shape that is not a valid address in the first place. #[inline] -fn nutlb_cacheable_segment(va: u64, is_64bit: bool) -> bool { +fn nutlb_cacheable_segment(va: u64, is_64bit: bool) -> bool { if !is_64bit { // Every 32-bit segment (kuseg/kseg0/kseg1/ksseg/kseg3) is a real // address whose finished translation is worth caching. return true; } - // xuseg: bits 63:40 must be zero to be a valid user address. Every other - // 64-bit segment (xsseg, xkphys, xkseg and the 0xFFFFFFFF_* compat - // windows) is fine — translate_64bit_impl raises before returning ok for - // the illegal xkphys sub-ranges, so nothing invalid reaches a fill. - (va >> 62) != 0 || (va >> 40) == 0 + // xuseg: bits 63:`C::VA_BITS` must be zero to be a valid user address. + // Every other 64-bit segment (xsseg, xkphys, xkseg and the 0xFFFFFFFF_* + // compat windows) is fine — translate_64bit_impl raises before returning ok + // for the illegal xkphys sub-ranges, so nothing invalid reaches a fill. + (va >> 62) != 0 || (va >> C::VA_BITS) == 0 +} + +/// The EntryHi bits this model keeps: region, VPN2 up to `C::VA_BITS` and ASID. +#[inline(always)] +const fn model_entryhi_mask() -> u64 { + 0xC000_0000_0000_00FF | crate::cpu::mips_tlb::vpn2_mask(C::VA_BITS) } /// Per-access-class tally of how far a guest load/store would get through the @@ -2202,7 +2208,9 @@ impl MipsExecutor { let mut core = MipsCore::new(); // Mapped pages decode their C field with this model's table, as KSEG0 // and xkphys do. + tlb.set_va_bits(C::VA_BITS); tlb.set_cache_attrs(Self::cache_attr_table()); + core.entryhi_mask = model_entryhi_mask::(); // Cache geometry comes from the model type C, so Config describes this CPU. // `mut` is only used by the Triton L2-enable sync below. @@ -2873,7 +2881,7 @@ impl MipsExecutor { // user address. Everything else is cacheable regardless of which // mode filled it, because privilege is enforced by flushing on // transition rather than by the tag. - if !nutlb_cacheable_segment(va, self.core.is_64bit_mode()) { + if !nutlb_cacheable_segment::(va, self.core.is_64bit_mode()) { return; } @@ -4518,11 +4526,11 @@ va={:#018x} phys={:#010x} (code pfn {:#x}, page {:#010x}, word {}/{})", let top_bits = virt_addr >> 62; match top_bits { - // xuseg: 0x0000_0000_0000_0000 - 0x0000_00FF_FFFF_FFFF (user mapped) - // Accessible from all privilege levels + // xuseg is user-mapped below 2^C::VA_BITS (0x0000_00FF_FFFF_FFFF + // for the 40-bit models) and accessible from every privilege level. 0 => { - if (virt_addr >> 40) != 0 { - // Bits 63:40 must be zero for valid user address + if (virt_addr >> C::VA_BITS) != 0 { + // Bits above the model's implemented user VA must be zero if !DEBUG { self.core.cp0_badvaddr = virt_addr; } return TranslateResult::exc(exec_exception(if access_type == AccessType::Write { EXC_ADES } else { EXC_ADEL })); } @@ -4673,8 +4681,12 @@ va={:#018x} phys={:#010x} (code pfn {:#x}, page {:#010x}, word {}/{})", #[inline] fn update_tlb_exception_registers(&mut self, virt_addr: u64) { const EH_VPN2_32: u64 = 0x0000_0000_FFFF_E000; - const EH_VPN2_64: u64 = 0x0000_00FF_FFFF_E000; const EH_REGION: u64 = 0xC000_0000_0000_0000; + // VPN2 is bits 43:13 on the R10000 family (R10000 UM pp. 219, 288) and + // 39:13 on the R4x00 and R5000. The R12000 is inferred to match (its + // manual lists no TLB change); the R14000 and R16000 are a model choice + // that follows the same width. + let eh_vpn2_64 = crate::cpu::mips_tlb::vpn2_mask(C::VA_BITS); self.core.cp0_badvaddr = virt_addr; @@ -4697,9 +4709,9 @@ va={:#018x} phys={:#010x} (code pfn {:#x}, page {:#010x}, word {}/{})", // // This is MAME r4000's `EH_WM` (0xC000_00FF_FFFF_E0FF) less the ASID // bits, which are carried separately below. - const EH_VPN_MASK: u64 = EH_REGION | EH_VPN2_64; // 0xC000_00FF_FFFF_E000 + let eh_vpn_mask = EH_REGION | eh_vpn2_64; let asid = self.core.cp0_entryhi & 0xFF; - self.core.cp0_entryhi = (virt_addr & EH_VPN_MASK) | asid; + self.core.cp0_entryhi = (virt_addr & eh_vpn_mask) | asid; // Context: PTEBase[63:23] preserved, BadVPN2 = virt_addr[31:13] in bits [22:4]. // Always 32-bit VPN — Context is used by the 32-bit UTLB handler. @@ -4720,10 +4732,6 @@ va={:#018x} phys={:#010x} (code pfn {:#x}, page {:#010x}, word {}/{})", // the address it computes lands outside the page table it wired, the // nested miss is unrecoverable, and IRIX panics with "Entered tlbmiss // with invalid vaddr". - // - // EntryHi keeps the 40-bit VPN2 field on every model: a 64-bit kernel - // zero-fills those upper bits itself when it builds an EntryHi value, - // and widening the mask here breaks it. self.core.cp0_xcontext = xcontext_on_miss(C::VA_BITS, self.core.cp0_xcontext, virt_addr); } @@ -7129,12 +7137,12 @@ va={:#018x} phys={:#010x} (code pfn {:#x}, page {:#010x}, word {}/{})", let g1 = (self.core.cp0_entrylo1 & 1) != 0; let g_combined = if g0 && g1 { 1u64 << 12 } else { 0 }; - // EH_WM = 0xC000_00FF_FFFF_E0FF — same mask MAME applies on TLBWI (clears reserved bits). - // Then clear bit 12 (G bit from EntryHi) and set combined G from Lo0&Lo1. - const EH_WM: u64 = 0xC000_00FF_FFFF_E0FF; + // EntryHi's writable mask (0xC000_00FF_FFFF_E0FF on a 40-bit model, MAME's + // `EH_WM`) clears reserved bits, as on TLBWI. Then clear bit 12 (G bit + // from EntryHi) and set combined G from Lo0&Lo1. TlbEntry { page_mask: self.core.cp0_pagemask, - entry_hi: (self.core.cp0_entryhi & EH_WM & !0x1000) | g_combined, + entry_hi: (self.core.cp0_entryhi & self.core.entryhi_mask & !0x1000) | g_combined, entry_lo: [self.core.cp0_entrylo0, self.core.cp0_entrylo1], selector_bit_shift: 0, // all derived fields overwritten by MipsTlb::write() vcmp32: 0, vpn_hi32: 0, vcmp64: 0, vpn_hi64: 0, diff --git a/src/cpu/mips_exec_test.rs b/src/cpu/mips_exec_test.rs index 4098813ec..45e787b9e 100644 --- a/src/cpu/mips_exec_test.rs +++ b/src/cpu/mips_exec_test.rs @@ -6418,4 +6418,54 @@ mod tests { } assert_eq!(outcome[0], outcome[1], "an explanation must not change what the guest sees"); } + + /// Run `f` on a thread with room for a `MipsTlb`, which is large. + fn on_big_stack(f: impl FnOnce() + Send + 'static) { + std::thread::Builder::new().stack_size(16 * 1024 * 1024).spawn(f).unwrap().join().unwrap(); + } + + /// A 64-bit kernel executor with one TLB entry, written through CP0 and + /// TLBWI, that maps the page at `va` to `phys` (cached, valid, dirty). + fn map_page>>( + bus: Arc, va: u64, phys: u64, + ) -> Box> { + let cfg = MipsCpuConfig::for_model::(Arc::new(crate::ip22_bus::memory_map(false))); + let mut exec = Box::new(MipsExecutor::new(bus, crate::cpu::mips_tlb::MipsTlb::default(), &cfg)); + disable_jitv2(&mut exec); + // MTC0 must refresh the boxed executor's translation/accessor state. + exec.install_status_cb(); + let status = (exec.core.cp0_status & !crate::cpu::mips_core::STATUS_ERL) | STATUS_KX; + let entrylo = ((phys >> 12) << 6) | (3 << 3) | (1 << 2) | (1 << 1); + for (rd, value) in [(12, status as u64), (10, va), (0, 0), (5, 0), (2, entrylo), (3, 0)] { + exec.core.write_gpr(1, value); + assert_eq!(exec.exec((OP_COP0 << 26) | (RS_DMTC0 << 21) | (1 << 16) | (rd << 11)), EXEC_COMPLETE); + } + assert_eq!(exec.exec((OP_COP0 << 26) | (RS_TLB << 21) | FUNCT_TLBWI), EXEC_COMPLETE); + exec + } + + /// EntryHi and the TLB keep every implemented VPN2 bit: 44 virtual-address + /// bits on the R10000, 40 on the R4400. An IRIX n64 stack can sit + /// above the 40-bit limit. + #[test] + fn cp0_entryhi_and_tlb_match_full_xuseg_va() { + on_big_stack(|| { + const VA: u64 = 0x0000_0123_4000_0000; + fn check>>(full_va: bool) { + let mem = Arc::new(MockMemory::new()); + mem.set_word(0x5_0000, 0xdead_beef); + let mut exec = map_page::(mem, VA, 0x5_0000); + let result = exec.read_data::<4>(VA); + if full_va { + assert_eq!(exec.core.cp0_entryhi, VA); + assert_eq!(result, Ok(0xdead_beef)); + } else { + assert_eq!(exec.core.cp0_entryhi, VA & 0xC000_00FF_FFFF_E0FF); + assert_eq!((result.unwrap_err() >> 2) & 0x1f, EXC_ADEL); + } + } + check::(false); + check::(true); + }); + } } diff --git a/src/cpu/mips_tlb.rs b/src/cpu/mips_tlb.rs index 05680e0c6..275f513e0 100644 --- a/src/cpu/mips_tlb.rs +++ b/src/cpu/mips_tlb.rs @@ -295,6 +295,11 @@ pub enum TlbResult { }, } +/// VPN2 bits of a virtual address `bits` wide: bit 13 up to `bits - 1`. +pub const fn vpn2_mask(bits: u32) -> u64 { + ((1u64 << bits) - 1) & !0x1fff +} + /// TLB interface for MIPS address translation /// /// The TLB translates virtual page numbers to physical page numbers. @@ -351,6 +356,8 @@ pub trait Tlb { fn power_on(&mut self) {} /// Install the CPU model's meaning for each EntryLo C value. fn set_cache_attrs(&mut self, _table: [CacheAttr; 8]) {} + /// Set the implemented VPN width before installing entries. + fn set_va_bits(&mut self, _bits: u32) {} fn save_state(&self) -> toml::Value { toml::Value::Table(Default::default()) } fn load_state(&mut self, _v: &toml::Value) -> Result<(), String> { Ok(()) } @@ -534,6 +541,7 @@ pub struct MipsTlb { /// Live JTLB entries for the CPU model in use (<= TLB_MAX_ENTRIES). num_entries: usize, + vpn_mask64: u64, /// What each EntryLo C value means on the CPU model in use; the /// executor installs its table with `set_cache_attrs`. c_attr: [CacheAttr; 8], @@ -564,6 +572,7 @@ impl MipsTlb { mru_head: [0u8; MRU_LISTS], mru_next: [[MRU_NONE; TLB_MAX_ENTRIES]; MRU_LISTS], num_entries, + vpn_mask64: 0xC000_00FF_FFFF_E000, c_attr: DEFAULT_C_ATTR, vmap: [VMAP_MISS; VMAP_SIZE], #[cfg(feature = "tlbcheck")] @@ -1065,7 +1074,7 @@ impl Tlb for MipsTlb { entry.selector_bit_shift = (mask.trailing_ones() - 1) as u8; entry.vcmp32 = !mask & 0x0000_0000_FFFF_E000; entry.vpn_hi32 = entry.entry_hi & entry.vcmp32; - entry.vcmp64 = !mask & 0xC000_00FF_FFFF_E000; + entry.vcmp64 = !mask & self.vpn_mask64; entry.vpn_hi64 = entry.entry_hi & entry.vcmp64; entry.offset_mask = (1u64 << entry.selector_bit_shift) - 1; let pfn_mask = 0xFF_FFFF_FFFF_000 & !entry.offset_mask; @@ -1194,6 +1203,10 @@ impl Tlb for MipsTlb { output } + fn set_va_bits(&mut self, bits: u32) { + self.vpn_mask64 = 0xC000_0000_0000_0000 | vpn2_mask(bits); + } + fn set_cache_attrs(&mut self, table: [CacheAttr; 8]) { self.c_attr = table; for i in 0..self.num_entries { @@ -1237,7 +1250,7 @@ impl Tlb for MipsTlb { let entry_hi = words[1]; let mask = page_mask | 0x1FFF; let vcmp32 = !mask & 0x0000_0000_FFFF_E000; - let vcmp64 = !mask & 0xC000_00FF_FFFF_E000; + let vcmp64 = !mask & self.vpn_mask64; let shift = (mask.trailing_ones() - 1) as u8; let offset_mask = (1u64 << shift) - 1; let lo = [words[2], words[3]]; From 8df7747aa2cdecb6c6571949096b3de69752adcf Mon Sep 17 00:00:00 2001 From: James Braid Date: Fri, 30 Oct 2026 05:16:09 -0700 Subject: [PATCH 02/31] cpu: apply FrameMask to EntryLo on a TLB write on the R10000 family FrameMask (CP0 register 21) is new with the R10000. Bits 15:0 clear EntryLo bits 33:18 as a TLB write passes them on, and the rest of the register reads as zero (R10000 UM v2.0 p. 260). iris had no register 21: mfc0 read zero and mtc0 was dropped, so a kernel that sets FrameMask to keep software flags out of the physical address got no effect. The core holds the register; the executor sets which bits are writable, all 16 on the R10000 and none on the R4400 and R5000. TLBWI and TLBWR pass the masked EntryLo values, and TLBR returns the masked entry. The register is saved with the rest of CP0 and loads as zero from an older save. CpuModel::R10K_FAMILY says a part is in the R10000 family; for now that is the R10000. Nothing changes while FrameMask is zero, its reset value, and the write path runs only on a TLB write. Tests write FrameMask on the R10000 and the R4400, map a page and check the physical address and the readback; a second writes with TLBWR as well as TLBWI and reads the entry back with TLBR. --- CHANGELOG.md | 4 +++ src/cpu/mips_cache_shadow.rs | 1 + src/cpu/mips_cache_v2.rs | 7 ++++ src/cpu/mips_core.rs | 23 ++++++++++++ src/cpu/mips_exec.rs | 12 +++++-- src/cpu/mips_exec_test.rs | 68 ++++++++++++++++++++++++++++++++++-- 6 files changed, 110 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 76c6d4372..e9cc467e1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -120,6 +120,10 @@ hashes are given where a change is easiest to understand by reading the commit. `r10000`) EntryHi, the TLB match and the user-segment check keep VA bits 43:40; they stopped at bit 39 before, so an address above 1 TiB took an Address Error. The R4400 and R5000 are unchanged. +- **FrameMask (CP0 register 21)** on the R10000 clears the EntryLo bits it + names, 33:18, on every TLB write, so a kernel that keeps software flags there + gets the physical address it meant. The register read as zero and ignored + writes before. The R4400 and R5000 have none. ### `IRIS_GUI_GL` works in iris-gui diff --git a/src/cpu/mips_cache_shadow.rs b/src/cpu/mips_cache_shadow.rs index 53b6673eb..73ab609ba 100644 --- a/src/cpu/mips_cache_shadow.rs +++ b/src/cpu/mips_cache_shadow.rs @@ -347,6 +347,7 @@ impl< _ => "shadow", }; const CACHE_KIND: &'static str = "shadow"; + const R10K_FAMILY: bool = Self::R10K_CACHE_OPS; const R10K_CACHE_OPS: bool = MODEL == model::R10000; } diff --git a/src/cpu/mips_cache_v2.rs b/src/cpu/mips_cache_v2.rs index 3dffd558c..8b6b49e15 100644 --- a/src/cpu/mips_cache_v2.rs +++ b/src/cpu/mips_cache_v2.rs @@ -565,6 +565,12 @@ pub trait CpuModel: MipsCache { /// and `Index_Load_Data` returns into it, neither of which is true of the /// R4000 hit operations that share those encodings. const R10K_CACHE_OPS: bool = false; + /// The part is in the R10000 family, which shares one programming model: + /// FrameMask, and 44 virtual and 40 physical address bits. + /// Established for the R10000. The R12000 is inferred to match: its manual's + /// list of changes from the R10000 has no TLB or CP0 change. The R14000 and + /// R16000 are a model choice that gives them the same TLB. + const R10K_FAMILY: bool = false; } pub trait MipsCache: Send + Sync { @@ -3595,6 +3601,7 @@ impl self.cp0_watchlo as u64, 19 => self.cp0_watchhi as u64, 20 => self.cp0_xcontext, + 21 => self.cp0_framemask as u64, 25 => { // PerfCnt: R4400 has no performance counters. Log any probe so // a guest expecting them (and routing their overflow onto the @@ -1692,6 +1702,7 @@ impl MipsCore { 18 => self.cp0_watchlo as u64, 19 => self.cp0_watchhi as u64, 20 => self.cp0_xcontext, + 21 => self.cp0_framemask as u64, 26 => self.cp0_ecc as u64, 27 => self.cp0_cacheerr as u64, 28 => self.cp0_taglo, @@ -1701,6 +1712,14 @@ impl MipsCore { } } + /// An EntryLo value as a TLB write passes it on: FrameMask bits 15:0 clear + /// EntryLo bits 33:18, so the flags IRIX keeps above the PFN's low bits + /// never reach the physical address (R10000 UM v2.0 p. 260). + #[inline] + pub fn framemask_entrylo(&self, lo: u64) -> u64 { + lo & !((self.cp0_framemask as u64) << 18) + } + /// Materialize the virtual CP0 Count: advance `cp0_count` by the whole /// hardware ticks elapsed since the anchor at `count_hz`, move the anchor /// forward by exactly the duration those ticks consumed (the sub-tick @@ -2257,6 +2276,10 @@ impl MipsCore { } self.cp0_xcontext = value; } + // FrameMask: bits 15:0 are implemented, the rest read as zero + // (R10000 UM v2.0 p. 260). Other parts have no register 21 and ignore + // the write. + 21 => self.cp0_framemask = value as u32 & self.framemask_mask, 25 => { #[cfg(feature = "developer_ip7")] eprintln!("[ip7] MTC0 PerfCnt (reg 25) write {:#018x} (ignored)", value); diff --git a/src/cpu/mips_exec.rs b/src/cpu/mips_exec.rs index 76b099e2c..a05a83923 100644 --- a/src/cpu/mips_exec.rs +++ b/src/cpu/mips_exec.rs @@ -2211,6 +2211,9 @@ impl MipsExecutor { tlb.set_va_bits(C::VA_BITS); tlb.set_cache_attrs(Self::cache_attr_table()); core.entryhi_mask = model_entryhi_mask::(); + if C::R10K_FAMILY { + core.framemask_mask = 0xFFFF; + } // Cache geometry comes from the model type C, so Config describes this CPU. // `mut` is only used by the Triton L2-enable sync below. @@ -7143,7 +7146,10 @@ va={:#018x} phys={:#010x} (code pfn {:#x}, page {:#010x}, word {}/{})", TlbEntry { page_mask: self.core.cp0_pagemask, entry_hi: (self.core.cp0_entryhi & self.core.entryhi_mask & !0x1000) | g_combined, - entry_lo: [self.core.cp0_entrylo0, self.core.cp0_entrylo1], + entry_lo: [ + self.core.framemask_entrylo(self.core.cp0_entrylo0), + self.core.framemask_entrylo(self.core.cp0_entrylo1), + ], selector_bit_shift: 0, // all derived fields overwritten by MipsTlb::write() vcmp32: 0, vpn_hi32: 0, vcmp64: 0, vpn_hi64: 0, offset_mask: 0, pfn_base: [0; 2], @@ -11028,7 +11034,7 @@ impl Saveable for MipsCpu cp0u64!(cp0_taglo); cp0u32!(cp0_taghi); cp0u64!(cp0_badvaddr); cp0u64!(cp0_epc); cp0u64!(cp0_errorepc); cp0u64!(cp0_entrylo0); cp0u64!(cp0_entrylo1); cp0u64!(cp0_context); - cp0u64!(cp0_pagemask); cp0u64!(cp0_entryhi); cp0u64!(cp0_xcontext); + cp0u64!(cp0_pagemask); cp0u64!(cp0_entryhi); cp0u64!(cp0_xcontext); cp0u32!(cp0_framemask); tbl.insert("cp0".into(), toml::Value::Table(cp0)); // FPU @@ -11093,7 +11099,7 @@ impl Saveable for MipsCpu ld32!(cp0_ecc); ld32!(cp0_cacheerr); ld64!(cp0_taglo); ld32!(cp0_taghi); ld64!(cp0_entrylo0); ld64!(cp0_entrylo1); ld64!(cp0_context); ld64!(cp0_pagemask); ld64!(cp0_badvaddr); ld64!(cp0_entryhi); - ld64!(cp0_xcontext); ld64!(cp0_epc); ld64!(cp0_errorepc); + ld64!(cp0_xcontext); ld32!(cp0_framemask); ld64!(cp0_epc); ld64!(cp0_errorepc); // Restart the virtual count from the restored value and re-arm // the compare timer — Instants from the previous run are // meaningless here — with a live ticket for the restored deadline. diff --git a/src/cpu/mips_exec_test.rs b/src/cpu/mips_exec_test.rs index 45e787b9e..e8468873a 100644 --- a/src/cpu/mips_exec_test.rs +++ b/src/cpu/mips_exec_test.rs @@ -3744,6 +3744,56 @@ mod tests { check::(0xF_FFFF_FFFF); } + /// FrameMask bits 15:0 clear EntryLo bits 33:18 as a TLB write takes them, + /// on the R10000 only (R10000 UM v2.0 p. 260). Physical bit 24 is EntryLo bit 18, + /// mask bit 0; the R4400 has no such register. + #[test] + fn framemask_clears_entrylo_bits_on_a_tlb_write() { + fn phys>>(framemask: u64) -> (u64, u64) { + const VA: u64 = 0x4000_0000; + let mut exec = map_page_masked::(Arc::new(MockMemory::new()), VA, 0x0300_5000, framemask); + (exec.core.read_cp0(21), exec.debug_translate(VA).phys) + } + use crate::cpu::mips_cache_shadow::R10000ShadowCache; + on_big_stack(|| { + // Mask bit 0 is written with bits above 15, which are ignored. + let all = 0xFFFF_0001; + assert_eq!(phys::(0), (0, 0x0300_5000)); + assert_eq!(phys::(all), (1, 0x0200_5000)); + assert_eq!(phys::(all), (0, 0x0300_5000)); + }); + } + + /// TLBWR masks as TLBWI does, and TLBR returns what the TLB holds: the + /// masked EntryLo, not the value that was written. + #[test] + fn framemask_applies_to_tlbwr_and_tlbr_reads_back_the_masked_entry() { + use crate::cpu::mips_cache_shadow::R10000ShadowCache; + // Write with `funct`, find the entry with TLBP, clear EntryLo0, TLBR, + // and return (physical address of the page, EntryLo0 after TLBR). + fn entry>>(funct: u32) -> (u64, u64) { + const VA: u64 = 0x4000_0000; + let mut exec = map_page_with::(Arc::new(MockMemory::new()), VA, 0x0300_5000, 0xFFFF_0001, funct); + let phys = exec.debug_translate(VA).phys; + for (rd, value) in [(10, VA), (2, 0)] { + exec.core.write_gpr(1, value); + assert_eq!(exec.exec((OP_COP0 << 26) | (RS_DMTC0 << 21) | (1 << 16) | (rd << 11)), EXEC_COMPLETE); + } + assert_eq!(exec.exec((OP_COP0 << 26) | (RS_TLB << 21) | FUNCT_TLBP), EXEC_COMPLETE); + assert_eq!(exec.exec((OP_COP0 << 26) | (RS_TLB << 21) | FUNCT_TLBR), EXEC_COMPLETE); + (phys, exec.core.read_cp0(2)) + } + on_big_stack(|| { + // FrameMask bit 0 clears EntryLo bit 18, physical bit 24. + let unmasked = ((0x0300_5000u64 >> 12) << 6) | (3 << 3) | (1 << 2) | (1 << 1); + let masked = unmasked & !(1 << 18); + for funct in [FUNCT_TLBWI, FUNCT_TLBWR] { + assert_eq!(entry::(funct), (0x0200_5000, masked), "R10000 {funct:#x}"); + assert_eq!(entry::(funct), (0x0300_5000, unmasked), "R4400 {funct:#x}"); + } + }); + } + /// The Indy, Indigo2 and O2 decode 32 physical bits, and IRIX relies on /// it: on the IP28 the kernel stack's wired entry carries PTE software /// bits as PA[34:33]. `phys_mask` must fold them away before the bus. @@ -6428,6 +6478,20 @@ mod tests { /// TLBWI, that maps the page at `va` to `phys` (cached, valid, dirty). fn map_page>>( bus: Arc, va: u64, phys: u64, + ) -> Box> { + map_page_masked::(bus, va, phys, 0) + } + + /// `map_page` with CP0 FrameMask (register 21) written before the TLB write. + fn map_page_masked>>( + bus: Arc, va: u64, phys: u64, framemask: u64, + ) -> Box> { + map_page_with::(bus, va, phys, framemask, FUNCT_TLBWI) + } + + /// `map_page_masked`, writing the entry with `funct` (TLBWI or TLBWR). + fn map_page_with>>( + bus: Arc, va: u64, phys: u64, framemask: u64, funct: u32, ) -> Box> { let cfg = MipsCpuConfig::for_model::(Arc::new(crate::ip22_bus::memory_map(false))); let mut exec = Box::new(MipsExecutor::new(bus, crate::cpu::mips_tlb::MipsTlb::default(), &cfg)); @@ -6436,11 +6500,11 @@ mod tests { exec.install_status_cb(); let status = (exec.core.cp0_status & !crate::cpu::mips_core::STATUS_ERL) | STATUS_KX; let entrylo = ((phys >> 12) << 6) | (3 << 3) | (1 << 2) | (1 << 1); - for (rd, value) in [(12, status as u64), (10, va), (0, 0), (5, 0), (2, entrylo), (3, 0)] { + for (rd, value) in [(12, status as u64), (10, va), (0, 0), (5, 0), (21, framemask), (2, entrylo), (3, 0)] { exec.core.write_gpr(1, value); assert_eq!(exec.exec((OP_COP0 << 26) | (RS_DMTC0 << 21) | (1 << 16) | (rd << 11)), EXEC_COMPLETE); } - assert_eq!(exec.exec((OP_COP0 << 26) | (RS_TLB << 21) | FUNCT_TLBWI), EXEC_COMPLETE); + assert_eq!(exec.exec((OP_COP0 << 26) | (RS_TLB << 21) | funct), EXEC_COMPLETE); exec } From d6ed97eb8331c4cc2969001118ed4a63b8258d26 Mon Sep 17 00:00:00 2001 From: James Braid Date: Fri, 30 Oct 2026 05:16:10 -0700 Subject: [PATCH 03/31] cpu: cover the R10000 family's EntryLo PFN and uncached attribute On the R10000 EntryLo holds a 28-bit PFN in bits 33:6, a 40-bit physical address, and the two-bit uncached attribute in bits 63:62; 61:34 are reserved (R10000 UM v2.0 pp. 239-240). iris kept the R4400's 30 bits, so the attribute and PFN bits 33:30 read back as zero and never reached the TLB. The core now has an EntryLo mask that the executor sets per part: the R4400 and R5000 keep 0x3FFF_FFFF, the R10000 gets 0xC000_0003_FFFF_FFFF. The TLB carries the PFN up to physical 39:12, but a TLB write clears the PFN bits below the page size, so TLBR does not return what was written to them (R10000 UM v2.0 pp. 299-300, TLBWI and TLBWR; NEC VR10000 instruction set manual U12754EJ1V0UM pp. 199-200). That is documented for the R10000; for the R12000, R14000 and R16000 no manual describes a TLB write, so they do the same as a model choice. The R4400 and R5000 keep the whole PFN. The uncached attribute is stored and read back only; it is not carried to the bus. Bits 33:32 are physical 39:38, and the Octane's guest keeps software flags there. FrameMask (previous commit) clears them on a TLB write, so the page lands at its real address and the Octane's 40-bit physical mask needs no narrowing. On the Octane, the IP28 and an O2 set to the R10000, mfc0 of EntryLo now returns bits 63:62 and PFN bits 33:30 if the guest wrote them; before they read as zero. The R4400 and R5000 are unchanged. A test checks the readback on the R10000 and the R4400, and that a page with PFN bits 33:32 set maps to physical 39:38 and is brought back by FrameMask. The Octane PROM's tlb diagnostic writes 0xa5a5a5a4 and 0x5a5a5a58 to EntryLo with 16 KB pages and expects 0xa5a5a524 and 0x5a5a5a18 back: EntryLo bits 7:6 are physical bits 13:12, inside the page. It reported 320 compare errors. A write now clears EntryLo bits PageMask >> 7; translation already ignored them, so only TLBR changes. A second test checks the diagnostic's patterns through TLBWI and TLBWR on the R10000 and the R4400, and that a 16 KB page translates the same on both. --- CHANGELOG.md | 9 ++++ src/cpu/mips_core.rs | 17 ++++--- src/cpu/mips_exec.rs | 18 ++++++-- src/cpu/mips_exec_test.rs | 94 +++++++++++++++++++++++++++++++++++++++ src/ip30_machine.rs | 5 ++- 5 files changed, 131 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e9cc467e1..ec2b7c80a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -124,6 +124,15 @@ hashes are given where a change is easiest to understand by reading the commit. names, 33:18, on every TLB write, so a kernel that keeps software flags there gets the physical address it meant. The register read as zero and ignored writes before. The R4400 and R5000 have none. +- **EntryLo** on the R10000 keeps its 28-bit PFN (bits 33:6, a 40-bit physical + address) and the uncached attribute in 63:62; it kept 30 bits before. The + attribute is stored and read back, not sent to the bus. +- **TLB writes on the R10000** store zeros in the PFN bits below the page size, + so TLBR reads them back as zero, as the part does; on 16 KB pages that is + EntryLo bits 7:6. Translation is unchanged. The PROM's `ide` `tlb` test + failed 320 compares on this. The manual documents it for the R10000; the + R12000, R14000 and R16000 do the same as a model choice. The R4400 and + R5000 still keep the whole PFN. ### `IRIS_GUI_GL` works in iris-gui diff --git a/src/cpu/mips_core.rs b/src/cpu/mips_core.rs index d2ecf2ebe..7f9ef7e9a 100644 --- a/src/cpu/mips_core.rs +++ b/src/cpu/mips_core.rs @@ -890,6 +890,10 @@ pub struct MipsCore { /// Writable EntryHi bits: region, ASID and as much VPN2 as the model keeps /// (`CpuModel::VA_BITS`). Set once by the executor. pub entryhi_mask: u64, + /// Writable EntryLo bits: the PFN and flags (29:0, a 36-bit physical + /// address), or on the R10000 family (`CpuModel::R10K_FAMILY`) the PFN in + /// 33:6 (a 40-bit physical address) and the uncached attribute in 63:62. + pub entrylo_mask: u64, /// CP0 PRId / CP1 FIR reset values, supplied by the CPU model at construction. pub reset_prid: u32, pub reset_fir: u32, @@ -1425,6 +1429,7 @@ impl MipsCore { cp0_errorepc: 0, tlb_entries: 48, entryhi_mask: 0xC000_00FF_FFFF_E0FF, + entrylo_mask: 0x3FFF_FFFF, reset_prid: 0x0000_0440, reset_fir: 0x0000_0500, cp0_random_cycle: 0, @@ -1625,8 +1630,8 @@ impl MipsCore { self.update_random(); self.cp0_random as u64 } - 2 => self.cp0_entrylo0 & 0x3FFFFFFF, // PFN is 24 bits (29:6), flags in lower bits - 3 => self.cp0_entrylo1 & 0x3FFFFFFF, // PFN is 24 bits (29:6), flags in lower bits + 2 => self.cp0_entrylo0 & self.entrylo_mask, + 3 => self.cp0_entrylo1 & self.entrylo_mask, 4 => self.cp0_context, 5 => self.cp0_pagemask & 0x01FFE000, // PageMask: only bits 24:13 are valid 6 => self.cp0_wired as u64 & 0x3F, // Wired: only bits 5:0 are valid, R4000 manual Table 4-10 @@ -1684,8 +1689,8 @@ impl MipsCore { match reg { 0 => self.cp0_index as u64, 1 => self.cp0_random as u64, - 2 => self.cp0_entrylo0 & 0x3FFFFFFF, - 3 => self.cp0_entrylo1 & 0x3FFFFFFF, + 2 => self.cp0_entrylo0 & self.entrylo_mask, + 3 => self.cp0_entrylo1 & self.entrylo_mask, 4 => self.cp0_context, 5 => self.cp0_pagemask & 0x01FFE000, 6 => self.cp0_wired as u64 & 0x3F, @@ -2034,8 +2039,8 @@ impl MipsCore { 0 => self.cp0_index = (value as u32) & (crate::cpu::mips_exec::CP0_INDEX_P | crate::cpu::mips_exec::CP0_INDEX_SLOT_MASK), 1 => { /* Random is read-only */ } - 2 => self.cp0_entrylo0 = value & 0x3FFFFFFF, // PFN is 24 bits (29:6), flags in lower bits - 3 => self.cp0_entrylo1 = value & 0x3FFFFFFF, // PFN is 24 bits (29:6), flags in lower bits + 2 => self.cp0_entrylo0 = value & self.entrylo_mask, + 3 => self.cp0_entrylo1 = value & self.entrylo_mask, 4 => { //eprintln!("MTC0 Context = {:#018x} (PTEBase={:#018x})", value, value & 0xFFFFFFFF_FF800000); self.cp0_context = value; diff --git a/src/cpu/mips_exec.rs b/src/cpu/mips_exec.rs index a05a83923..e197e3620 100644 --- a/src/cpu/mips_exec.rs +++ b/src/cpu/mips_exec.rs @@ -2213,6 +2213,9 @@ impl MipsExecutor { core.entryhi_mask = model_entryhi_mask::(); if C::R10K_FAMILY { core.framemask_mask = 0xFFFF; + // PFN in bits 33:6 and the uncached attribute in 63:62 (R10000 UM v2.0 + // pp. 239-240); 61:34 are reserved. + core.entrylo_mask = 0xC000_0003_FFFF_FFFF; } // Cache geometry comes from the model type C, so Config describes this CPU. @@ -7143,13 +7146,20 @@ va={:#018x} phys={:#010x} (code pfn {:#x}, page {:#010x}, word {}/{})", // EntryHi's writable mask (0xC000_00FF_FFFF_E0FF on a 40-bit model, MAME's // `EH_WM`) clears reserved bits, as on TLBWI. Then clear bit 12 (G bit // from EntryHi) and set combined G from Lo0&Lo1. + // + // The R10000 family stores zeros in the PFN bits the page size leaves + // out of the address, so TLBR does not return them (R10000 UM v2.0 + // pp. 299-300, TLBWI and TLBWR). PageMask bit n covers physical bit + // n-1, EntryLo bit n-7: 0x6000 (16 KB) clears EntryLo 7:6. The R4400 + // keeps the whole PFN. The manual documents this for the R10000 only; + // for the R12000, R14000 and R16000 it is a model choice, as no manual + // describes their TLB write. + let unused_pfn = if C::R10K_FAMILY { (self.core.cp0_pagemask & 0x01FF_E000) >> 7 } else { 0 }; + let lo = |v: u64| self.core.framemask_entrylo(v) & !unused_pfn; TlbEntry { page_mask: self.core.cp0_pagemask, entry_hi: (self.core.cp0_entryhi & self.core.entryhi_mask & !0x1000) | g_combined, - entry_lo: [ - self.core.framemask_entrylo(self.core.cp0_entrylo0), - self.core.framemask_entrylo(self.core.cp0_entrylo1), - ], + entry_lo: [lo(self.core.cp0_entrylo0), lo(self.core.cp0_entrylo1)], selector_bit_shift: 0, // all derived fields overwritten by MipsTlb::write() vcmp32: 0, vpn_hi32: 0, vcmp64: 0, vpn_hi64: 0, offset_mask: 0, pfn_base: [0; 2], diff --git a/src/cpu/mips_exec_test.rs b/src/cpu/mips_exec_test.rs index e8468873a..366e3fe8d 100644 --- a/src/cpu/mips_exec_test.rs +++ b/src/cpu/mips_exec_test.rs @@ -3794,6 +3794,100 @@ mod tests { }); } + /// The R10000 writes zeros into the PFN bits a page size leaves out of the + /// address, so TLBR returns them as zero (R10000 UM v2.0 pp. 299-300, TLBWI + /// and TLBWR); the later parts do the same as a model choice, since no + /// manual describes their TLB write. The R4400 keeps the whole PFN. The values are the PROM + /// diagnostic's patterns on 16 KB pages, where EntryLo bits 7:6 go unused. + /// Translation is the same on both: those bits never reach the address. + #[test] + fn tlb_write_clears_pfn_bits_below_the_page_size_on_the_r10000() { + use crate::cpu::mips_cache_shadow::R10000ShadowCache; + // Write (lo0, lo1) with PageMask 0x6000 through TLBWI or TLBWR, then + // find the entry with TLBP and TLBR it back. Returns (EntryLo0, EntryLo1, physical address of + // VA + 0x3000). + fn entry>>(funct: u32, lo0: u64, lo1: u64) -> (u64, u64, u64) { + const VA: u64 = 0x4000_0000; + let cfg = MipsCpuConfig::for_model::(Arc::new(crate::ip22_bus::memory_map(false))); + let mut exec = Box::new(MipsExecutor::::new( + Arc::new(MockMemory::new()) as Arc, crate::cpu::mips_tlb::MipsTlb::default(), &cfg)); + disable_jitv2(&mut exec); + exec.install_status_cb(); + let status = (exec.core.cp0_status & !crate::cpu::mips_core::STATUS_ERL) | STATUS_KX; + let dmtc0 = |rd: u32| (OP_COP0 << 26) | (RS_DMTC0 << 21) | (1 << 16) | (rd << 11); + for (rd, value) in [(12, status as u64), (10, VA), (0, 0), (5, 0x6000), (2, lo0), (3, lo1)] { + exec.core.write_gpr(1, value); + assert_eq!(exec.exec(dmtc0(rd)), EXEC_COMPLETE); + } + assert_eq!(exec.exec((OP_COP0 << 26) | (RS_TLB << 21) | funct), EXEC_COMPLETE); + let phys = exec.debug_translate(VA + 0x3000).phys; + // TLBWR picked the slot: find it with TLBP. + assert_eq!(exec.exec((OP_COP0 << 26) | (RS_TLB << 21) | FUNCT_TLBP), EXEC_COMPLETE); + for rd in [2, 3] { + exec.core.write_gpr(1, 0); + assert_eq!(exec.exec(dmtc0(rd)), EXEC_COMPLETE); + } + assert_eq!(exec.exec((OP_COP0 << 26) | (RS_TLB << 21) | FUNCT_TLBR), EXEC_COMPLETE); + (exec.core.read_cp0(2), exec.core.read_cp0(3), phys) + } + on_big_stack(|| { + for funct in [FUNCT_TLBWI, FUNCT_TLBWR] { + let (lo0, lo1) = (0xFFFF_FFFF_A5A5_A5A4, 0x5A5A_5A58); + // EntryLo 7:6 cleared; 63:62 and 33:6 otherwise kept. + assert_eq!(entry::(funct, lo0, lo1).0, 0xC000_0003_A5A5_A524, "R10000 {funct:#x}"); + assert_eq!(entry::(funct, lo0, lo1).1, 0x5A5A_5A18, "R10000 {funct:#x}"); + assert_eq!(entry::(funct, lo1, lo0).0, 0x5A5A_5A18, "R10000 {funct:#x}"); + // The R4400 keeps bits 29:0 as written. + assert_eq!(entry::(funct, lo0, lo1).0, 0x25A5_A5A4, "R4400 {funct:#x}"); + assert_eq!(entry::(funct, lo0, lo1).1, 0x1A5A_5A58, "R4400 {funct:#x}"); + + // A 16 KB page at 0x0300_4000 written with PFN bits for + // 0x0300_7000: both parts translate from the 16 KB frame. + let lo = ((0x0300_7000u64 >> 12) << 6) | (3 << 3) | (1 << 2) | (1 << 1); + let (r10k, _, r10k_pa) = entry::(funct, lo, 0); + let (r4k, _, r4k_pa) = entry::(funct, lo, 0); + assert_eq!((r10k, r10k_pa), (lo & !0xC0, 0x0300_7000), "R10000 {funct:#x}"); + assert_eq!((r4k, r4k_pa), (lo, 0x0300_7000), "R4400 {funct:#x}"); + } + }); + } + + /// On the R10000 family EntryLo holds a 28-bit PFN (33:6) and the uncached + /// attribute in 63:62 (R10000 UM v2.0 pp. 239-240); the R4400 and R5000 keep a + /// 24-bit PFN and nothing above it. The TLB carries the whole PFN. + #[test] + fn entrylo_covers_the_pfn_and_attribute_bits_of_the_r10000_family() { + fn readback>>() -> u64 { + let bus: Arc = Arc::new(MockMemory::new()); + let mut exec = MipsExecutor::::new(bus, PassthroughTlb::default(), &MipsCpuConfig::indy()); + disable_jitv2(&mut exec); + exec.core.write_gpr(1, u64::MAX); + // DMTC0 $1, EntryLo0 and EntryLo1; both read back the same field. + for reg in [2u32, 3] { + assert_eq!(exec.exec((0x10 << 26) | (5 << 21) | (1 << 16) | (reg << 11)), EXEC_COMPLETE); + } + let (lo0, lo1) = (exec.core.read_cp0(2), exec.core.read_cp0(3)); + assert_eq!(lo0, lo1); + lo0 + } + assert_eq!(readback::(), 0x3FFF_FFFF); + // Bits 61:34 are not PFN: they read back as zero, so only 33:32 reach physical 39:38. + assert_eq!(readback::(), 0xC000_0003_FFFF_FFFF); + + // The TLB carries the whole PFN: bits 33:32 are physical 39:38, and FrameMask + // is what keeps software flags there out of the address. + on_big_stack(|| { + const VA: u64 = 0x4000_0000; + let mut exec = map_page::( + Arc::new(MockMemory::new()), VA, 0xC0_2054_C000); + assert_eq!(exec.debug_translate(VA).phys, 0xC0_2054_C000); + // FrameMask 0xC000 is EntryLo 33:32: the page lands where its PFN says. + let mut exec = map_page_masked::( + Arc::new(MockMemory::new()), VA, 0xC0_2054_C000, 0xC000); + assert_eq!(exec.debug_translate(VA).phys, 0x2054_C000); + }); + } + /// The Indy, Indigo2 and O2 decode 32 physical bits, and IRIX relies on /// it: on the IP28 the kernel stack's wired entry carries PTE software /// bits as PA[34:33]. `phys_mask` must fold them away before the bus. diff --git a/src/ip30_machine.rs b/src/ip30_machine.rs index cdaa19b16..37051b4d8 100644 --- a/src/ip30_machine.rs +++ b/src/ip30_machine.rs @@ -599,8 +599,9 @@ impl OctaneMachine { // IRIX zeroes Count and arms Compare a few instructions later; see // `MipsCore::count_write_samples`. exec.core.count_write_samples = true; - // The R10000 drives 40 physical address bits; HEART decodes them - // all, and the attribute bits above fold away. + // The R10000 drives 40 physical address bits; HEART decodes them all. + // EntryLo bits 33:32 are physical 39:38; a guest that keeps software + // flags there clears them with FrameMask, so no narrowing is needed. exec.phys_mask = (1 << 40) - 1; exec.core.cpu_id = id; exec.core.smp = cpu_count > 1; From 28de2653147ae178943a10e7d336c083b92a50ac Mon Sep 17 00:00:00 2001 From: James Braid Date: Sun, 1 Nov 2026 19:02:01 -0700 Subject: [PATCH 04/31] cpu: choose the R10000 refill vector by address space The R10000 picks the TLB refill vector from the address space the missing address lies in and that space's Status bit: user addresses follow UX, xsseg SX, and kseg3, sseg/ksseg and xkseg KX. The current mode only decides which space an address is in (R10000 UM v2.0 pp. 333-334, 342). The manual contrasts this with the R4400, which picks by the current mode's bit. iris did the R4400 thing on every model. So a 64-bit kernel (KX=1, UX=0) missing on a kuseg address went to the XTLB vector at 0x080 on the R10000, where the part uses 0x000. The Octane PROM's utlb diagnostic loads VA 0 in exactly that state and waits at the 32-bit vector. It got "Read TLB miss exception" from the XTLB vector instead and failed. On the R10000 family a refill miss now reads the region of the address, sign-extended from 32 bits in 32-bit mode, and tests UX, SX or KX. Only the miss path changes. The TLB compare width still follows the mode. The test misses in each space with each bit on the R10000, and checks the R4400 keeps choosing by mode. --- CHANGELOG.md | 5 ++++ src/cpu/mips_exec.rs | 23 +++++++++++++++++- src/cpu/mips_exec_test.rs | 51 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 78 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ec2b7c80a..a2332d721 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -133,6 +133,11 @@ hashes are given where a change is easiest to understand by reading the commit. failed 320 compares on this. The manual documents it for the R10000; the R12000, R14000 and R16000 do the same as a model choice. The R4400 and R5000 still keep the whole PFN. +- **TLB refill vector on the R10000** follows the address space that missed: + a user address takes the XTLB vector only with UX set, xsseg with SX, and + kernel addresses with KX. A 64-bit kernel touching a 32-bit user address now + refills through 0x000, as the part does, not 0x080. The R4400 and R5000 still + choose by the current mode. ### `IRIS_GUI_GL` works in iris-gui diff --git a/src/cpu/mips_exec.rs b/src/cpu/mips_exec.rs index e197e3620..b2ceffa1b 100644 --- a/src/cpu/mips_exec.rs +++ b/src/cpu/mips_exec.rs @@ -4663,7 +4663,8 @@ va={:#018x} phys={:#010x} (code pfn {:#x}, page {:#010x}, word {}/{})", TlbResult::Miss { .. } => { if !DEBUG { self.update_tlb_exception_registers::(virt_addr); } // Miss: XTLB vector (0x080) for 64-bit extended addresses, UTLB vector (0x000) otherwise - TranslateResult::exc(if XTLB != 0 { exec_xtlb_miss(tlb_miss_code) } else { exec_tlb_miss(tlb_miss_code) }) + let xrefill = if C::R10K_FAMILY { self.r10k_xtlb_refill::(virt_addr) } else { XTLB != 0 }; + TranslateResult::exc(if xrefill { exec_xtlb_miss(tlb_miss_code) } else { exec_tlb_miss(tlb_miss_code) }) } TlbResult::Invalid { .. } => { if !DEBUG { self.update_tlb_exception_registers::(virt_addr); } @@ -4677,6 +4678,26 @@ va={:#018x} phys={:#010x} (code pfn {:#x}, page {:#010x}, word {}/{})", } } + /// Whether a refill miss on `virt_addr` takes the XTLB vector on the + /// R10000 family. The R10000 chooses by the space the address lies in, + /// not the mode it was referenced from: a user address follows UX, xsseg + /// SX, and kseg3, sseg/ksseg and xkseg follow KX (R10000 UM v2.0 pp. + /// 333-334, 342). The R4400 chooses by the current mode, which is what + /// `XTLB` encodes. So a 64-bit kernel (KX=1, UX=0) touching kuseg takes + /// the 32-bit refill vector here. In 32-bit mode only the low word of the + /// address counts, so it is sign-extended before the region is read. + #[cold] + fn r10k_xtlb_refill(&self, virt_addr: u64) -> bool { + use crate::cpu::mips_core::{STATUS_KX, STATUS_SX, STATUS_UX}; + let va = if XTLB != 0 { virt_addr } else { virt_addr as u32 as i32 as i64 as u64 }; + let x = match va >> 62 { + 0 => STATUS_UX, + 1 => STATUS_SX, + _ => STATUS_KX, + }; + self.core.cp0_status & x != 0 + } + /// Debug helper to translate address without side effects pub fn debug_translate(&mut self, virt_addr: u64) -> TranslateResult { self.translate_impl::(virt_addr, AccessType::Debug) diff --git a/src/cpu/mips_exec_test.rs b/src/cpu/mips_exec_test.rs index 366e3fe8d..3a40d7eac 100644 --- a/src/cpu/mips_exec_test.rs +++ b/src/cpu/mips_exec_test.rs @@ -5832,6 +5832,57 @@ mod tests { assert_ne!(st & EXEC_IS_XTLB_REFILL, 0, "true xkseg is unchanged: XTLB"); } + /// The R10000 picks the refill vector by the address space a missing + /// address lies in and that space's Status bit: user addresses follow + /// UX, xsseg SX, kernel and compat supervisor addresses KX (R10000 UM + /// v2.0 pp. 333-334). The R4400 picks by the current mode. A 64-bit + /// kernel (KX=1, UX=0) missing on kuseg therefore takes the 32-bit refill + /// vector on the R10000 and XTLB on the R4400; the PROM's utlb + /// diagnostic waits at the 32-bit vector for exactly that miss. + #[test] + fn r10000_refill_vector_follows_the_address_space() { + use crate::cpu::mips_cache_shadow::R10000ShadowCache; + use crate::cpu::mips_core::{STATUS_ERL, STATUS_EXL, STATUS_SX, STATUS_UX}; + use crate::cpu::mips_exec::EXEC_IS_XTLB_REFILL; + // Kernel mode with `xbits` set in Status, empty TLB; is a read of + // `va` a refill miss through XTLB? + fn xrefill>>(xbits: u32, va: u64) -> bool { + let cfg = MipsCpuConfig::for_model::(Arc::new(crate::ip22_bus::memory_map(false))); + let mut exec = Box::new(MipsExecutor::::new( + Arc::new(MockMemory::new()), crate::cpu::mips_tlb::MipsTlb::default(), &cfg)); + disable_jitv2(&mut exec); + exec.install_status_cb(); + let status = (exec.core.cp0_status & !(STATUS_ERL | STATUS_EXL | STATUS_KX | STATUS_SX | STATUS_UX | 0x18)) | xbits; + exec.core.write_gpr(1, status as u64); + assert_eq!(exec.exec((OP_COP0 << 26) | (RS_DMTC0 << 21) | (1 << 16) | (12 << 11)), EXEC_COMPLETE); + let st = exec.read_data::<4>(va).expect_err("empty TLB must miss"); + assert_ne!(st & EXEC_IS_TLB_REFILL, 0, "{va:#x} should be a refill miss"); + st & EXEC_IS_XTLB_REFILL != 0 + } + const KUSEG: u64 = 0; + const XSSEG: u64 = 0x4000_0000_0000_0000; + const CKSSEG: u64 = 0xFFFF_FFFF_C000_0000; + const XKSEG: u64 = 0xC000_0000_0000_0000; + on_big_stack(|| { + type R10k = R10000ShadowCache; + // 64-bit kernel, 32-bit user space: kuseg refills through 0x000. + assert!(!xrefill::(STATUS_KX, KUSEG)); + assert!(xrefill::(STATUS_KX, KUSEG), "R4400 follows the mode"); + assert!(xrefill::(STATUS_KX | STATUS_UX, KUSEG)); + // xsseg follows SX, compat ksseg and xkseg follow KX. + assert!(!xrefill::(STATUS_KX, XSSEG)); + assert!(xrefill::(STATUS_KX | STATUS_SX, XSSEG)); + assert!(xrefill::(STATUS_KX, CKSSEG)); + assert!(xrefill::(STATUS_KX, XKSEG)); + // 32-bit kernel: kseg3-side addresses use 0x000, kuseg follows UX. + assert!(!xrefill::(0, CKSSEG)); + assert!(!xrefill::(STATUS_UX, CKSSEG)); + assert!(!xrefill::(0, KUSEG)); + assert!(xrefill::(STATUS_UX, KUSEG)); + assert!(!xrefill::(STATUS_UX, KUSEG), "R4400 follows the mode"); + }); + } + #[test] fn test_xtlb_compare_width_disambiguates_region_field() { std::thread::Builder::new() From 0160a104b87433c6fd23d8c2810684aef038efad Mon Sep 17 00:00:00 2001 From: James Braid Date: Sun, 1 Nov 2026 23:00:04 -0700 Subject: [PATCH 05/31] cpu: invalidate overlapping TLB entries on a write on the R10000 family The R10000 and R12000 do not shut down on a multiple match. A TLB write that would let one lookup match two entries first invalidates every entry it overlaps, the wired ones included, then writes the new entry and sets Status.TS (NEC U10278EJ4V0UM, Status register TS bit p.223 and "Avoiding TLB Conflict" p.300). iris stored the entry and left the duplicates matchable, so which one answered depended on how the lookup map happened to be rebuilt. On the two-processor Tezro the firmware flushes the TLB with 64 invalid entries on the kernel's page number, and the kernel then writes the valid global 16 MiB entry over them; a later write elsewhere left an invalid duplicate answering, and the fetch took TLB Invalid. The R14000 and R16000 get the same rule as a model choice, matching QEMU's r10k_invalidate_conflicts(); no manual for them was read. The R4400 and R5000 keep their behaviour. An overlap is two entries a lookup could match together: same region and page number under the wider of the two page masks, and equal ASIDs or either entry global. Invalid entries count. The removed entry reads back as an unwritten slot except for its PageMask, as in QEMU: the kernel TLBRs its wired sentinel entries and reuses PageMask for the next write, and a zeroed mask made it write a per-CPU mapping as 4 KiB. The cost is confined to TLB writes on these models. An entry that overlaps the new one covers a lookup-map slot the new one covers, so a write whose slots are all empty or its own skips the scan; any other write compares against the (at most 63) entries written since power-on. A PageMask that is not a run of ones, which software can write, defeats the map's ranges and turns the shortcut off. The hit path, the vmap fast path and the miss path are unchanged, and the rule is off on every other model. TLBWI and TLBWR are never compiled by jitv2, so both reach the same write and set TS in the interpreter. Tests cover the Tezro sequence, wired entries, PageMask kept on a read-back, entries that cannot match together, and TS from TLBWI and TLBWR. They fail with the rule off, and the R4400 and R5000 test fails if the rule is enabled for them. --- src/cpu/mips_exec.rs | 7 ++ src/cpu/mips_exec_test.rs | 58 ++++++++++++ src/cpu/mips_tlb.rs | 112 +++++++++++++++++++++- src/cpu/mips_tlb_test.rs | 189 ++++++++++++++++++++++++++++++++++++++ 4 files changed, 364 insertions(+), 2 deletions(-) diff --git a/src/cpu/mips_exec.rs b/src/cpu/mips_exec.rs index b2ceffa1b..2c4fbf900 100644 --- a/src/cpu/mips_exec.rs +++ b/src/cpu/mips_exec.rs @@ -2210,6 +2210,7 @@ impl MipsExecutor { // and xkphys do. tlb.set_va_bits(C::VA_BITS); tlb.set_cache_attrs(Self::cache_attr_table()); + tlb.set_conflict_invalidation(C::R10K_FAMILY); core.entryhi_mask = model_entryhi_mask::(); if C::R10K_FAMILY { core.framemask_mask = 0xFFFF; @@ -7241,6 +7242,9 @@ va={:#018x} phys={:#010x} (code pfn {:#x}, page {:#010x}, word {}/{})", let entry = self.create_tlb_entry_from_cp0(); self.ip28_trace_tlb_write("tlbwi", index, &entry); self.tlb.write(index, entry); + if self.tlb.take_write_conflict() { + self.core.cp0_status |= crate::cpu::mips_core::STATUS_TS; + } // Flushes the nutlb too — the TLB it caches just changed. self.nanotlb_invalidate(); @@ -7265,6 +7269,9 @@ va={:#018x} phys={:#010x} (code pfn {:#x}, page {:#010x}, word {}/{})", let entry = self.create_tlb_entry_from_cp0(); self.ip28_trace_tlb_write("tlbwr", index, &entry); self.tlb.write(index, entry); + if self.tlb.take_write_conflict() { + self.core.cp0_status |= crate::cpu::mips_core::STATUS_TS; + } self.nanotlb_invalidate(); if mips_log(MIPS_LOG_TLB) { dlog_dev!(LogModule::Mips, "TLBWR: Write Random Index {}\n{}", index, self.tlb.format_entry(index)); } diff --git a/src/cpu/mips_exec_test.rs b/src/cpu/mips_exec_test.rs index 3a40d7eac..afc15cf7b 100644 --- a/src/cpu/mips_exec_test.rs +++ b/src/cpu/mips_exec_test.rs @@ -1227,6 +1227,64 @@ mod tests { 0xAAAAAAAA means a stale nanotlb slot survived the switch"); } + /// Two writes that would match together: on the R10000 family the second + /// removes the first and sets Status.TS, for TLBWI and TLBWR alike; on the + /// R4400 and R5000 both entries stay and TS stays clear. The compiled-code + /// path has nothing to add: jitv2 never admits TLBWI or TLBWR into a + /// region (see the `tlb_ops_are_never_admitted_but_eret_is` test), so + /// they always run this interpreter code. + fn overlapping_tlb_writes(r10k: bool) + where C: CpuModel + From> { + use crate::cpu::mips_tlb::{MipsTlb, Tlb}; + use crate::cpu::mips_core::STATUS_TS; + let mem: Arc = Arc::new(MockMemory::new()); + let mut exec = MipsExecutor::::new(mem, MipsTlb::default(), &MipsCpuConfig::indy()); + disable_jitv2(&mut exec); + let tlbwi = (OP_COP0 << 26) | (0x10 << 21) | 0x02; + let tlbwr = (OP_COP0 << 26) | (0x10 << 21) | 0x06; + let write = |exec: &mut MipsExecutor, instr: u32, index: u32, pfn: u64| { + exec.core.cp0_index = index; + exec.core.cp0_pagemask = 0; + exec.core.cp0_entryhi = (0x100 << 13) | 10; + exec.core.cp0_entrylo0 = (pfn << 6) | (3 << 3) | (1 << 2) | (1 << 1); + exec.core.cp0_entrylo1 = ((pfn + 1) << 6) | (3 << 3) | (1 << 2) | (1 << 1); + assert_eq!(exec.exec(instr), EXEC_COMPLETE); + }; + let first_survives = |exec: &MipsExecutor| exec.tlb.read(5).entry_hi == (0x100 << 13) | 10; + + write(&mut exec, tlbwi, 5, 0x50); + assert_eq!(exec.core.cp0_status & STATUS_TS, 0, "a write that overlaps nothing leaves TS clear"); + + write(&mut exec, tlbwi, 6, 0x60); + assert_eq!(exec.core.cp0_status & STATUS_TS != 0, r10k, "TLBWI TS"); + assert_eq!(!first_survives(&exec), r10k, "TLBWI removes the overlapped entry"); + if r10k { + assert_eq!(exec.tlb.read(5).page_mask, 0, "PageMask kept"); + } + + // Software clears TS by writing Status; the next overlap sets it again. + exec.core.cp0_status &= !STATUS_TS; + write(&mut exec, tlbwi, 5, 0x50); + exec.core.cp0_status &= !STATUS_TS; + exec.core.cp0_wired = 7; // Random stays out of 0..7, so it cannot pick entries 5 or 6 + write(&mut exec, tlbwr, 0, 0x70); + assert_eq!(exec.core.cp0_status & STATUS_TS != 0, r10k, "TLBWR TS"); + assert_eq!(!first_survives(&exec), r10k, "TLBWR removes the overlapped entry"); + } + + #[test] + fn overlapping_tlb_writes_follow_the_cpu_family() { + std::thread::Builder::new() + .stack_size(64 * 1024 * 1024) + .spawn(|| { + overlapping_tlb_writes::(true); + overlapping_tlb_writes::(true); + overlapping_tlb_writes::(false); + overlapping_tlb_writes::(false); + }) + .unwrap().join().unwrap(); + } + #[cfg(feature = "tlbcheck")] #[test] fn test_tlbwi_breaks_on_duplicate_entry() { diff --git a/src/cpu/mips_tlb.rs b/src/cpu/mips_tlb.rs index 275f513e0..7315c28ba 100644 --- a/src/cpu/mips_tlb.rs +++ b/src/cpu/mips_tlb.rs @@ -358,6 +358,17 @@ pub trait Tlb { fn set_cache_attrs(&mut self, _table: [CacheAttr; 8]) {} /// Set the implemented VPN width before installing entries. fn set_va_bits(&mut self, _bits: u32) {} + /// Make a TLB write invalidate every other entry it would match together + /// with, as the R10000 and R12000 do (NEC U10278EJ4V0UM, Status register + /// TS bit, p.223, and "Avoiding TLB Conflict", p.300): they do not shut + /// down on a multiple match. The R14000 and R16000 are given the same + /// rule as a model choice, matching QEMU's r10k_invalidate_conflicts(); + /// no manual for them was read. Off for the R4400 and R5000, whose + /// behaviour on a multiple match is not modelled. + fn set_conflict_invalidation(&mut self, _on: bool) {} + /// Whether a write since the last call invalidated another entry. The + /// executor reports it in Status.TS. + fn take_write_conflict(&mut self) -> bool { false } fn save_state(&self) -> toml::Value { toml::Value::Table(Default::default()) } fn load_state(&mut self, _v: &toml::Value) -> Result<(), String> { Ok(()) } @@ -560,9 +571,74 @@ pub struct MipsTlb { vmap_touched: std::collections::HashSet, #[cfg(feature = "tlbstats")] pub stats: TlbStats, + /// R10000 family: a write invalidates the entries it overlaps. See + /// [`Tlb::set_conflict_invalidation`]. + invalidate_on_write: bool, + /// Entries written since power-on; a never-written slot cannot conflict. + written: u64, + /// A write invalidated another entry since `take_write_conflict`. + write_conflict: bool, + /// Some entry written since power-on has a PageMask that is not a run of + /// ones from bit 13 (software can write any of bits 24:13). The map + /// ranges of such an entry are not exact, so `invalidate_overlapped` then + /// scans instead of asking the map. + irregular_mask: bool, } impl MipsTlb { + /// The R10000 family never lets a write create a multiple match: every + /// other entry the incoming one overlaps, wired or not, is taken out of + /// matching before the write lands, and Status.TS records that it + /// happened (see [`Tlb::set_conflict_invalidation`]). Overlap is + /// a lookup that could match both: the region and page number agree + /// under the wider of the two page masks, and the ASIDs agree or either + /// entry is global. Entries that are invalid count: they match and + /// fault. The entry reads back as a never-written slot does, zeros + /// except for its PageMask. Runs on the write path only; a lookup is unchanged. + fn invalidate_overlapped(&mut self, index: usize, new: &TlbEntry) { + // An entry that overlaps the new one covers a lookup-map slot the new + // one covers (the map is keyed on VA[31:13], and agreeing on more bits + // agrees on these), and a slot covered by two entries is tagged + // VMAP_MULTI or names the other one. So when every slot of the new + // entry's range is empty or names only `index` (the entry being + // replaced), nothing overlaps and the scan can be skipped. + if !self.irregular_mask { + let (start, count) = Self::vmap_range(new); + let tag = index as u8; + if self.vmap[start..start + count].iter().all(|&v| v == VMAP_MISS || v == tag) { + return; + } + } + let mut live = self.written & !(1u64 << index); + while live != 0 { + let i = live.trailing_zeros() as usize; + live &= live - 1; + if i >= self.num_entries { + continue; + } + let other = &self.entries[i]; + if !(new.is_global() || other.is_global() || new.asid() == other.asid()) { + continue; + } + let both = new.page_mask | other.page_mask | 0x1FFF; + if (new.entry_hi ^ other.entry_hi) & !both & self.vpn_mask64 != 0 { + continue; + } + let old_range = Self::vmap_range(other); + // TLBR of the entry reads zeros for EntryHi and EntryLo but still + // loads PageMask: software that sets PageMask once and relies on a + // read-back restoring it would otherwise write its next entry + // with a 4 KiB mask. + let mut gone = TlbEntry::new(); + gone.page_mask = other.page_mask; + self.entries[i] = gone; + self.shadow[i] = ShadowEntry::invalid(); + self.written &= !(1u64 << i); + self.vmap_vacate(i, old_range); + self.write_conflict = true; + } + } + pub fn new(num_entries: usize) -> Self { assert!(num_entries > 1 && num_entries <= TLB_MAX_ENTRIES, "MipsTlb supports 2..={} entries, got {}", TLB_MAX_ENTRIES, num_entries); @@ -579,6 +655,10 @@ impl MipsTlb { vmap_touched: std::collections::HashSet::new(), #[cfg(feature = "tlbstats")] stats: TlbStats::default(), + invalidate_on_write: false, + written: 0, + write_conflict: false, + irregular_mask: false, }; for list in 0..MRU_LISTS { tlb.mru_head[list] = 0; @@ -678,6 +758,15 @@ impl MipsTlb { /// step 3 that it now also aliases. #[inline] fn vmap_replace(&mut self, index: usize, old_range: (usize, usize)) { + self.vmap_vacate(index, old_range); + let (new_start, new_count) = Self::vmap_range(&self.entries[index]); + self.vmap_fill_range(index, new_start, new_count); + } + + /// Steps 1 to 3 of [`Self::vmap_replace`]: take `index` out of the map + /// and let the other entries reclaim what it covered. + #[inline] + fn vmap_vacate(&mut self, index: usize, old_range: (usize, usize)) { let (old_start, old_count) = old_range; self.vmap_clear(old_start, old_count); let old_end = old_start + old_count; @@ -694,8 +783,6 @@ impl MipsTlb { self.vmap_fill_range(i, lo, hi - lo); } } - let (new_start, new_count) = Self::vmap_range(&self.entries[index]); - self.vmap_fill_range(index, new_start, new_count); } /// Full internal consistency check (feature = "tlbcheck"). @@ -1082,6 +1169,11 @@ impl Tlb for MipsTlb { (entry.entry_lo[0] << 6) & pfn_mask, (entry.entry_lo[1] << 6) & pfn_mask, ]; + self.irregular_mask |= mask & (mask + 1) != 0; + if self.invalidate_on_write { + self.invalidate_overlapped(index, &entry); + } + self.written |= 1 << index; let old_range = Self::vmap_range(&self.entries[index]); self.entries[index] = entry; self.shadow[index] = ShadowEntry::from_entry(&entry, &self.c_attr); @@ -1203,6 +1295,14 @@ impl Tlb for MipsTlb { output } + fn set_conflict_invalidation(&mut self, on: bool) { + self.invalidate_on_write = on; + } + + fn take_write_conflict(&mut self) -> bool { + std::mem::take(&mut self.write_conflict) + } + fn set_va_bits(&mut self, bits: u32) { self.vpn_mask64 = 0xC000_0000_0000_0000 | vpn2_mask(bits); } @@ -1225,6 +1325,9 @@ impl Tlb for MipsTlb { self.mru_next[list][self.num_entries - 1] = MRU_NONE; } self.vmap.fill(VMAP_MISS); + self.written = 0; + self.write_conflict = false; + self.irregular_mask = false; #[cfg(feature = "tlbcheck")] { self.vmap_touched.clear(); } } @@ -1281,6 +1384,11 @@ impl Tlb for MipsTlb { self.mru_next[list][self.num_entries - 1] = MRU_NONE; } self.vmap.fill(VMAP_MISS); + self.written = !0u64 >> (64 - self.num_entries); + self.irregular_mask = self.entries[..self.num_entries].iter().any(|e| { + let mask = e.page_mask | 0x1FFF; + mask & (mask + 1) != 0 + }); for i in 0..self.num_entries { let (start, count) = Self::vmap_range(&self.entries[i]); self.vmap_fill_range(i, start, count); diff --git a/src/cpu/mips_tlb_test.rs b/src/cpu/mips_tlb_test.rs index 4ebbaeda1..d594d6e74 100644 --- a/src/cpu/mips_tlb_test.rs +++ b/src/cpu/mips_tlb_test.rs @@ -4,6 +4,195 @@ mod tests { use crate::cpu::mips_tlb::*; use crate::cpu::mips_exec::CacheAttr; + /// Region 3 base for the overlap tests below; the frame is arbitrary. + const BASE: u64 = 0xc000_0000_0000_0000; + + fn entry(mask: u64, hi: u64, lo: [u64; 2]) -> TlbEntry { + let mut e = TlbEntry::new(); + e.page_mask = mask; + e.entry_hi = hi; + e.entry_lo = lo; + e + } + + /// CPU 1 of a two-processor Tezro as its TLB looks after the firmware's + /// flush (every entry written invalid: region 3, page number 0, ASID 0, + /// not global, 2 MiB pages) and the kernel's write of the valid global + /// 16 MiB entry at index 1 over the same page number. The firmware + /// entries that overlap it are true duplicates: every compared bit agrees. + fn flushed_then_kernel_mapped(invalidate: bool) -> MipsTlb { + let mut tlb = MipsTlb::new(64); + tlb.set_conflict_invalidation(invalidate); + for index in (0..64).rev() { + tlb.write(index, entry(0x1fe000, BASE, [0; 2])); + } + // PFN 0x40, cacheable, dirty, valid, global, both halves. + tlb.write(1, entry(0x1ffe000, BASE | 0x1000, [(0x40 << 6) | (5 << 3) | 0x7; 2])); + tlb + } + + /// An entry elsewhere, as the kernel writes at index 0 next (wired). + fn write_wired_elsewhere(tlb: &mut MipsTlb) { + tlb.write(0, entry(0x6000, 0xc000_0fff_ffff_9000, [1, 0x2e42f])); + } + + /// Writing the valid entry takes the overlapped firmware entries out of + /// matching, as the R10000 family does, so the fetch the kernel makes + /// next is answered by the valid entry however the lookup map was + /// rebuilt in between. Without the rule the highest-numbered firmware + /// entry answers, as invalid. + #[test] + fn a_write_invalidates_the_entries_it_overlaps() { + std::thread::Builder::new().stack_size(64 << 20).spawn(|| { + let va = BASE + 0x1234; + let mut off = flushed_then_kernel_mapped(false); + write_wired_elsewhere(&mut off); + assert!(matches!(off.translate::<1>(va, 0, AccessType::Fetch), TlbResult::Invalid { .. }), + "without the rule, the duplicate answers"); + assert!(!off.take_write_conflict()); + assert_eq!(off.read(2).entry_hi, BASE, "without the rule, the duplicate stays"); + + let mut tlb = flushed_then_kernel_mapped(true); + assert!(tlb.take_write_conflict(), "the writes overlapped entries"); + write_wired_elsewhere(&mut tlb); + match tlb.translate::<1>(va, 0, AccessType::Fetch) { + TlbResult::Hit { phys_addr, .. } => assert_eq!(phys_addr, 0x1234), + other => panic!("expected the valid entry to answer, got {other:?}"), + } + assert_eq!(tlb.read(2).entry_hi, 0); + assert_eq!(tlb.read(1).page_mask, 0x1ffe000); + // An entry in another region overlaps nothing and removes nothing. + tlb.write(5, entry(0x1fe000, 0x8000_0000_0000_0000, [0; 2])); + assert!(!tlb.take_write_conflict()); + assert!(matches!(tlb.translate::<1>(va, 0, AccessType::Fetch), TlbResult::Hit { .. })); + }).unwrap().join().unwrap(); + } + + /// The rule reaches the wired entries (the low indexes the kernel + /// reserves): a new entry over a wired one removes the wired one. + #[test] + fn a_write_invalidates_wired_entries_too() { + std::thread::Builder::new().stack_size(64 << 20).spawn(|| { + let va = BASE + 0x2345; + let lo = |pfn: u64| [(pfn << 6) | (5 << 3) | 0x7; 2]; + let run = |on: bool| { + let mut tlb = MipsTlb::new(64); + tlb.set_conflict_invalidation(on); + tlb.write(0, entry(0, BASE | 0x2000, lo(0x10))); + tlb.write(40, entry(0, BASE | 0x2000, lo(0x20))); + tlb + }; + let on = run(true); + assert!(on.read(0).entry_hi == 0 && on.read(0).entry_lo == [0; 2], "wired entry 0 is gone"); + assert_eq!(on.read(40).entry_hi, BASE | 0x2000); + let mut on = on; + assert!(on.take_write_conflict()); + match on.translate::<1>(va, 0, AccessType::Read) { + TlbResult::Hit { phys_addr, .. } => assert_eq!(phys_addr, (0x20 << 12) | 0x345), + other => panic!("expected the new entry to answer, got {other:?}"), + } + let off = run(false); + assert_eq!(off.read(0).entry_hi, BASE | 0x2000, "without the rule the wired entry stays"); + }).unwrap().join().unwrap(); + } + + /// TLBR of an invalidated entry still loads its PageMask. The kernel + /// reads a wired sentinel back and reuses PageMask for its next write; a + /// zeroed mask makes that write 4 KiB. + #[test] + fn an_invalidated_entry_keeps_its_page_mask() { + std::thread::Builder::new().stack_size(64 << 20).spawn(|| { + let mut tlb = MipsTlb::new(64); + tlb.set_conflict_invalidation(true); + tlb.write(3, entry(0x1fe000, BASE, [0x7; 2])); + tlb.write(9, entry(0x1ffe000, BASE, [0x7; 2])); + let gone = tlb.read(3); + assert_eq!(gone.page_mask, 0x1fe000); + assert_eq!(gone.entry_hi, 0); + assert_eq!(gone.entry_lo, [0; 2]); + assert_eq!(tlb.read(9).page_mask, 0x1ffe000); + }).unwrap().join().unwrap(); + } + + /// Entries that cannot match together are left alone: a different ASID + /// when neither is global, and a page number that differs above the + /// wider mask. + #[test] + fn entries_that_cannot_match_together_are_kept() { + std::thread::Builder::new().stack_size(64 << 20).spawn(|| { + let mut tlb = MipsTlb::new(64); + tlb.set_conflict_invalidation(true); + tlb.write(3, entry(0, BASE | 0x4000 | 1, [0x6; 2])); // ASID 1, not global + tlb.write(4, entry(0, BASE | 0x4000 | 2, [0x6; 2])); // ASID 2 + tlb.write(5, entry(0, BASE | 0x8000 | 1, [0x6; 2])); // other page + assert!(!tlb.take_write_conflict()); + for i in [3, 4, 5] { + assert_ne!(tlb.read(i).entry_hi, 0, "entry {i} stays"); + } + // A global entry over that page does match both ASIDs. + tlb.write(6, entry(0, BASE | 0x4000 | 0x1000, [0x7; 2])); // global + assert!(tlb.take_write_conflict()); + assert_eq!(tlb.read(3).entry_hi, 0); + assert_eq!(tlb.read(4).entry_hi, 0); + assert_ne!(tlb.read(5).entry_hi, 0); + }).unwrap().join().unwrap(); + } + + /// The overlap rule written out directly: what a write must remove. + fn overlaps(new: &TlbEntry, other: &TlbEntry) -> bool { + if !(new.is_global() || other.is_global() || new.asid() == other.asid()) { + return false; + } + let both = new.page_mask | other.page_mask | 0x1FFF; + (new.entry_hi ^ other.entry_hi) & !both & 0xC000_00FF_FFFF_E000 == 0 + } + + /// Random writes against a plain list that applies the rule by comparing + /// every pair. The lookup map is how a write finds out it overlaps + /// nothing, so the TLB must agree with the list after every write, with + /// regular page masks and with masks that are not a run of ones. + #[test] + fn writes_remove_exactly_the_overlapped_entries() { + std::thread::Builder::new().stack_size(64 << 20).spawn(|| { + for masks in [&[0u64, 0x6000, 0x1e000, 0x7e000, 0x1fe000, 0x7fe000, 0x1ffe000][..], + &[0u64, 0x6000, 0x2000, 0xa000, 0x1fe000, 0x1ffe000][..]] { + let mut rng = 0x9e37_79b9_7f4a_7c15u64; + let mut next = move || { rng ^= rng << 13; rng ^= rng >> 7; rng ^= rng << 17; rng }; + let mut tlb = MipsTlb::new(64); + tlb.set_conflict_invalidation(true); + let mut model: Vec> = vec![None; 64]; + for _ in 0..4000 { + let index = (next() % 64) as usize; + let mask = masks[(next() % masks.len() as u64) as usize]; + let region = if next() % 4 == 0 { 0x8000_0000_0000_0000 } else { BASE }; + // Few page numbers, so that writes collide often. + let page = (next() % 24) << 13; + let asid = next() % 3; + let global = if next() % 3 == 0 { 0x1000 } else { 0 }; + let new = entry(mask, region | page | global | asid, [(next() % 8) << 1; 2]); + let mut expect_conflict = false; + for (i, slot) in model.iter_mut().enumerate() { + if i != index && slot.as_ref().is_some_and(|o| overlaps(&new, o)) { + let mut gone = TlbEntry::new(); + gone.page_mask = slot.as_ref().unwrap().page_mask; + *slot = Some(gone); + expect_conflict = true; + } + } + model[index] = Some(new); + tlb.write(index, new); + assert_eq!(tlb.take_write_conflict(), expect_conflict); + for (i, slot) in model.iter().enumerate() { + let got = tlb.read(i); + let want = slot.unwrap_or_else(TlbEntry::new); + assert_eq!((got.entry_hi, got.entry_lo, got.page_mask), + (want.entry_hi, want.entry_lo, want.page_mask), "entry {i}"); + } + } + } + }).unwrap().join().unwrap(); + } + #[test] fn test_tlb_entry_flags() { let mut entry = TlbEntry::new(); From cd96de49173adc30cef282610f65cd5e2d522e3d Mon Sep 17 00:00:00 2001 From: James Braid Date: Sat, 31 Oct 2026 09:34:16 -0700 Subject: [PATCH 06/31] cpu: complete refused uncached stores on the R10000 family A store that nothing answers raised a synchronous Data Bus Error with a precise PC on every CPU. The R10000 cannot do that. It sends an uncached store as a write request, and a write request gets no response; Bus Error comes only from an error response to a read or upgrade request, or from uncorrectable read data (R10000 UM pp. 104, 316). The processor never learns that a store failed. The R12000 manual documents the same no-response write path, and the later parts are taken to match, so the test is the R10000 family and not the R10000 alone. Add CpuModel::POSTS_UNCACHED_STORES, true for the R10000 family's caches. On such a model a refused store logs and completes in write_data_impl and in the masked-store path (SWL, SWR, SDL, SDR), which the interpreter and the compiled code both go through. An uncached SC or SCD that is refused completes and succeeds (rt = 1); that is a model choice, since the manual says nothing about uncached store-conditional. Loads, instruction fetches, cached accesses and the R4400 and R5000 keep the exception, and device return values are untouched because BUS_ERR also tells non-CPU callers that a request was declined. The error goes no further. Nothing in iris raises an interrupt or sets an error register for a store the Octane's bus refuses. The IP22/IP28 bus already completes a write to an unanswered address, latching MC CPU_ERROR/GIO_ERROR and raising no interrupt, so the stores that change here are the ones a device refuses itself. The log line stays visible ("Unsupported uncached write ... posted, completes") but a probe loop must not flood stderr: it prints the first four refused stores and then each power of two, with the running count. The Octane's ide diagnostics store 0x591204 and then 0x4 to physical 0x1e130400, where nothing answers. Before, the first store took a Data Bus Error; now both print one line and the run carries on. Tests: the interpreter on the R10000 (full and shadow caches) and the R5000 for SW, SWL and SC plus the load that still faults; the JIT against the interpreter for SW and SWL on the R10000 and for the load and the R5000 store that fault; the report schedule. cargo test --release --features jitv2 --lib: 2422 passed, 0 failed. --- CHANGELOG.md | 8 +++++ src/cpu/jitv2/equiv_test.rs | 70 ++++++++++++++++++++++++++++++++++++ src/cpu/mips_cache_shadow.rs | 1 + src/cpu/mips_cache_v2.rs | 9 +++++ src/cpu/mips_exec.rs | 47 +++++++++++++++++++++--- src/cpu/mips_exec_test.rs | 47 ++++++++++++++++++++++++ 6 files changed, 178 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a2332d721..81386bf4e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -139,6 +139,14 @@ hashes are given where a change is easiest to understand by reading the commit. refills through 0x000, as the part does, not 0x080. The R4400 and R5000 still choose by the current mode. +### Refused uncached stores on the R10000 family + +- On the R10000 family (Octane, IP28, O2), an uncached store that nothing answers now + completes instead of raising a Data Bus Error, as the processor does. iris prints + "Unsupported uncached write ... (posted, completes)" for the first four such + stores and then at each power of two. Loads, instruction fetches, the R4400 and + the R5000 are unchanged. + ### `IRIS_GUI_GL` works in iris-gui - The GUI's GL capture path (`IRIS_GUI_GL=1`, or `[debug] gui_gl_capture`) never diff --git a/src/cpu/jitv2/equiv_test.rs b/src/cpu/jitv2/equiv_test.rs index 315eaa286..114dd3e47 100644 --- a/src/cpu/jitv2/equiv_test.rs +++ b/src/cpu/jitv2/equiv_test.rs @@ -7879,4 +7879,74 @@ mod tests { println!("{:<12} {:8.3} ns/instruction", name, ns); } } + + /// R10000: a refused uncached store completes in compiled code exactly as + /// in the interpreter, through both store sites (SW, and SWL by the + /// masked-store helper), and the instructions after it run. The same access + /// as a load, and a store on the R5000, still fault in both engines. + #[test] + fn refused_uncached_store_matches_the_interpreter_on_the_r10000_and_r5000() { + use crate::cpu::mips_cache_v2::{R10000Cache, R5000Cache, CpuModel}; + use crate::cpu::mips_core::STATUS_EXL; + use crate::cpu::mips_isa::{OP_ADDIU, OP_LW, OP_SW, OP_SWL}; + + /// Nothing answers: every read and write is refused (the trait's defaults). + struct RefusingBus; + impl BusDevice for RefusingBus {} + + fn run>>(op: u32) -> (CoreSnapshot, CoreSnapshot) { + let pc = 0xFFFF_FFFF_8000_0000u64; + let mut gpr = [0u64; 32]; + gpr[16] = 0xFFFF_FFFF_A100_0000; // kseg1: uncached, and nothing answers + gpr[6] = 0x1234_5678; + let words = [make_i(op, 16, 6, 0), make_i(OP_ADDIU, 0, 7, 2)]; + let build = || { + let bus: Arc = Arc::new(RefusingBus); + let cfg = MipsCpuConfig::for_model::(Arc::new(crate::ip22_bus::memory_map(false))); + let mut exec = Box::new(MipsExecutor::::new(bus, PassthroughTlb::default(), &cfg)); + exec.core.gpr = gpr; + exec.core.pc = pc; + exec.core.hot.interrupts.store(0, std::sync::atomic::Ordering::Relaxed); + #[cfg(feature = "fetchverify")] + { exec.core.fetch_verify_on = false; } + exec + }; + let mut interp = build(); + interp.jitv2_dispatch_enabled = false; + for (i, &word) in words.iter().enumerate() { + if interp.core.pc != pc + 4 * i as u64 { + break; // an exception redirected the PC + } + interp.exec(word); + } + + let mut page = [0u32; ENTRIES_PER_PAGE]; + page[..words.len()].copy_from_slice(&words); + let page_base = (pc & !(PAGE_SIZE as u64 - 1)) as u32; + let mut analyzer = Analyzer::new(); + let (walked, non_empty) = analyzer.walk_bounded(&page, 0, page_base, words.len()); + assert!(non_empty); + let mut instrs_owned = *walked; + let mut codegen = Codegen::new(); + let jit_fn: JitFn = codegen.compile_region(&mut instrs_owned, 0, true, false) + .expect("region must be compilable for this test to be meaningful"); + let mut jit = build(); + jit.install_jit_hooks(); + unsafe { jit_fn(&mut jit.core as *mut MipsCore) }; + std::mem::forget(codegen); + (CoreSnapshot::capture(&interp.core), CoreSnapshot::capture(&jit.core)) + } + + for op in [OP_SW, OP_SWL] { + let (interp, jit) = run::(op); + assert_eq!(jit, interp, "op {op:#x}: JIT and interpreter diverged"); + assert_eq!(jit.cp0_status & STATUS_EXL, 0, "op {op:#x}: no exception"); + assert_eq!(jit.gpr[7], 2, "op {op:#x}: the instruction after the store ran"); + } + for (name, (interp, jit)) in [("R10000 load", run::(OP_LW)), ("R5000 store", run::(OP_SW))] { + assert_eq!(jit, interp, "{name}: JIT and interpreter diverged"); + assert_ne!(jit.cp0_status & STATUS_EXL, 0, "{name}: must take its Data Bus Error"); + assert_eq!(jit.gpr[7], 0, "{name}: nothing after the fault ran"); + } + } } diff --git a/src/cpu/mips_cache_shadow.rs b/src/cpu/mips_cache_shadow.rs index 73ab609ba..7047f78ef 100644 --- a/src/cpu/mips_cache_shadow.rs +++ b/src/cpu/mips_cache_shadow.rs @@ -349,6 +349,7 @@ impl< const CACHE_KIND: &'static str = "shadow"; const R10K_FAMILY: bool = Self::R10K_CACHE_OPS; const R10K_CACHE_OPS: bool = MODEL == model::R10000; + const POSTS_UNCACHED_STORES: bool = Self::R10K_CACHE_OPS; } impl< diff --git a/src/cpu/mips_cache_v2.rs b/src/cpu/mips_cache_v2.rs index 8b6b49e15..fd541c82b 100644 --- a/src/cpu/mips_cache_v2.rs +++ b/src/cpu/mips_cache_v2.rs @@ -571,6 +571,14 @@ pub trait CpuModel: MipsCache { /// list of changes from the R10000 has no TLB or CP0 change. The R14000 and /// R16000 are a model choice that gives them the same TLB. const R10K_FAMILY: bool = false; + /// An uncached store the system refuses completes instead of raising a + /// Data Bus Error. The R10000 sends it as a write request, which gets no + /// response, and Bus Error comes only from an error response to a read or + /// upgrade request or from uncorrectable read data (R10000 UM pp. 104, 316). + /// The R12000 manual documents the same path and the later parts are taken + /// to match. An uncached SC succeeding is a model choice. Loads and fetches + /// still fault. + const POSTS_UNCACHED_STORES: bool = false; } pub trait MipsCache: Send + Sync { @@ -3603,6 +3611,7 @@ impl() -> u64 { 0xC000_0000_0000_00FF | crate::cpu::mips_tlb::vpn2_mask(C::VA_BITS) } +/// Whether the `n`th refused store is reported: the first four, then each power of two. +pub(crate) fn report_refused_store(n: u64) -> bool { + n <= 4 || n.is_power_of_two() +} + /// Per-access-class tally of how far a guest load/store would get through the /// Per-access-class tally of how far a guest load/store would get through the /// checks the JIT's inline memory path emits, before falling back to Rust. @@ -859,6 +864,9 @@ pub struct MipsExecutor { pub decoded_count: Arc, /// Count of instructions fetched from uncached address space. pub uncached_fetch_count: Arc, + /// Refused uncached stores so far (see `CpuModel::POSTS_UNCACHED_STORES`); + /// sets how often `post_refused_store` reports. + refused_stores: u64, /// Hot-path translation function pointer, updated whenever CP0 Status changes. /// Always the non-debug variant; selects the correct 32/64-bit × privilege specialisation. pub translate_fn: fn(&mut Self, u64, AccessType) -> TranslateResult, @@ -2199,6 +2207,22 @@ impl MipsExecutor { self.sysad.describe_fault(phys_addr, size).map_or_else(String::new, |why| format!(" [{why}]")) } + /// An uncached store the system refused, on a model that posts stores + /// (`CpuModel::POSTS_UNCACHED_STORES`): say so and let the store complete. + /// The caller returns `BUS_OK`. Always built, so a missing device or a + /// guest bug stays visible in every build. A probe loop would flood + /// stderr, so it reports the first four and then every power of two, with + /// the running count. + #[cold] + fn post_refused_store(&mut self, virt_addr: u64, phys_addr: u64, size: usize, val: u64, status: u32) { + self.refused_stores += 1; + let n = self.refused_stores; + if report_refused_store(n) { + eprintln!("Unsupported uncached write{} (posted, completes): PC={:016x} VA={:016x} PA={:016x} val={:016x} status={:08x}{} [store {n}]", + size * 8, self.core.pc, virt_addr, phys_addr, val, status, self.fault_note(phys_addr, size)); + } + } + /// Create a new executor from a config and a bus (sysad) and a TLB. /// The cache hierarchy is constructed internally as a unified R4000Cache. pub fn new(sysad: Arc, mut tlb: T, cfg: &MipsCpuConfig) -> Self @@ -2397,6 +2421,7 @@ impl MipsExecutor { ins: DecodedInstr::default(), // scratch slot for uncached fetches decoded_count: Arc::new(AtomicU64::new(0)), uncached_fetch_count: Arc::new(AtomicU64::new(0)), + refused_stores: 0, // Placeholder — overwritten immediately by update_translate_fn below. translate_fn: translate_32_kernel::, // Placeholder — overwritten immediately by update_fpr_mode below. @@ -5114,9 +5139,16 @@ va={:#018x} phys={:#010x} (code pfn {:#x}, page {:#010x}, word {}/{})", self.sysad.write64(phys_addr, val) }; if ws != BUS_OK && ws != BUS_BUSY && !DEBUG { - eprintln!("Bus error on uncached write{}: PC={:016x} VA={:016x} PA={:016x} val={:016x} status={:08x}{}", SIZE*8, self.core.pc, virt_addr, phys_addr, val, ws, self.fault_note(phys_addr, SIZE)); + if C::POSTS_UNCACHED_STORES { + self.post_refused_store(virt_addr, phys_addr, SIZE, val, ws); + BUS_OK + } else { + eprintln!("Bus error on uncached write{}: PC={:016x} VA={:016x} PA={:016x} val={:016x} status={:08x}{}", SIZE*8, self.core.pc, virt_addr, phys_addr, val, ws, self.fault_note(phys_addr, SIZE)); + ws + } + } else { + ws } - ws }; // jitv2_lockstep's load/store verification (see MipsCore's // lockstep_mem field doc comment, and read_data_impl's matching @@ -5170,9 +5202,16 @@ va={:#018x} phys={:#010x} (code pfn {:#x}, page {:#010x}, word {}/{})", } let ws = self.sysad.write64_masked(phys_addr, val, mask); if ws != BUS_OK && ws != BUS_BUSY && !DEBUG { - eprintln!("Bus error on uncached write64_masked: PC={:016x} VA={:016x} PA={:016x} val={:016x} mask={:016x} status={:08x}{}", self.core.pc, virt_addr, phys_addr, val, mask, ws, self.fault_note(phys_addr, 8)); + if C::POSTS_UNCACHED_STORES { + self.post_refused_store(virt_addr, phys_addr, mask.count_ones() as usize / 8, val, ws); + BUS_OK + } else { + eprintln!("Bus error on uncached write64_masked: PC={:016x} VA={:016x} PA={:016x} val={:016x} mask={:016x} status={:08x}{}", self.core.pc, virt_addr, phys_addr, val, mask, ws, self.fault_note(phys_addr, 8)); + ws + } + } else { + ws } - ws } } diff --git a/src/cpu/mips_exec_test.rs b/src/cpu/mips_exec_test.rs index afc15cf7b..66851a71d 100644 --- a/src/cpu/mips_exec_test.rs +++ b/src/cpu/mips_exec_test.rs @@ -6735,4 +6735,51 @@ mod tests { check::(true); }); } + + /// A bus where nothing answers: every read and write is refused (the + /// trait's defaults). + struct RefusingBus; + impl BusDevice for RefusingBus {} + + /// R10000: a store the system refuses leaves the processor as a write + /// request, which has no response, so it completes (R10000 UM pp. 104, + /// 316). An uncached SC succeeding is a model choice. A load keeps its + /// Data Bus Error, and so does every access on the R5000. SWL goes through + /// the masked-store path, a separate site. + #[test] + fn refused_uncached_store_completes_only_on_the_r10000() { + fn run>>(op: u32) -> (u32, u64) { + let bus: Arc = Arc::new(RefusingBus); + let mut exec = MipsExecutor::::new(bus, PassthroughTlb::default(), &MipsCpuConfig::indy()); + disable_jitv2(&mut exec); + exec.cache.set_llbit(true); + exec.core.write_gpr(4, 0xFFFF_FFFF_A100_0000); // kseg1: uncached + exec.core.write_gpr(6, 0x1234_5678); + let st = exec.exec(make_i(op, 4, 6, 4)); + if st == EXEC_COMPLETE { + assert_eq!(exec.core.cp0_status & crate::cpu::mips_core::STATUS_EXL, 0, "no exception taken"); + assert_eq!(exec.exec(make_i(OP_ADDIU, 0, 5, 7)), EXEC_COMPLETE); + assert_eq!(exec.core.read_gpr(5), 7, "the next instruction runs"); + } + (st, exec.core.read_gpr(6)) + } + use crate::traits::BUS_ERR; + type R10K = crate::cpu::mips_cache_v2::R10000Cache; + type R10KShadow = crate::cpu::mips_cache_shadow::R10000ShadowCache; + type R5K = crate::cpu::mips_cache_shadow::R5000ShadowCache; + for (op, name) in [(OP_SW, "SW"), (OP_SWL, "SWL")] { + assert_eq!(run::(op).0, EXEC_COMPLETE, "R10000 {name}"); + assert_eq!(run::(op).0, EXEC_COMPLETE, "R10000 shadow {name}"); + assert_eq!(run::(op).0, BUS_ERR, "R5000 {name}"); + } + assert_eq!(run::(OP_SC), (EXEC_COMPLETE, 1), "SC succeeds"); + assert_eq!(run::(OP_SC).0, BUS_ERR, "R5000 SC"); + assert_eq!(run::(OP_LW).0, BUS_ERR, "a refused load still faults"); + } + + #[test] + fn refused_store_reports_are_sparse() { + let shown: Vec = (1..=40).filter(|&n| crate::cpu::mips_exec::report_refused_store(n)).collect(); + assert_eq!(shown, [1, 2, 3, 4, 8, 16, 32]); + } } From f02b8b67bdceb0fbad8a5891b60d98504fa17559 Mon Sep 17 00:00:00 2001 From: James Braid Date: Fri, 30 Oct 2026 05:16:09 -0700 Subject: [PATCH 07/31] cpu: match the O2 memory alias after the physical address mask A mapped page whose physical address has a bit set above bit 31 could reach the PCI window instead of RAM on the O2. The O2 aliases RAM at 0x8000_0000, and fold_phys tested that window on the full translated address before cutting it to the 32 bits the board decodes. IRIX keeps software bits in its page-table entries just above the PFN, so such an address missed the window, then lost the high bit in the mask and landed on the PCI range at 0x8000_0000. Cut to the decoded bits first, then apply the alias, then cut its target. The window lies within the decoded bits, so a page already inside it lands where it did. The extra AND runs only on an alias hit. Only the O2 has a physical alias; the Indy, Indigo2 and Octane are unchanged. A test maps a page at 0x1_8000_0000 with the O2's mask and window and expects RAM at 0. It reads 0x8000_0000 without the fix. --- CHANGELOG.md | 3 +++ src/cpu/mips_exec.rs | 18 ++++++++++-------- src/cpu/mips_exec_test.rs | 25 +++++++++++++++++++++++++ 3 files changed, 38 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 81386bf4e..f55476cc5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -546,6 +546,9 @@ hashes are given where a change is easiest to understand by reading the commit. ### SGI O2 (IP32) +- **The O2's memory alias holds for pages with high PTE bits set.** A mapped page + whose address had a bit set above bit 31 missed the alias of RAM at + 0x8000_0000 and reached the PCI window instead. - **The O2 runs on the shadow cache from reset.** The full cache model, and the switch to and from it during POST and reboots, are gone (`src/ip32_handover.rs`), and with them a branch on every cached load, store diff --git a/src/cpu/mips_exec.rs b/src/cpu/mips_exec.rs index e95f1cb2c..ac09850a5 100644 --- a/src/cpu/mips_exec.rs +++ b/src/cpu/mips_exec.rs @@ -1037,12 +1037,12 @@ pub struct MipsExecutor { pub batch_hook: Option) + Send>>, /// A window of physical addresses that reaches memory at another address, /// `(base, size, target)`, applied to mapped and XKPHYS translations - /// before [`Self::phys_mask`]. The O2 has a no-ECC alias of its memory at + /// after [`Self::phys_mask`]. The O2 has a no-ECC alias of its memory at /// 0x8000_0000 that would otherwise reach the PCI window the bus decodes /// there. None on the other machines. pub phys_alias: Option<(u64, u64, u64)>, /// The physical address bits the board decodes, applied to mapped, - /// XKPHYS and ERL-identity translations after [`Self::phys_alias`]. The + /// XKPHYS and ERL-identity translations before [`Self::phys_alias`]. The /// CPU drives 36 or 40 bits, but the Indy, Indigo2 and O2 decode 32 and /// alias the rest away, and IRIX relies on it: its page-table entries keep /// software bits (the PFN lock among them) just above the PFN, so they @@ -4648,15 +4648,17 @@ va={:#018x} phys={:#010x} (code pfn {:#x}, page {:#010x}, word {}/{})", } } - /// A CPU physical address as the board decodes it: through - /// [`Self::phys_alias`] if it falls in it, then cut to [`Self::phys_mask`]. + /// A CPU physical address as the board decodes it: cut to + /// [`Self::phys_mask`], then through [`Self::phys_alias`] if it falls in it. #[inline(always)] fn fold_phys(&self, pa: u64) -> u64 { - let pa = match self.phys_alias { - Some((base, size, target)) if pa.wrapping_sub(base) < size => pa - base + target, + // Cut first: the PTE software bits above the PFN are not part of the + // address the alias window sees. Window and target lie within the mask. + let pa = pa & self.phys_mask; + match self.phys_alias { + Some((base, size, target)) if pa.wrapping_sub(base) < size => (pa - base + target) & self.phys_mask, _ => pa, - }; - pa & self.phys_mask + } } /// TLB translation. When `DEBUG` is true, CP0 side-effect registers are diff --git a/src/cpu/mips_exec_test.rs b/src/cpu/mips_exec_test.rs index 66851a71d..2b7fa23ee 100644 --- a/src/cpu/mips_exec_test.rs +++ b/src/cpu/mips_exec_test.rs @@ -6782,4 +6782,29 @@ mod tests { let shown: Vec = (1..=40).filter(|&n| crate::cpu::mips_exec::report_refused_store(n)).collect(); assert_eq!(shown, [1, 2, 3, 4, 8, 16, 32]); } + + /// The O2's no-ECC alias of RAM at 0x8000_0000 is matched on the address the + /// board decodes (32 bits), so a PTE software bit above the PFN cannot make + /// a mapped page miss the alias window. + #[test] + fn phys_alias_is_matched_after_the_physical_mask() { + std::thread::Builder::new() + .stack_size(16 * 1024 * 1024) + .spawn(|| { + use crate::cpu::mips_tlb::MipsTlb; + let (mut exec, _) = create_executor_with_tlb(MipsTlb::default()); + exec.core.cp0_status &= !crate::cpu::mips_core::STATUS_ERL; + exec.phys_mask = u32::MAX as u64; + exec.phys_alias = Some((0x8000_0000, 0x1000_0000, 0)); + // PA 0x1_8000_0000: bit 32 is above what the board decodes. + exec.core.cp0_index = 0; + exec.core.cp0_pagemask = 0; + exec.core.cp0_entryhi = 0x4000_0000; + exec.core.cp0_entrylo0 = (0x18_0000 << 6) | (3 << 3) | (1 << 2) | (1 << 1); + exec.core.cp0_entrylo1 = 0; + assert_eq!(exec.exec((OP_COP0 << 26) | (0x10 << 21) | 0x02), EXEC_COMPLETE); + assert_eq!(exec.debug_translate(0x4000_0000).phys, 0); + }) + .unwrap().join().unwrap(); + } } From 6612684ceb15cc0eff07e48f0e326b3204183479 Mon Sep 17 00:00:00 2001 From: James Braid Date: Fri, 30 Oct 2026 08:03:24 +0000 Subject: [PATCH 08/31] cpu: keep the upper word on a partial LWR on the R10000 In 64-bit mode a partial LWR (offset 0 to 2) sign-extended the merged word on every CPU. The manual calls the upper word of a partial LWR implementation specific (NEC U12754E pp. 125-126), and the reference results observed for the R10000 keep the old bits 63:32 at all three partial offsets. With sign extension a test that compares those results miscompares. The interpreter and the JIT now merge the old upper word on the R10000; a full word (offset 3) still sign-extends. The upper word is kept only when 64-bit operations are enabled for the current mode, matching QEMU; otherwise it sign-extends. The later parts are not covered, since nothing shows what they do. The choice is a JitConsts field set from CpuModel::PARTIAL_LWR_KEEPS_UPPER, so compile workers see it. Whether 64-bit operations are on is a byte in the core that the executor sets wherever it re-derives the privilege state, so a compiled LWR spends a byte load and a compare on it rather than decoding Status; the merge runs on the whole register and needs no more work than the old sign-extending sequence. The code-cache fingerprint gains lwr_upper=1 only when it is set, so the R4400 and R5000 machines keep their warm caches. Behaviour changes on the Octane, IP28 and an O2 with the R10000. The R4400 and R5000 are unchanged. Tests run LWR at every offset in the interpreter on the R4400, R5000 and R10000 in kernel, supervisor and user mode with the mode's own 64-bit bit set and clear, run the same offsets as compiled code on the R10000 and R5000 and compare with the interpreter, and check the fingerprint. --- CHANGELOG.md | 8 +++ src/cpu/jitv2/codegen.rs | 57 +++++++++++++++++++ src/cpu/jitv2/equiv_test.rs | 103 +++++++++++++++++++++++++++++++++++ src/cpu/jitv2/mod.rs | 1 + src/cpu/mips_cache_shadow.rs | 1 + src/cpu/mips_cache_v2.rs | 5 ++ src/cpu/mips_core.rs | 7 +++ src/cpu/mips_exec.rs | 18 +++++- src/cpu/mips_exec_test.rs | 96 ++++++++++++++++++++++++++++++++ 9 files changed, 294 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f55476cc5..21db22d3a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -147,6 +147,14 @@ hashes are given where a change is easiest to understand by reading the commit. stores and then at each power of two. Loads, instruction fetches, the R4400 and the R5000 are unchanged. +### Partial LWR on the R10000 + +- On the R10000 (Octane, IP28, and an O2 with the R10000), an `LWR` that loads fewer + than four bytes now keeps bits 63:32 of the destination instead of sign-extending, + as the reference results for the part show. R4400 and R5000 machines are unchanged. + The JIT matches the interpreter, and the code-cache key changes only for the + machines that have the behaviour. + ### `IRIS_GUI_GL` works in iris-gui - The GUI's GL capture path (`IRIS_GUI_GL=1`, or `[debug] gui_gl_capture`) never diff --git a/src/cpu/jitv2/codegen.rs b/src/cpu/jitv2/codegen.rs index ee31f90cf..ce266af9e 100644 --- a/src/cpu/jitv2/codegen.rs +++ b/src/cpu/jitv2/codegen.rs @@ -868,6 +868,10 @@ impl Codegen { for v in isa.isa_flags() { let _ = write!(s, " {v}"); } + // Only when set, so machines without the behaviour keep their warm cache. + if self.jit_consts.lwr_keeps_upper { + s.push_str(" lwr_upper=1"); + } s.push_str(" ops="); s.extend(crate::cpu::jitv2::opcode_support::enabled_snapshot().iter().map(|&b| if b { '1' } else { '0' })); Some(blake3::hash(s.as_bytes()).as_bytes()[..16].try_into().unwrap()) @@ -4379,6 +4383,11 @@ pub struct JitConsts { /// to loading from memory, exactly as before, so an un-stamped `Codegen` /// stays correct (just no faster). pub core: Option, + /// A partial LWR keeps bits 63:32 of the destination, as the R10000 + /// does (`CpuModel::PARTIAL_LWR_KEEPS_UPPER`). A model fact rather than a + /// core one, published beside the core address because both reach a worker's + /// `Codegen` the same way. The code-cache fingerprint carries it when set. + pub lwr_keeps_upper: bool, } impl JitConsts { @@ -10013,6 +10022,37 @@ fn emit_lwl_ldl(ctx: &mut EmitCtx, size: MemSize) { /// `MipsExecutor::exec_lwr`/`exec_ldr` exactly: shift amount is /// `(width-1-byte_offset)*8`, the opposite sense from LWL/LDL's plain /// `byte_offset*8`. +/// LWR on a model whose partial LWR keeps rt[63:32] (`lwr_keeps_upper`), +/// mirroring `MipsExecutor::exec_lwr`. The merge runs on the whole register, +/// so rt's upper word survives it (`!mask` has ones there). Offsets 0 to 2 +/// keep it while 64-bit operations are on; offset 3 and any word with them +/// off sign-extends the merged low half. The executor keeps +/// `core.lwr_upper_below` for that test, 3 while 64-bit operations are on +/// and 0 otherwise, so the choice is one unsigned compare of the offset with +/// a byte (a Status change inside a region, a fallback MTC0 or ERET, +/// resyncs it before the next instruction runs). `loaded` is the aligned +/// word, zero-extended. +fn emit_lwr_keeping_upper(ctx: &mut EmitCtx, loaded: Value, byte_offset: Value) { + let i64t = ir::types::I64; + // Bytes of the word the register keeps: 8 * (3 - offset). + let inverted = ctx.builder.ins().bxor_imm_s(byte_offset, 3); + let shift = ctx.builder.ins().ishl_imm_s(inverted, 3); + let ones = ctx.builder.ins().iconst(i64t, 0xFFFF_FFFF); + let mask = ctx.builder.ins().ushr(ones, shift); + let shifted_mem = ctx.builder.ins().ushr(loaded, shift); + let rt_val = emit_read_gpr(ctx, field_rt(ctx.raw)); + let not_mask = ctx.builder.ins().bnot(mask); + let preserved = ctx.builder.ins().band(rt_val, not_mask); + let merged = ctx.builder.ins().bor(shifted_mem, preserved); + let low = ctx.builder.ins().ireduce(ir::types::I32, merged); + let sign_extended = ctx.builder.ins().sextend(i64t, low); + let below_off = ir::immediates::Offset32::new(std::mem::offset_of!(MipsCore, lwr_upper_below) as i32); + let below = ctx.builder.ins().uload8(i64t, MemFlagsData::trusted(), ctx.core_ptr, below_off); + let keep = ctx.builder.ins().icmp(IntCC::UnsignedLessThan, byte_offset, below); + let result = ctx.builder.ins().select(keep, merged, sign_extended); + emit_write_gpr(ctx, field_rt(ctx.raw), result); +} + fn emit_lwr_ldr(ctx: &mut EmitCtx, size: MemSize) { let base = emit_read_gpr(ctx, field_rs(ctx.raw)); let imm = field_imm16_sext(ctx.builder, ctx.raw); @@ -10024,6 +10064,11 @@ fn emit_lwr_ldr(ctx: &mut EmitCtx, size: MemSize) { let loaded = emit_mem_read(ctx, aligned_addr, size); + if size == MemSize::B4 && ctx.jit_consts.lwr_keeps_upper { + emit_lwr_keeping_upper(ctx, loaded, byte_offset); + return; + } + let ity = size.ir_type(); let mem_val = if size == MemSize::B8 { loaded } else { ctx.builder.ins().ireduce(ity, loaded) }; let rt_val = emit_read_gpr(ctx, field_rt(ctx.raw)); @@ -10811,6 +10856,18 @@ mod tests { (op << 26) | (rs << 21) | (rt << 16) | (rd << 11) | (sa << 6) | funct } + #[test] + fn cache_fingerprint_includes_the_lwr_choice() { + let base = Codegen::new().cache_fingerprint(false).expect("default codegen is cacheable"); + let mut codegen = Codegen::new(); + codegen.jit_consts.lwr_keeps_upper = true; + assert_ne!( + codegen.cache_fingerprint(false).expect("lwr codegen is cacheable"), + base, + "the LWR choice must select a different persistent cache namespace", + ); + } + /// `is_inlinable` must reject on *both* of its conditions, not just the /// bounds one. The bounds half is what panicked /// (`clanker/jitv2/nested-delay-slot-across-page-boundary.md`); the diff --git a/src/cpu/jitv2/equiv_test.rs b/src/cpu/jitv2/equiv_test.rs index 114dd3e47..24a631f3f 100644 --- a/src/cpu/jitv2/equiv_test.rs +++ b/src/cpu/jitv2/equiv_test.rs @@ -4355,6 +4355,109 @@ mod tests { } } + /// Run `words` from `pc` once through the interpreter and once as a + /// compiled region, both on CPU model `C` over a fresh `bus()` and seeded + /// with `gpr` and CP0 Status `status`. The JIT learns the model's choices from `install_jit_hooks`, + /// as in production. + fn interpreter_and_jit(bus: impl Fn() -> Arc, pc: u64, gpr: [u64; 32], status: u32, words: &[u32]) + -> (CoreSnapshot, CoreSnapshot) + where + C: crate::cpu::mips_cache_v2::CpuModel + From>, + { + let build = || { + let cfg = MipsCpuConfig::for_model::(Arc::new(crate::ip22_bus::memory_map(false))); + let mut exec = Box::new(MipsExecutor::::new(bus(), PassthroughTlb::default(), &cfg)); + exec.core.gpr = gpr; + exec.core.pc = pc; + exec.set_cp0_status(status); + exec.core.hot.interrupts.store(0, std::sync::atomic::Ordering::Relaxed); + #[cfg(feature = "fetchverify")] + { exec.core.fetch_verify_on = false; } + exec + }; + let mut interp = build(); + interp.jitv2_dispatch_enabled = false; + for (i, &word) in words.iter().enumerate() { + if interp.core.pc != pc + 4 * i as u64 { + break; // an exception redirected the PC + } + interp.exec(word); + } + + let mut page = [0u32; ENTRIES_PER_PAGE]; + page[..words.len()].copy_from_slice(words); + let page_base = (pc & !(PAGE_SIZE as u64 - 1)) as u32; + let mut analyzer = Analyzer::new(); + let (walked, non_empty) = analyzer.walk_bounded(&page, 0, page_base, words.len()); + assert!(non_empty); + let mut instrs_owned = *walked; + let mut codegen = Codegen::new(); + codegen.jit_consts.lwr_keeps_upper = C::PARTIAL_LWR_KEEPS_UPPER; + let jit_fn: JitFn = codegen.compile_region(&mut instrs_owned, 0, true, false) + .expect("region must be compilable for this test to be meaningful"); + let mut jit = build(); + jit.install_jit_hooks(); + // The compile pool's workers read the model's choice from here, so a + // missing publication would not show if `codegen` were stamped by hand. + assert_eq!(jit.jitv2.lock().jit_consts.lock().lwr_keeps_upper, C::PARTIAL_LWR_KEEPS_UPPER); + unsafe { jit_fn(&mut jit.core as *mut MipsCore) }; + std::mem::forget(codegen); + (CoreSnapshot::capture(&interp.core), CoreSnapshot::capture(&jit.core)) + } + + /// The R10000 keeps bits 63:32 on a partial LWR while 64-bit operations are + /// enabled for the current mode and the R5000 sign-extends; with them off + /// (a 32-bit user process, UX=0) the R10000 sign-extends too. The JIT and + /// the interpreter must agree at every offset in every mode, and the + /// interpreter's result is the one `partial_lwr_keeps_the_upper_word_only_on_the_r10000` + /// pins. + #[test] + fn jit_lwr_matches_the_interpreter_on_the_r10000_and_r5000() { + use crate::cpu::mips_cache_shadow::{R10000ShadowCache, R5000ShadowCache}; + fn check>>(model_keeps_upper: bool) { + use crate::cpu::mips_core::{STATUS_ERL, STATUS_EXL, STATUS_KX, STATUS_SX, STATUS_UX}; + const RESERVED: u32 = 3 << 3; + const USER: u32 = 2 << 3; + const SUPER: u32 = 1 << 3; + // (Status, whether 64-bit operations are on): each mode's own + // bit decides, whatever the other two hold. + for (status, ops64, mode) in [ + (STATUS_KX, true, "kernel KX=1"), + (0, false, "kernel KX=0"), + (STATUS_UX | STATUS_SX, false, "kernel KX=0, UX/SX=1"), + (USER | STATUS_UX, true, "user UX=1"), + (USER | STATUS_KX | STATUS_SX, false, "user UX=0, KX/SX=1"), + (SUPER | STATUS_SX, true, "supervisor SX=1"), + (SUPER | STATUS_KX | STATUS_UX, false, "supervisor SX=0"), + (USER | STATUS_EXL | STATUS_KX, true, "EXL forces kernel, KX=1"), + (USER | STATUS_EXL | STATUS_UX, false, "EXL forces kernel, UX=1 KX=0"), + (SUPER | STATUS_ERL | STATUS_KX, true, "ERL forces kernel, KX=1"), + (SUPER | STATUS_ERL | STATUS_SX, false, "ERL forces kernel, SX=1 KX=0"), + (RESERVED | STATUS_KX, true, "reserved KSU counts as kernel, KX=1"), + (RESERVED | STATUS_UX | STATUS_SX, false, "reserved KSU counts as kernel, KX=0"), + ] { + for offset in 0..4u64 { + let mut gpr = [0u64; 32]; + gpr[1] = 0x2000 + offset; + gpr[2] = 0x1122_3344_5566_7788; + let bus = || -> Arc { + let mem = Arc::new(MockMemory::new_not_compilable()); + mem.set_word(0x2000, 0xDEAD_BEEF); + mem + }; + let lwr = make_i(crate::cpu::mips_isa::OP_LWR, 1, 2, 0); + let (interp, jit) = interpreter_and_jit::(bus, 0x1000, gpr, status, &[lwr]); + assert_eq!(jit, interp, "{} JIT/interpreter LWR offset {offset}, {mode}", C::NAME); + let upper_kept = interp.gpr[2] >> 32 == 0x1122_3344; + assert_eq!(upper_kept, model_keeps_upper && ops64 && offset != 3, + "{} LWR offset {offset}, {mode}", C::NAME); + } + } + } + check::(true); + check::(false); + } + #[test] fn lwl_lwr_combine_to_a_full_unaligned_word_load_matches_interpreter() { // The canonical compiler-generated idiom for an unaligned word diff --git a/src/cpu/jitv2/mod.rs b/src/cpu/jitv2/mod.rs index 60e01f749..6aeeb5b6e 100644 --- a/src/cpu/jitv2/mod.rs +++ b/src/cpu/jitv2/mod.rs @@ -92,6 +92,7 @@ mod zz_corpus { // (baked immediate vs load) is identical to production. crate::cpu::jitv2::codegen::JitConsts { core: core::num::NonZeroUsize::new(core as *mut MipsCore as usize), + ..Default::default() } }; // Synthetic corpus-measurement geometry: 16 KiB direct-mapped L1-D diff --git a/src/cpu/mips_cache_shadow.rs b/src/cpu/mips_cache_shadow.rs index 7047f78ef..b8eb37522 100644 --- a/src/cpu/mips_cache_shadow.rs +++ b/src/cpu/mips_cache_shadow.rs @@ -348,6 +348,7 @@ impl< }; const CACHE_KIND: &'static str = "shadow"; const R10K_FAMILY: bool = Self::R10K_CACHE_OPS; + const PARTIAL_LWR_KEEPS_UPPER: bool = MODEL == model::R10000; const R10K_CACHE_OPS: bool = MODEL == model::R10000; const POSTS_UNCACHED_STORES: bool = Self::R10K_CACHE_OPS; } diff --git a/src/cpu/mips_cache_v2.rs b/src/cpu/mips_cache_v2.rs index fd541c82b..185343caa 100644 --- a/src/cpu/mips_cache_v2.rs +++ b/src/cpu/mips_cache_v2.rs @@ -571,6 +571,10 @@ pub trait CpuModel: MipsCache { /// list of changes from the R10000 has no TLB or CP0 change. The R14000 and /// R16000 are a model choice that gives them the same TLB. const R10K_FAMILY: bool = false; + /// A partial LWR keeps bits 63:32 of the destination. Observed on the + /// R10000; whether the later parts do the same is not established, so they + /// keep the sign extension for now. + const PARTIAL_LWR_KEEPS_UPPER: bool = false; /// An uncached store the system refuses completes instead of raising a /// Data Bus Error. The R10000 sends it as a write request, which gets no /// response, and Bus Error comes only from an error response to a read or @@ -3610,6 +3614,7 @@ impl MipsExecutor { // set) -- the property a persistent code cache needs. // Several CPUs share compiled code, so no one core's address. core: if self.core.smp || std::env::var_os("IRIS_JIT_PIC").is_some() { None } else { core::num::NonZeroUsize::new(core_addr) }, + lwr_keeps_upper: C::PARTIAL_LWR_KEEPS_UPPER, }; *j.jit_consts.lock() = consts; // The inline-compile path (`jitv2_compile_inline`) takes this @@ -3260,6 +3261,7 @@ va={:#018x} phys={:#010x} (code pfn {:#x}, page {:#010x}, word {}/{})", pub fn update_translate_fn(&mut self) { use crate::cpu::mips_core::PrivilegeMode; let is_64bit = self.core.is_64bit_mode(); + self.core.lwr_upper_below = if C::PARTIAL_LWR_KEEPS_UPPER && is_64bit { 3 } else { 0 }; let privilege = self.core.get_privilege_mode(); self.translate_fn = match (is_64bit, privilege) { (false, PrivilegeMode::Kernel) => translate_32_kernel::, @@ -6431,8 +6433,20 @@ va={:#018x} phys={:#010x} (code pfn {:#x}, page {:#010x}, word {}/{})", let mask = 0xFFFFFFFFu32 >> shift; let result = (mem_word >> shift) | (rt_val & !mask); - // Sign-extend to 64 bits - self.core.write_gpr(rt_reg, result as i32 as i64 as u64); + // The manual leaves the upper word of a partial LWR implementation + // specific (NEC U12754E pp. 125-126). The R10000 keeps + // bits 63:32, as observed reference results show; the later + // parts are not established. A full word + // still sign-extends, and so does any word while 64-bit + // operations are off for the current mode (UX/SX/KX clear): + // there a register must hold a sign-extended 32-bit value. + let value = if C::PARTIAL_LWR_KEEPS_UPPER && byte_offset != 3 && self.core.is_64bit_mode() { + (self.core.read_gpr(rt_reg) & 0xffff_ffff_0000_0000) | result as u64 + } else { + // Sign-extend to 64 bits + result as i32 as i64 as u64 + }; + self.core.write_gpr(rt_reg, value); self.handle_exec_complete() } Err(status) => self.finish_status(status) diff --git a/src/cpu/mips_exec_test.rs b/src/cpu/mips_exec_test.rs index 2b7fa23ee..9cf022b64 100644 --- a/src/cpu/mips_exec_test.rs +++ b/src/cpu/mips_exec_test.rs @@ -6807,4 +6807,100 @@ mod tests { }) .unwrap().join().unwrap(); } + + /// A partial LWR keeps bits 63:32 of the destination on the R10000 while + /// 64-bit operations are enabled for the current mode, and sign-extends on + /// the R4400 and R5000 and on the R10000 when they are off (a 32-bit user + /// process, UX=0); a complete word sign-extends on all. + /// Compiled LWR reads `core.lwr_upper_below` rather than decoding Status, + /// so it must follow `is_64bit_mode()` through a Status write and through + /// ERET, which changes the mode without writing Status, and stay 0 on a + /// model that does not keep the upper word. + #[test] + fn lwr_upper_below_follows_the_mode_through_status_writes_and_eret() { + use crate::cpu::mips_core::{STATUS_EXL, STATUS_UX}; + fn check>>(keeps_upper: bool) { + let cfg = MipsCpuConfig::for_model::(Arc::new(crate::ip22_bus::memory_map(false))); + let mut exec = Box::new(MipsExecutor::::new( + Arc::new(MockMemory::new()), PassthroughTlb::default(), &cfg)); + disable_jitv2(&mut exec); + let expect = |exec: &MipsExecutor| { + if keeps_upper && exec.core.is_64bit_mode() { 3 } else { 0 } + }; + for status in 0..=0xffu32 { + exec.set_cp0_status(status); + assert_eq!(exec.core.lwr_upper_below, expect(&exec), "{} Status {status:#x}", C::NAME); + } + exec.set_cp0_status((2 << 3) | STATUS_UX | STATUS_EXL); + assert_eq!(exec.core.lwr_upper_below, 0, "EXL forces kernel mode, where KX is clear"); + exec.core.cp0_epc = 0x1000; + assert_eq!(exec.exec(0x4200_0018), EXEC_COMPLETE, "ERET"); + assert_eq!(exec.core.lwr_upper_below, if keeps_upper { 3 } else { 0 }, "ERET returns to user mode with UX set"); + } + check::(true); + check::(false); + } + + #[test] + fn partial_lwr_keeps_the_upper_word_only_on_the_r10000() { + use crate::cpu::mips_core::{STATUS_KX, STATUS_SX, STATUS_UX}; + const USER: u32 = 2 << 3; + const SUPER: u32 = 1 << 3; + fn check>>(model_keeps_upper: bool) { + let mem = Arc::new(MockMemory::new()); + mem.set_word(0x2000, 0x89ab_cdef); + let cfg = MipsCpuConfig::for_model::(Arc::new(crate::ip22_bus::memory_map(false))); + let mut exec = Box::new(MipsExecutor::::new(mem, PassthroughTlb::default(), &cfg)); + disable_jitv2(&mut exec); + exec.core.write_gpr(1, 0x2000); + let old = 0x8123_4567_a1b2_c3d4u64; + // (Status, 64-bit operations on): each mode's own bit decides, + // whatever the other two hold. + for (status, ops64, mode) in [ + (STATUS_KX, true, "kernel KX=1"), + (0, false, "kernel KX=0"), + (USER | STATUS_UX, true, "user UX=1"), + (USER | STATUS_KX | STATUS_SX, false, "user UX=0"), + (SUPER | STATUS_SX, true, "supervisor SX=1"), + (SUPER | STATUS_KX | STATUS_UX, false, "supervisor SX=0"), + ] { + exec.core.cp0_status = status; + for (offset, word) in [0xa1b2_c389u32, 0xa1b2_89ab, 0xa189_abcd, 0x89ab_cdef].into_iter().enumerate() { + exec.core.write_gpr(2, old); + assert_eq!(exec.exec(make_i(OP_LWR, 1, 2, offset as u16)), EXEC_COMPLETE); + let expected = if model_keeps_upper && ops64 && offset != 3 { + (old & 0xffff_ffff_0000_0000) | word as u64 + } else { + word as i32 as i64 as u64 + }; + assert_eq!(exec.core.read_gpr(2), expected, "{} LWR offset {offset}, {mode}", C::NAME); + } + } + } + check::(false); + check::(false); + check::(true); + } + + /// FrameMask and the wide EntryLo belong to the whole R10000 family, not to + /// the parts that share the R10000's partial LWR. No R12000 type exists on + /// this branch, so this cannot run a family part whose LWR differs from + /// the R10000's; it checks the executor's masks against the family constant + /// the gate reads, and that the two behaviours are separate constants. + #[test] + fn framemask_and_entrylo_widths_follow_the_family_constant() { + fn check>>() { + let cfg = MipsCpuConfig::for_model::(Arc::new(crate::ip22_bus::memory_map(false))); + let exec = Box::new(MipsExecutor::::new(Arc::new(MockMemory::new()), PassthroughTlb::default(), &cfg)); + let (framemask, entrylo) = if C::R10K_FAMILY { (0xFFFF, 0xC000_0003_FFFF_FFFF) } else { (0, 0x3FFF_FFFF) }; + assert_eq!(exec.core.framemask_mask, framemask, "{} FrameMask", C::NAME); + assert_eq!(exec.core.entrylo_mask, entrylo, "{} EntryLo", C::NAME); + } + use crate::cpu::mips_cache_shadow::{R10000ShadowCache, R4400ShadowCache, R5000ShadowCache}; + assert!(R10000ShadowCache::R10K_FAMILY); + assert!(!R4400ShadowCache::R10K_FAMILY && !R5000ShadowCache::R10K_FAMILY); + check::(); + check::(); + check::(); + } } From 54f72ec46cdc179e1d32387f0b04d86bf157401a Mon Sep 17 00:00:00 2001 From: James Braid Date: Fri, 30 Oct 2026 08:03:24 +0000 Subject: [PATCH 09/31] cpu: keep primary-cache diagnostic data on the R10000 A PROM's cache walk reads and writes the primary caches with Index Load Data and Index Store Data. Only the secondary cache had a data array, so an Index Load Data on a primary cache always returned zero and an Index Store Data was dropped. Give the primary caches data arrays and mask each store to its diagnostic field (R10000 UM pp. 262, 271-272): 36 data bits and 1 parity bit for the instruction cache, 32 and 4 for the data cache, 64 and 10 for the secondary. An Index Store Data to the secondary cache writes a quadword, so the other doubleword becomes zero and takes the same check bits (p. 91). A mapped primary Index Load Data or Index Store Data still takes TLB Refill and Invalid, so CACHE now translates the operand for those two; the other index operations still do not. The secondary data slot is now the address >> 3 where it was >> 4, so the walk gets one slot per doubleword instead of two doublewords sharing one. This affects the Octane, the IP28 and an O2 with an R10000. Tests cover a primary round trip and the quadword store. Index Store Data is a diagnostic walk's operation, so it runs out of line in a cold function, and CACHE tests the primary-cache pair against a bit mask. Written inline they grew cache_op and exec_cache in every CPU model and moved the interpreter's hot loops, which cost sys/llsc about 2% on a ship build. --- CHANGELOG.md | 6 +++ src/cpu/mips_cache_shadow.rs | 95 +++++++++++++++++++++++++++--------- src/cpu/mips_exec.rs | 16 ++++-- 3 files changed, 91 insertions(+), 26 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 21db22d3a..4a89bc1d4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -155,6 +155,12 @@ hashes are given where a change is easiest to understand by reading the commit. The JIT matches the interpreter, and the code-cache key changes only for the machines that have the behaviour. +### R10000 cache operations + +- **Index Load Data and Index Store Data** on the R10000's primary caches keep their + data (they read back zero before). The secondary cache's data slot is the address + shifted right by 3 instead of 4: one slot per doubleword. + ### `IRIS_GUI_GL` works in iris-gui - The GUI's GL capture path (`IRIS_GUI_GL=1`, or `[debug] gui_gl_capture`) never diff --git a/src/cpu/mips_cache_shadow.rs b/src/cpu/mips_cache_shadow.rs index b8eb37522..b30f8cf29 100644 --- a/src/cpu/mips_cache_shadow.rs +++ b/src/cpu/mips_cache_shadow.rs @@ -204,11 +204,11 @@ impl< lladdr: UnsafeCell::new(0), ecc_in: UnsafeCell::new(0), ecc_out: UnsafeCell::new(0), - ic: UnsafeCell::new(Shadow::new(Self::IC_LINES, 0)), - dc: UnsafeCell::new(Shadow::new(Self::DC_LINES, 0)), - // Only the R10000 secondary keeps a data shadow: it is the one - // the IP28 PROM walks with Index_Store_Data, and a 1 MB array is - // cheap once. The R4000 family has no index data ops. + // The R10000 keeps data arrays for all three caches, for the PROM's + // Index_Load_Data / Index_Store_Data walks. The R4000 family has no + // index data ops. + ic: UnsafeCell::new(Shadow::new(Self::IC_LINES, if Self::R10K { IC_SIZE / 4 } else { 0 })), + dc: UnsafeCell::new(Shadow::new(Self::DC_LINES, if Self::R10K { DC_SIZE / 4 } else { 0 })), l2: UnsafeCell::new(Shadow::new(Self::L2_LINES, if Self::R10K { L2_SIZE / 8 } else { 0 })), tc_base: UnsafeCell::new(std::ptr::null_mut()), tc_bitmap: UnsafeCell::new(0), @@ -296,17 +296,54 @@ impl< /// Data slot for an R10000 `Index_Load_Data` / `Index_Store_Data`. /// - /// Same shape: way in bit 0, the rest addressing the array. The PROM walks - /// it at `…00`, `…01`, `…10`, `…11`, `…20` — one doubleword per operation - /// with the way bit shifted in beneath it. - fn data_slot(&self, len: usize, virt_addr: u64) -> usize { + /// Same shape: way in bit 0, the rest addressing the array, with four-byte + /// primary and eight-byte secondary slots. This is a model choice, enough + /// for a data walk to read back what it wrote. + fn data_slot(&self, sel: u32, len: usize, virt_addr: u64) -> usize { if len == 0 { return 0; } let way = (virt_addr as usize) & (WAYS - 1); - let word = (virt_addr as usize) >> 4; + let shift = if matches!(sel, CACH_PI | CACH_PD) { 2 } else { 3 }; + let word = (virt_addr as usize) >> shift; (word * WAYS + way) % len } + + /// R10000 `Index_Store_Data`, kept out of line: it runs only in a PROM's + /// diagnostic walk, and inlining it grew `cache_op` and moved the code + /// laid out after it. + #[cold] + #[inline(never)] + fn index_store_data(&self, sel: u32, virt_addr: u64, phys_addr: u64) { + let s = self.shadow(sel); + if s.data.is_empty() { + return; + } + let slot = self.data_slot(sel, s.data.len(), virt_addr); + // Diagnostic field widths: R10000 UM pp. 262, 271-272. + let (data_mask, ecc_mask) = if sel == CACH_PI { + (0xf_ffff_ffff, 1) + } else if sel == CACH_PD { + (0xffff_ffff, 0xf) + } else { + (u64::MAX, 0x3ff) + }; + let ecc = unsafe { *self.ecc_in.get() } & ecc_mask; + s.data[slot] = phys_addr & data_mask; + s.ecc[slot] = ecc; + if matches!(sel, CACH_SI | CACH_SD) { + // Index Store Data (S) writes a quadword: the selected + // doubleword plus a zero partner, sharing the check bits + // (R10000 UM p. 91, figure 5-7). + let partner = slot ^ WAYS; + s.data[partner] = 0; + s.ecc[partner] = ecc; + } + if l2_op_log(CACHE_LOG_HIT) { + crate::dlog!(LogModule::L2c, "shadow: -> ISD slot={slot} data={phys_addr:#018x} ecc={:#x}", + s.ecc[slot]); + } + } } impl< @@ -610,7 +647,7 @@ impl< C_R10K_CBARRIER if Self::R10K && sel == CACH_PI => 0, C_R10K_ILD if Self::R10K && matches!(sel, CACH_PI | CACH_PD | CACH_SD) => { let s = self.shadow(sel); - let slot = self.data_slot(s.data.len(), virt_addr); + let slot = self.data_slot(sel, s.data.len(), virt_addr); if s.data.is_empty() { 0 } else { @@ -622,17 +659,8 @@ impl< s.data[slot] } } - C_R10K_ISD if Self::R10K && matches!(sel, CACH_SI | CACH_SD) => { - let s = self.shadow(sel); - let slot = self.data_slot(s.data.len(), virt_addr); - if !s.data.is_empty() { - s.data[slot] = phys_addr; - s.ecc[slot] = unsafe { *self.ecc_in.get() }; - if l2_op_log(CACHE_LOG_HIT) { - crate::dlog!(LogModule::L2c, "shadow: -> ISD slot={slot} data={phys_addr:#018x} ecc={:#x}", - s.ecc[slot]); - } - } + C_R10K_ISD if Self::R10K => { + self.index_store_data(sel, virt_addr, phys_addr); 0 } @@ -791,12 +819,18 @@ mod tests { } } - /// Index_Store_Data / Index_Load_Data likewise. + /// Index_Store_Data / Index_Load_Data likewise, in the secondary and the + /// primary caches: a PROM's data walk round-trips. #[test] fn secondary_data_round_trips() { let c = cache(); c.cache_op(C_R10K_ISD | CACH_SD, 0x40, 0xdeadbeef); assert_eq!(c.cache_op(C_R10K_ILD | CACH_SD, 0x40, 0), 0xdeadbeef); + + c.cache_op(C_R10K_ISD | CACH_PI, 4, 0x1234); + assert_eq!(c.cache_op(C_R10K_ILD | CACH_PI, 4, 0), 0x1234); + c.cache_op(C_R10K_ISD | CACH_PD, 8, 0x5678); + assert_eq!(c.cache_op(C_R10K_ILD | CACH_PD, 8, 0), 0x5678); } /// The MRU bit is **shared between the ways of a set**: written through @@ -864,6 +898,21 @@ mod tests { assert_eq!(c.cache_op_ecc(), 0x3fe, "way 1's check bits"); } + /// The NEC layout: ten check bits, and a secondary store writes a + /// quadword whose other doubleword is zero and shares the check bits. + #[test] + fn nec_layout_stores_a_quadword_with_ten_check_bits() { + let c = cache(); + c.set_cache_ecc(0); + c.cache_op(C_R10K_ISD | CACH_SD, 8, 0xdead_beef); + c.set_cache_ecc(0xffff_fc01); + c.cache_op(C_R10K_ISD | CACH_SD, 0, 0x1111_2222_3333_4444); + assert_eq!(c.cache_op(C_R10K_ILD | CACH_SD, 0, 0), 0x1111_2222_3333_4444); + assert_eq!(c.cache_op_ecc(), 0x001, "ten check bits"); + assert_eq!(c.cache_op(C_R10K_ILD | CACH_SD, 8, 0), 0, "zero partner"); + assert_eq!(c.cache_op_ecc(), 0x001, "shared check bits"); + } + /// An untouched set reports no MRU, so an ordinary tag read is not /// contaminated by it. #[test] diff --git a/src/cpu/mips_exec.rs b/src/cpu/mips_exec.rs index 19913ddd5..ecfe79c38 100644 --- a/src/cpu/mips_exec.rs +++ b/src/cpu/mips_exec.rs @@ -6632,6 +6632,12 @@ va={:#018x} phys={:#010x} (code pfn {:#x}, page {:#010x}, word {}/{})", // R4000 hit operations — translating them would fault on an index that // is not a valid virtual address. See C_R10K_ISD in mips_cache_v2. let sel = cache_op & 3; + // Bit `cache_op` of this mask: the primary-cache index data operations. + const R10K_PRIMARY_DATA_OPS: u32 = (1 << (C_R10K_ILD | CACH_PI)) + | (1 << (C_R10K_ILD | CACH_PD)) + | (1 << (C_R10K_ISD | CACH_PI)) + | (1 << (C_R10K_ISD | CACH_PD)); + let primary_data_op = C::R10K_CACHE_OPS && (1u32 << cache_op) & R10K_PRIMARY_DATA_OPS != 0; let r10k_index_op = C::R10K_CACHE_OPS && match op { C_R10K_CBARRIER => sel == CACH_PI, @@ -6639,8 +6645,11 @@ va={:#018x} phys={:#010x} (code pfn {:#x}, page {:#010x}, word {}/{})", C_R10K_ISD => matches!(sel, CACH_SI | CACH_SD), _ => false, }; - let needs_translation = - !r10k_index_op && matches!(op, C_CDX | C_HINV | C_HWBINV | C_HWB | C_HSV); + // A primary Index Load/Store Data selects by VA, but a mapped one still + // takes TLB Refill and Invalid (VR10000 Instruction Set UM p. 15). The other index + // operations keep not translating their operand. + let needs_translation = primary_data_op + || (!r10k_index_op && matches!(op, C_CDX | C_HINV | C_HWBINV | C_HWB | C_HSV)); let phys_addr = if needs_translation { // Hit operations need address translation @@ -6660,7 +6669,7 @@ va={:#018x} phys={:#010x} (code pfn {:#x}, page {:#010x}, word {}/{})", // that reads only TagLo sees a different tag from the one written. // Harmless for R4400/R5000, whose tags fit in 32 bits and who leave // TagHi zero: the shift-in contributes nothing there. - let stores_taglo = op == C_IST || (r10k_index_op && op == C_R10K_ISD); + let stores_taglo = op == C_IST || ((r10k_index_op || primary_data_op) && op == C_R10K_ISD); let phys_addr_or_taglo = if stores_taglo { ((self.core.cp0_taghi as u64) << 32) | (self.core.cp0_taglo as u32 as u64) } else { @@ -6733,6 +6742,7 @@ va={:#018x} phys={:#010x} (code pfn {:#x}, page {:#010x}, word {}/{})", // Split back the way it arrived. Zeroing TagHi unconditionally // threw away the top of every tag wider than 32 bits. self.core.cp0_taglo = result & 0xFFFF_FFFF; + // A primary data cache access leaves TagHi zero (a model choice). self.core.cp0_taghi = (result >> 32) as u32; } From 8c71dbfccb35f6fc5f37fe93b3bc855f38f161b0 Mon Sep 17 00:00:00 2001 From: James Braid Date: Fri, 30 Oct 2026 08:45:00 +0000 Subject: [PATCH 10/31] cpu: mask the R10000's secondary-cache tags Index Store Tag stored all 64 bits of a secondary-cache tag, so a walking-1s test over the tag read back bits the R10000 defines as reserved. The manual says they return zero (R10000 UM p. 270, figure 11-31). Keep MRU, the physical tag, state, virtual index and ECC, which assembled is 0x8000_000f_ffff_cdff, and drop the rest. The mask is CpuModel::SECONDARY_TAG_MASK, all ones except on the R10000 family. MRU stays because the PROM writes it as TagHi bit 31 and reads it back. The later parts share the tag layout (the R12000 manual agrees; the R14000 and R16000 are taken to match), and a PROM's tag test on the R14000 expects the reserved bits to read as zero. Until the R12000 and later models exist here, the family is just the R10000. Tests: an all-ones store on the R10000 and R10000 O2 reads back the mask, an R4400 control keeps TagLo bits 12 and 13, and the walking-1s round trip expects the reserved bits as zero. --- CHANGELOG.md | 2 ++ src/cpu/mips_cache_shadow.rs | 70 +++++++++++++++++++++++------------- src/cpu/mips_cache_v2.rs | 3 ++ 3 files changed, 50 insertions(+), 25 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4a89bc1d4..32f8de788 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -160,6 +160,8 @@ hashes are given where a change is easiest to understand by reading the commit. - **Index Load Data and Index Store Data** on the R10000's primary caches keep their data (they read back zero before). The secondary cache's data slot is the address shifted right by 3 instead of 4: one slot per doubleword. +- **Secondary-cache tags** on the R10000 family read the reserved bits (TagLo 9, 12, 13 and + TagHi 30:4) as zero; the MRU bit and the other tag fields are kept. ### `IRIS_GUI_GL` works in iris-gui diff --git a/src/cpu/mips_cache_shadow.rs b/src/cpu/mips_cache_shadow.rs index b30f8cf29..e58d37a29 100644 --- a/src/cpu/mips_cache_shadow.rs +++ b/src/cpu/mips_cache_shadow.rs @@ -23,7 +23,9 @@ //! to it and returns them unchanged, so a walking-1s test over tag or data //! SRAM passes without anyone having to know the hardware's field layout. //! Where the emulator *must* know the layout — to decide whether a line is -//! valid, say — that is a separate question from storing the bits. +//! valid, say — that is a separate question from storing the bits. The one +//! exception is the R10000's secondary tag, whose reserved bits read as zero +//! (`R10K_L2_TAG_MASK`). //! //! The shadow is deliberately not consulted by `read`/`write`/`fetch`. If it //! ever needs to be, this type is the wrong shape and should say so loudly @@ -90,15 +92,11 @@ const MRU_BIT: u64 = 1 << 63; /// R4400 and R5000 index differently — see `tag_slot`. const WAYS: usize = 2; -/// Bits a secondary-cache tag retains. -/// -/// All of them, now. A 36-bit mask was inferred here from the PROM's -/// walking-0s phase, which writes an all-ones TagLo and expects back -/// 0x0000000f_ffffcdfe — but that truncation comes from the tag being -/// assembled as `(TagHi << 32) | TagLo[31:0]`, not from the array dropping -/// bits. Once the executor carried TagHi properly the mask did nothing except -/// discard the MRU bit, which the PROM writes as TagHi[31] and reads back. -const L2_TAG_MASK: u64 = u64::MAX; +/// Bits an R10000 secondary-cache tag retains: MRU (bit 63), the physical tag +/// (TagHi 3:0 and TagLo 31:14), state, virtual index and ECC. TagLo bits 9, 12 +/// and 13 and TagHi 30:4 are reserved and read as zero (R10000 UM p. 270, +/// figure 11-31). Other parts keep every bit; see `CpuModel::SECONDARY_TAG_MASK`. +const R10K_L2_TAG_MASK: u64 = 0x8000_000f_ffff_cdff; /// Is the shadow cache's operation trace on, at this level of detail? /// @@ -388,6 +386,7 @@ impl< const PARTIAL_LWR_KEEPS_UPPER: bool = MODEL == model::R10000; const R10K_CACHE_OPS: bool = MODEL == model::R10000; const POSTS_UNCACHED_STORES: bool = Self::R10K_CACHE_OPS; + const SECONDARY_TAG_MASK: u64 = if Self::R10K_CACHE_OPS { R10K_L2_TAG_MASK } else { u64::MAX }; } impl< @@ -602,12 +601,16 @@ impl< } match op { - // Index_Store_Tag / Index_Load_Tag. Stored and returned verbatim: - // a tag test is a round trip, and round trips do not require - // knowing what the bits mean. + // Index_Store_Tag / Index_Load_Tag. Stored and returned verbatim + // unless the part sets a secondary mask: a tag test is a round trip, + // and round trips do not require knowing what the bits mean. C_IST => { let idx = self.tag_slot(sel, virt_addr); - let mask = if matches!(sel, CACH_SI | CACH_SD) { L2_TAG_MASK } else { u64::MAX }; + let mask = if matches!(sel, CACH_SI | CACH_SD) { + ::SECONDARY_TAG_MASK + } else { + u64::MAX + }; if l2_op_log(CACHE_LOG_OP) { crate::dlog!(LogModule::L2c, "shadow: -> IST va={virt_addr:#012x} idx={idx} stores {:#018x}", phys_addr & mask & !MRU_BIT); @@ -798,22 +801,23 @@ mod tests { } /// A tag test is a round trip, so the shadow must return exactly the bits - /// it was given — including any the real layout would not use. The cache + /// it was given, except the reserved ones the manual reads as zero. The cache /// model that preceded this one decoded TagLo into ptag/state/pidx fields /// and re-encoded on the way out, which silently dropped the low seven /// bits and every state code it did not recognise. A walking-1s test then /// failed on its very first bit. #[test] - fn tags_round_trip_every_bit() { + fn tags_round_trip_every_defined_bit() { let c = cache(); for bit in 0..64 { // 32 and 63 are MRU control/state, not tag storage. if bit == 32 || bit == 63 { continue; } let v = 1u64 << bit; c.cache_op(C_IST | CACH_SD, 0, v); + let kept = ::SECONDARY_TAG_MASK & v; assert_eq!( c.cache_op(C_ILT | CACH_SD, 0, 0), - v, + kept, "bit {bit} did not survive a store/load tag round trip" ); } @@ -918,18 +922,19 @@ mod tests { #[test] fn an_untouched_set_has_no_mru_bit() { let c = cache(); - c.cache_op(C_IST | CACH_SD, 0, 0xdead_beef); - assert_eq!(c.cache_op(C_ILT | CACH_SD, 0, 0), 0xdead_beef); + // Only defined tag bits (TagLo bits 9, 12 and 13 read as zero). + c.cache_op(C_IST | CACH_SD, 0, 0xdead_8cef); + assert_eq!(c.cache_op(C_ILT | CACH_SD, 0, 0), 0xdead_8cef); } /// Separate lines must not alias onto one another. #[test] fn distinct_lines_hold_distinct_tags() { let c = cache(); - c.cache_op(C_IST | CACH_SD, 0, 0x1111_1111); - c.cache_op(C_IST | CACH_SD, 128, 0x2222_2222); - assert_eq!(c.cache_op(C_ILT | CACH_SD, 0, 0), 0x1111_1111); - assert_eq!(c.cache_op(C_ILT | CACH_SD, 128, 0), 0x2222_2222); + c.cache_op(C_IST | CACH_SD, 0, 0x1111_0111); + c.cache_op(C_IST | CACH_SD, 128, 0x2222_0022); + assert_eq!(c.cache_op(C_ILT | CACH_SD, 0, 0), 0x1111_0111); + assert_eq!(c.cache_op(C_ILT | CACH_SD, 128, 0), 0x2222_0022); } /// Bit 0 of a CACHE index selects the **way**, and the two ways of a set @@ -957,14 +962,14 @@ mod tests { for set in 0..4u64 { for way in 0..2u64 { let va = set * 128 + way; - c.cache_op(C_IST | CACH_SD, va, 0x1000 + set * 16 + way); + c.cache_op(C_IST | CACH_SD, va, 0x4000 + set * 16 + way); } } for set in 0..4u64 { for way in 0..2u64 { let va = set * 128 + way; assert_eq!(c.cache_op(C_ILT | CACH_SD, va, 0), - 0x1000 + set * 16 + way, + 0x4000 + set * 16 + way, "set {set} way {way} aliased"); } } @@ -1012,6 +1017,21 @@ mod tests { assert_eq!(::NAME, "shadow", "existing IP28 snapshots"); } + /// The R10000 returns zero for the reserved bits of a secondary tag and + /// keeps MRU, tag, state, index and ECC (R10000 UM p. 270); the R4400 + /// stores the tag as written. + #[test] + fn the_r10000_masks_secondary_tags() { + fn stored>>(tag: u64) -> u64 { + let c = C::from(Arc::new(Memory::new(1)) as Arc); + c.cache_op(C_IST | CACH_SD, 0, tag); + c.cache_op(C_ILT | CACH_SD, 0, 0) + } + assert_eq!(stored::(u64::MAX), 0x8000_000f_ffff_cdff); + assert_eq!(stored::(u64::MAX), 0x8000_000f_ffff_cdff); + assert_eq!(stored::(0x3000) & 0x3000, 0x3000, "R4400 keeps TagLo bits 12 and 13"); + } + /// The O2 R10000 differs from the IP28's only in its secondary line size. #[test] fn the_o2_r10000_is_the_ip28_part_with_shorter_secondary_lines() { diff --git a/src/cpu/mips_cache_v2.rs b/src/cpu/mips_cache_v2.rs index 185343caa..c1f7db120 100644 --- a/src/cpu/mips_cache_v2.rs +++ b/src/cpu/mips_cache_v2.rs @@ -583,6 +583,9 @@ pub trait CpuModel: MipsCache { /// to match. An uncached SC succeeding is a model choice. Loads and fetches /// still fault. const POSTS_UNCACHED_STORES: bool = false; + /// Bits `Index_Store_Tag` keeps for the secondary cache: all of them, except + /// on the R10000 family, whose reserved tag bits read as zero (R10000 UM p. 270). + const SECONDARY_TAG_MASK: u64 = u64::MAX; } pub trait MipsCache: Send + Sync { From eb69808ebf76f01a303af8c7ec697079b2619120 Mon Sep 17 00:00:00 2001 From: James Braid Date: Fri, 30 Oct 2026 08:30:00 +0000 Subject: [PATCH 11/31] cpu: add the R12000, R14000 and R16000 as selectable models The Octane shipped with an R12000 or R14000 and the O2 with an R12000, but `cpu` took only r4400, r5000 and r10000. An Octane hinv could not show the later parts, and CP0 Config reported 1 MB of secondary cache for every R10000-family part, so a 2 MB model printed "1 Mbyte". Add r12000, r14000 and r16000. They share the R10000's programming model (44-bit virtual, 40-bit physical, the R10000 cache operations) through model::r10k_family. Config SS is now log2(secondary size / 512 KB). The Octane's R14000 is the R12000's 2 MB type with its PRId set at run time, as the Octane R10000 already does, so only three new executors are compiled, not four. Octane r10000, r12000, r14000 (195, 360, 600 MHz by default) O2 r5000, r10000, r12000 (400 MHz; 64-byte secondary lines, because with 128 the O2 PROM panics before its banner) IP28 r10000 only There are two R16000 parts, r16000 (revision 2.2, 800 MHz, PRId 0x0f42) and r16000_700 (revision 2.1, 700 MHz, PRId 0x0f41). Both have a 4 MB secondary cache and present their own Config reset word; the 700 MHz part is the 800 MHz type with PRId and that word set at run time, as the Octane R14000 sets its PRId, so it adds no executor. The 800 MHz, 8 MB part would need a third executor, as the cache size is a type parameter, and is left out. The R16000 has no machine yet. The Indy and Indigo2 profiles build any part, and iris-bench uses that for r16000 and r16000-700 cells. The bench harness names an implementation 0x0f CPU an R16000 above major revision 2, and the prebuilt guest image is refreshed for it. On the R12000 and later, Index Writeback Invalidate (D) also clears the line's primary data tag, as Index Load Tag then shows; the R10000 keeps it. The new parts also keep the upper word on a partial LWR, as the R10000 does: it is implementation-defined, and keeping it matches what the IRIX CPU instruction test expects and the qemu-sgi-origami model, so for the R12000 and later it is a model choice. R10000 hot path: +0.09% host instructions per guest instruction on iris-bench, inside the ~1% noise (an R4400 control moved +0.86%). --- CHANGELOG.md | 10 +++ HACKING.md | 5 +- HELP.md | 4 +- bench/README.md | 7 +- bench/harness/main.c | 51 ++++++++----- bench/prebuilt/PROVENANCE | 4 +- bench/prebuilt/irisbench.elf | Bin 308876 -> 300876 bytes cpu-tests/harness/iris.h | 7 +- docs/o2-ip32.md | 14 ++-- docs/octane-ip30.md | 8 ++- src/bin/iris_bench.rs | 19 ++++- src/config.rs | 123 ++++++++++++++++++++++++++----- src/cpu/jitv2/equiv_test.rs | 22 +++--- src/cpu/mips_cache_shadow.rs | 135 ++++++++++++++++++++++++++++++++--- src/cpu/mips_cache_v2.rs | 11 +++ src/cpu/mips_exec.rs | 17 +++-- src/cpu/mips_exec_test.rs | 91 ++++++++++++++++++++--- src/ip30_machine.rs | 90 ++++++++++++++++++++--- src/ip32_machine.rs | 38 +++++++++- src/machine.rs | 28 +++++++- 20 files changed, 579 insertions(+), 105 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 32f8de788..8eb32cf1f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -163,6 +163,16 @@ hashes are given where a change is easiest to understand by reading the commit. - **Secondary-cache tags** on the R10000 family read the reserved bits (TagLo 9, 12, 13 and TagHi 30:4) as zero; the MRU bit and the other tag fields are kept. +### R12000, R14000 and R16000 + +- `[machine] cpu` and `--cpu` take `r12000`, `r14000`, `r16000` (revision 2.2, + the 800 MHz part) and `r16000_700` (revision 2.1, the 700 MHz part). The Octane + takes the R10000, R12000 and R14000 (defaulting to 195, 360 and 600 MHz), the + O2 the R5000, R10000 and R12000 (400 MHz for the R12000), the IP28 the R10000. + The R16000 has no machine yet; `iris-bench` is its only user. CP0 Config now + reports the part's own secondary-cache size, where every part read 1 MB, and + each R16000 presents its own reset word. + ### `IRIS_GUI_GL` works in iris-gui - The GUI's GL capture path (`IRIS_GUI_GL=1`, or `[debug] gui_gl_capture`) never diff --git a/HACKING.md b/HACKING.md index e62f2ecae..54373e395 100644 --- a/HACKING.md +++ b/HACKING.md @@ -18,7 +18,8 @@ Other documents: IRIS emulates five SGI workstations: the Indy (IP24), the Indigo2 (IP22), the Indigo2 IMPACT (IP28), the O2 (IP32) and the Octane (IP30). The CPU is an -R4400, R5000 or R10000, chosen at runtime (the Octane takes the R10000 only). +R4400, R5000, R10000, R12000, R14000 or R16000 (two parts), chosen at runtime (the IP28 takes the R10000; +the Octane the R10000, R12000 or R14000; the O2 the R5000, R10000 or R12000). Graphics is Newport (REX3), GR2 (XZ/Extreme) or IMPACT on the Indy and Indigo2, GBE on the O2 and Solid IMPACT on the Octane. It boots the real PROMs and IRIX 5.3 and 6.5 to a desktop with networking. @@ -371,7 +372,7 @@ consumer thread never takes `rec`. handler sets the PC itself, including delay slots. - The CPU model is the `CpuModel` trait (`MIPS4`, `PRID`, `FIR`, `TLB_ENTRIES`, ...), implemented by each cache model. The executor is - monomorphised over it, so all three CPUs are in every binary with no + monomorphised over it, so every CPU is in every binary with no per-model branch on the hot path. `Machine::new` picks one from `[machine] cpu`. - With `opcodefusion` (implied by `lightning`), the interpreter executes diff --git a/HELP.md b/HELP.md index 8b2494b9c..56c599dd0 100644 --- a/HELP.md +++ b/HELP.md @@ -620,7 +620,7 @@ scsi_deferred_int = true # needed by OpenBSD/NetBSD; see --no-scsi-deferr [machine] profile = "indy_ip24" # "indigo2_ip22" (--ip22), "indigo2_ip28", or "o2_ip32" (an [o2] section: docs/o2-ip32.md); --machine NAME picks any -cpu = "r4400" # "r5000" or "r10000" (--cpu); use R10000 on IP28 +cpu = "r4400" # "r5000", "r10000", "r12000", "r14000", "r16000" or "r16000_700" (--cpu); IP28: r10000, Octane: r10000/r12000/r14000, O2: r5000/r10000/r12000 # cpus = 2 # Octane only: a second processor (docs/Octane/multiprocessor.md) [graphics] @@ -787,7 +787,7 @@ Configuration --machine Machine to emulate (a [machine] profile name); `help` lists them --ip22 Same as --machine indigo2_ip22 --graphics newport, xz, extreme, solidimpact, highimpact, maximpact, v6, v8, v10, v12 - --cpu r4400 (default), r5000, or r10000 + --cpu r4400 (default), r5000, r10000, r12000, r14000, r16000 or r16000_700 --bank0..--bank3 RAM bank sizes (0/8/16/32/64/128; IP28: also 256/512) --clock-fixed-mhz CP0 Count frequency (IP22/IP24: 33; IP28: 97.5) diff --git a/bench/README.md b/bench/README.md index ab7380fb0..f347de119 100644 --- a/bench/README.md +++ b/bench/README.md @@ -328,6 +328,11 @@ for each cell, because a cell's features live in its binary. | `r5000-jitv2` | `jitv2` | `r5000` | | `r4400-lightning` | `lightning` | `r4400` | | `r4400-jitv2-lightning` | `jitv2,lightning` | `r4400` | +| `r10000-interp`, `r12000-interp`, `r14000-interp`, `r16000-interp`, `r16000-700-interp` | (default) | `r10000`, `r12000`, `r14000`, `r16000`, `r16000_700` | + +The R10000 family always runs the shadow caches, so these share the default +build. No shipped machine takes the R16000 yet; the Indy and Indigo2 +(non-IP28) profiles build any part, and the cell relies on that. Each build is copied to `bench/build/iris-` before the next one starts — the next `cargo build` overwrites `target/release/iris`, and a matrix that races @@ -397,7 +402,7 @@ device on real hardware, so results come back over serial with CP0 Count as the time base; the header says so. **On which SGI machines?** The suite identifies any MIPS CPU by name from PRId -— R4000, R4400, R4600, R4700, R5000, R8000, R10000, R12000, R14000, RM5200, +— R4000, R4400, R4600, R4700, R5000, R8000, R10000, R12000, R14000, R16000, RM5200, RM7000 — and an implementation it has no name for prints as `MIPS-imp-0xNN` rather than refusing. That is safe to do because **nothing here is CPU-specific**: the kernels are ordinary compiled MIPS III, and `golden.h` is diff --git a/bench/harness/main.c b/bench/harness/main.c index 75adb4bf3..5548b10ca 100644 --- a/bench/harness/main.c +++ b/bench/harness/main.c @@ -220,34 +220,50 @@ static void run_one(const struct bench *b) /* * The CPU by name, from PRId alone — so this is right on any MIPS machine, - * not only on the two the emulator models. The revision split on imp 0x04 is - * the standard one: an R4000 and an R4400 report the same implementation and - * are distinguished by major revision. + * not only on the two the emulator models. Three implementation numbers are + * shared by two parts that differ in major revision: an R4000 and an R4400 + * (imp 0x04, the R4400 from major 4), and an R14000 and an R16000 (imp 0x0f, + * the R16000 from major 3, the same split as Linux's cpu-probe.c). The table + * is searched in order, so the more specific row comes first. * * An imp we do not have a name for still runs; it just prints as "MIPS imp * 0xNN", which is more use to whoever is holding that machine than "unknown". */ +static const struct { + u32 imp; + u32 min_major; /* the row applies from this major revision up */ + const char *name; +} cpu_names[] = { + { IMP_R4400, 4, "R4400" }, + { IMP_R4000, 0, "R4000" }, + { IMP_R10000, 0, "R10000" }, + { IMP_R4300, 0, "R4300" }, + { IMP_R12000, 0, "R12000" }, + { IMP_R14000, 3, "R16000" }, + { IMP_R14000, 0, "R14000" }, + { IMP_R8000, 0, "R8000" }, + { IMP_R4600, 0, "R4600" }, + { IMP_R4700, 0, "R4700" }, + { IMP_R4650, 0, "R4650" }, + { IMP_R5000, 0, "R5000" }, + { IMP_RM7000, 0, "RM7000" }, + { IMP_RM5200, 0, "RM5200" }, +}; + static const char *cpu_name(void) { #if defined(BENCH_HOST) return "host"; #else static char other[20]; + unsigned row; - switch (PRID_IMP(cpu_prid)) { - case IMP_R4000: return PRID_REV_MAJOR(cpu_prid) >= 4 ? "R4400" : "R4000"; - case IMP_R10000: return "R10000"; - case IMP_R4300: return "R4300"; - case IMP_R12000: return "R12000"; - case IMP_R14000: return "R14000"; - case IMP_R8000: return "R8000"; - case IMP_R4600: return "R4600"; - case IMP_R4700: return "R4700"; - case IMP_R4650: return "R4650"; - case IMP_R5000: return "R5000"; - case IMP_RM7000: return "RM7000"; - case IMP_RM5200: return "RM5200"; - default: { + for (row = 0; row < sizeof cpu_names / sizeof cpu_names[0]; row++) { + if (cpu_names[row].imp == PRID_IMP(cpu_prid) && + PRID_REV_MAJOR(cpu_prid) >= cpu_names[row].min_major) + return cpu_names[row].name; + } + { /* No snprintf here — freestanding. Build "MIPS-imp-0xNN" by hand. * * No spaces, deliberately: this string is emitted as the value of @@ -265,7 +281,6 @@ static const char *cpu_name(void) other[i] = 0; return other; } - } #endif } diff --git a/bench/prebuilt/PROVENANCE b/bench/prebuilt/PROVENANCE index 235efddb8..d8942d3db 100644 --- a/bench/prebuilt/PROVENANCE +++ b/bench/prebuilt/PROVENANCE @@ -4,6 +4,6 @@ # CI compares it and fails if the image was not refreshed alongside a # source change. The ELF's own hash is informational only — two correct # builds of identical source differ, because -g records the build path. -sources sha256 b2aa6db09ad6f42b68754e4199c66e0802e8cbc9f3c886e6a2c92b1f3502bf67 -image sha256 21dc4eade5b3326afcf77199b405767a56775168c0682e65c9ba5134253b857a 308876 bytes +sources sha256 0e8ab96acd3b21308342b5b6dce317c0dc70f57fcaf7f9c5333b18a98cbd0ecc +image sha256 14e47c0c85aa4e153eb7352f6d0017d12c96e6b80df26edeb8663e9db7b541cb 300876 bytes inputs 32 files diff --git a/bench/prebuilt/irisbench.elf b/bench/prebuilt/irisbench.elf index 12916e07b670125d400fc1ac65a31849c437f316..377116345c72efb5c4664f1fcbe7691e5a81b792 100755 GIT binary patch literal 300876 zcmeFadwi7Tng4%1&rFgD1ek;XgHoMgX2KvKLk$}3eBH++3<0$UMVB6SHzh%1(QUfz z?xx@E>*th%rZq}z?JoP>?!KlRG}c70-P*L@Z94^}*j7usK?#c`^@J^Uoo;odn)3YK z*F7^KpjO@8{(k>Fqv1K+=g)ne?(050w=BCNZP}KL|7;0xZ}o)Dn|krz{b_C^QZHe# zB_TCZt>OG98mAENNUQi}xAMM1-*kzxfWKNSt$}aUiKi-i!twbMuqs=l)sD>4_o>Oe zThFzsCmQda-D-WbGjhiRB7=V}5+4*lVg4C6xvx)*`|ZCJ`F0oK>m^9o*o3fdt4#vt zIZ!W412XFLwCSVyIhoGlwNISPR*3~%*XdcqeT#XYdQVSu-nVDB&il4I>dWTisX0B7 zd0+allM3`i;(;Tr)IS=ZKfATMQv8}=kiVd~WAT9arv}b5ne*(L%!>b{?4C$(<5R() zBbE9pWq;S{ZIWPbqtuEol;ZDFv4bh!DD`d%#?4iN@wZ49BHf(dm(H;ZZPBs{dFh>& zOFfo7HOE}jPle63>8X&EHKe4X;Zi9Nr{uP-NPjBu-K_ZLHj3}!4R=&YD7i^0 zlElnW5=W_wu!&SBmK#pGEI8I{w3?VzTo(i`()aZmEta$FVm7Y zNO)iwe@zk?xI(H1E|=gyT51PYN@!q(%o%8w*g%UctQ{3QRwiNog2@izER#fS=G42C zUu@|J#}?iy7GX*9s3*KkBoP3x_oZe>2d!``*M%Dn&w6i0&PpsghQXbnW{@Ne7 z&sY+Q-%;V7fOeiuKI^(?_M1KrI8P-^Kl}tJA!LP{ zU$|C218-lZ7>du?n6fjpG8_kk%2WmS%R>>_8;bO1l-f4;x8UZdZwmP~ zo_#m@_RHE(bYTa1W@Mq}M?O^oeP~|rT1K93h*E&`rpXWblc(vI&hUbPd&o26zIH~Q z{p9&C6Y`u@%yaUoJS~&w@5?V%X3NxoXB^DhD_mQ@n6Sx>bV(w&Cg$BCmY2@ zruc))q(4?6(cp3;k5o?iQx&xHS>$Q8*xG*icrZ4w@u`sV!oc;&TdoTm;r(?_#q@q^ zz~4Qv_Nm$e9>?#lb>xewU{B-$c`69TYqd;`_us{4YQTE0$;;o9OjTH!*wgM9e5m?1 zlA3g`JJ@k#$I_L^jt${Rs&eN}`CKxVs>;gJ`S~gGqhx&Gvts&O+t|AYJ+}fm5RkGs zIuOC5%3oWx;r`+5J~(PApbgF|Mp5#ph|r zDe|A@7T-6J_KJl~GHG?2Sd=$O`;odWA6R=NrFoXh?%fD>8>E*0@Nd~6{<%rZ1X{eW88s4;T8PnZ6MB5m_5tklZG& z*Z^TW5edu@)nnK#ZZIboSUx9`+$cWU`Cjne8puP+tyqhZzsO=M z`7Cj08=&z|IBBP-=<@q%q;=VqpmH^@4QVuQ=Nhc%Obp?H}`g-@I z-j`$}?fw~U2+HPAo!Um&HGiaSroES7)A=Z?OdQj;^umWFh}xYL>yc{N9**?&kf+)= zZDF;Qtn3S}Tf5hNHTeQImd4rNu%LESX2&*>2l0F1d2MH`gLeIb`yCCDUcEMlBeX4` zGC}Sw##?UURTbi)A8!;20-peNCBEhz9Cd%GX*551xM9KSQK=cnWqUF2Ci*c0KAQ@0 zNb}plO3DEr+7=Mvm9XF~Kj_F{2QL$5@Vh4w>3g=&R&8hPbM9ESgprOY?KE-iE3;Jjh zW0sY*?MwWBLl2YvNOWma-knUE_XDTB%D5^=P2~u8~|%TxD!<53&K- z$Dglu4>kiMmzeE)A()jbWP`j1%;*NS1?A6pMi_-_q%8Qq8Eq^o$$TiFCRO>H3)aoQ z-~y?-jHc!Tv&i~P4vO?aoutg?GE)&;rg%4Ub`T9byi7z}P8^eplAMPek)j6uu zlG$yl!=c}{VBD_L_DbcvZS>J9Xwy&l4*CuHTD=`v!hKyG_=PiZsouK4zJ6;&ezP@t zR-05nk@32#Nyks|Q_N;wa}PR z_0ar^^*p~WNnYX}((+|D_aDD}6WxTgFP2efQh-?YDke`MZAS)?~1aaR~g`hYf>H_oJ6z zkF~olvfrg|{fe^|)7yCbZtAb&5n~_d`vLOheV^e6rDt(p*0}mBr87FWQSGRQ2J71& z8dN!<_O9}^$z$#DyEVS%TP(xYG>n7IxIx48R}9y7;0!VL%c*v3GVLf&+13)3gA?1K z?_!OMplf8RHW-#^KfRYiOgme}ug??Yhv9imqiG9i+?O14mEMpC$r8O88q@NiG3^u5t-NB&l?AW?q`ypAny?nWS=tpYrcS z8k$JU2&+0%e_y!-LFW-yL4aed17q=BMSrt5sth# zRH~yoo)r$Aq);o1Zy-)G! zo7a0L<$YyhdA--1!bkCF&A8i2;S+J<{g?Fp#CNqB_f^G{Jin-6(2b2e?PlEjF|Hk4 z@8bHGTyIO&_C)U7aJ&y*@GA`zc`mN^PzRqvFBTOId0O+T@| zl((vcRS}=7w0xBdv~OZN-h?OI<;a+piTUaCJE8lxsPmi2n-bp{S!3*@>EXrHJF4|o zI{p8E4g-{BD?VDD-XCc$(ENnHQ5|dQbL5&r7`#=c{ibnj#>hcA`JbBhzu)$g&z9;_ zlHG#5RND}Fi?3eo!$Xc<)$Y*ug?Eo1qZS;(@;ku|+@4F=W#*Mav*m1DOF_kPFMDFaanIMFT{ zi{gikI(iK|(Y#(0PPA38NhccCYl9Qb>eWe|9Txe}q1g1cg6wW?I-&Vs3%XnB4OeNfJIz7>=kJ}8d!EmN#1gE|;nOdm@1F#-Go zy^O}nf{Y`KZVsuPAuS@4Y^`6*dXcNnVJSv%cv$YM_0$175a75=Mv$sKG_zzk;r9sCNGBtXM z+!?MD|M}grS+9Tfn%vn{XFaN8wT2q?#|3Yf8B%fHCAFV(e{;q4etbTz&fRHSG@fD`T<6}Oz0Gx3cNY9T+D3y3;iL&LH+4)lCnDRz znxaq+z9Plq2gsJw>dU()wax{)6DQ{P8($1Ipz+NVX^(j$4YQ;`XMOR)v%mPdAtXYRLHR#kYAUcHA+1qTe@u&$uO>@#_2G z`zgh66V|n0RSrKMX~pS`>s8CHy~6z^?eG)-93*i@rgUa}{Z7O;$l40H?2hfUEhPRy z9c40J9~=nCF54X&tO`i{B{TjUj`xz+@B>#vg6+zVi5}(msBAD^4H9s${liXJvzNtBt;;fAm*-8~8x`tP{W=OPBX0Y zZOoj2b?tfXvsz{~K7HC9keiq@+J-N$)A48fsVnh)hUQgV7X1Ro7N~elpA9A-K!zyp z&3VQzfsilEe7^5L(4Y5SEWSD2+Fu>*jn&G_&>8b9yeofiVf_cPv14sC-m%a6n)0UH z7An~9;LkX>POg_Z)E}Qiu%>o1{#(LTX7skx$C@Ac#MW7&=?%6jZ#E#KZ!P%I@Fl*k z@71@X{bBh+3mD7UmQY9LtC&*^!utX3PpP;hrS#Ep%=(&7bpb!^2TdpIk_$|}w-xfO z&AGqTIGVPAkI~r$|1fewu$x?DipnEo%xmbwKn?mv$C@JhE{zbe46XGaBFSx(nNHaj^$_Y-x$x z53GqUYPWQb%!*C6Y+L!IO2Q&N)FY?!3Vx%Lei&Lnno3`JXn{N$idbppS~fBV(oSC@ zyIOBRPQs5GPQ0=ufm__xpUOuEz}L!>S1!l=fRphhoX8zeT^X*2#Xj-DKkljwzDbR@ z|5W|ZSwFX^pT^NJ@?13d@3bA9?6v40KMzGUZ{&od z{CyO9I0`K=Zfd|^ki=i`R(`Tp>Yt(RIyd9`&Pm2uL*-I;+g^(-sJ_Iusdx4s5$p$* zWxnjkkr_|QS@B0`!+y%%+z_D;%B+xM`agyrKll*(1zFek4A?)<^%q*Vvye1P9AsFX z+^BPd&;oMKrjK+kE_n~MP@c^Au#E;DMUM8nzZ#(Z$oH>Xhq@x#M~ZJOWa*gpNv*HJ zUEJ;Z{**o7yrn&$Yq?7HKtcawldAk`JtfEV*+xcS59IZJYM`=VV3qR5A5yNprHs<` zE}=7Jl!$uCvj{v*;}Yn>uqM8krihi+v;Qv`bO-(tI$=W z@K;+`+| zGg_E`#Xgl4gi)W>j{7sMi(-YIXzpG4(iklUqZZahNzl466vH(EH#EFWhf%Rr%6vS$)_H#P>5^i4sjHcakaBlog6# z^SqwD-poZkhaG^|3VowLUAGc&Qc5RqSRCKZgWA{RzdHto$gEqq--SK?|8NCc#iM8! z>^AeE_zAG7(SPW>0Q83x-H)9;sYTn5zE=SYiw>(+9MeqJ*Vk_!IE*o8st zh~TDfm%KFZ>w}HeCO_Umqn4DNN8Q8m>x0-4JFz3mu_HE2up_#VHS0=tM5d)^V(c<1misRhAd@Z(nnP{-uMyUddYJ?Z|Q88rBK5reyM(OJ!2)r7}6%Ad{vy zuy%BjtYq%jA88PqbwRf?udtGNTmM|1YbJz;PtboSPKA%hFQ$*46npwf@w0r#z6RP{ z6SW#!qcZseXwg+7-%myiy*5%;)>%-uXmAu;bXy{7l{Z8eK0V_>=1o3+PO<* z$^{MLKf6I5OD=*|7GXCuU`OinVxAY%?>wJ1Av}2k3{IR1ACF&5f5A!doqbZu*Mlv> z`kv0HY(zf#fqcr1VpU?hV24=uK&O*f2X#*(N?%s6jyh_!v_uz0q0ctxllB<;)HYQh zdY)^D>$r7hmAgll2kVrs#_e}7n_#zt4ZQw)!lAZQ-NO%#f}|6p*nMEU}M+{Ge#|g zU#>6m%TD+u2fz42jnWyaW8KB;8m42@foB+dpI77;WZnRIqf-jDOCvm^`Rf`pQQKt= zYt<-cMT_AdagdK?uG_bl%9b_S%74l)IetuamNKke%j=?Blu(Jydq~C5nmoF6=h;xbM2xynBvXR*Lg!uDyg48*Kkc62{YTQ_Kx6GM+-G00@H1#%+-Xbwq>R)!g6{?* z!$(%I724vlXHLpk8~wsh_2}wn+>_?n4(05dln1pB@hS2V}eYuTwV>@lB{fhe^T7T>+C*I5RmF_dKW47+wv97z^{h8JUIy!5j z>@;!{8mq6!P@u}vv)Gr?5Hb3s9h)K`&XLb*4U%zrG%-_cJJ}YjlP2^EbufBmKYE2Q zWOiWnv+gSz|0MN?RuWS4R(w@^!dI|)3|~zORAsg8I*##W(e>C9&`W44?_EO3D|B=l z;;WEG{Qx=S0_$2a0sQaFD(w)zf$@Km@qYsUfX;~rpf~csACEsmpD`@9i})CI ze^jsVta&b7=j}CGly^xT`*wt*v5gsN-Uw|lhmyt~hc4{e7WY>#JJ7{3SqFW-5ZmcX z)NS^K)fY166ymD?L+=qS*i_UZ$NIEPk>1*%x62CjX7W{9+wQ(*Xl)aD-sZj>+viK5 zzb$Qx&WlZDO=WD~`?AQ?Fftb0c`o;bRp&o2H?RJt2FBz`#^edcxqV24rR^r@j*how0knQeTP>3IoBM=d8EPNvfFKvQ~dII|3HNY4#q(&3i!)f?TH zH61b_)11}Rnm+hmy)MIhrtL#-7uG-fgF{NAW=)|+K<~X-*juHcU00Uc_3&kf4r%=# zZamy&u1$wOX|B%UZgaizP{v%ZIQ(gIz5HFz+-t5ahwnDm6^HxM z7IqrHG(NBNr0Nan$+P#UU6S?%E=$|IFH2_9Wewj;PpRCHrjOHA)`m1HNYl{fIcY2D zq${QHjQmJ5eob4&aG#W=1A)eLWne;hpb;FO)pFC8NOLBjWs51xk`k7I0xkuaVQ^Ht zn;(7uf|gOh^NrzenQj7}!J;hb%0mat^A!c$hc7=gYMz%J zdcj;*9x^_W;pRgp%yY{~z+6|1h{|NGXF3WF=Yi*fBA!w3oXh(-c*enV5qQ>t*Fx|N zPrx$*p7ST*nINCY1UwVqIe!A4JR3YISM@-17CfVrqh&?jA)b`+Zt+C!?$$P|oo0Nw zv3#s|S$>ZCV^g(a)!q8OEONNT_+ZiXC0n56n^l>hYvVKz(uMBy=ff%^^nKmk4Ti4o zynsBf&6_ozH2`hQ!Wx$7On7wC>LtJ7)J|U$ef~?03bt!h8*WqIQ}5XJhs! zvyrDBaZfbw)E)tk=qX&~er%pxnD~wvPafG&h@-lzd0XLrJU;41E>G5}T-31Uov1s0 z===g*+el(~M%tXX?&Z__)i=7o(mXAl{|OlRo!gZ_nBbpK{Y~BlTDRrqgWps6jBjC3 zEPMlg=7`p@mNyW8+?T9dBG=cBx{t@_yY8KpU}Q_XV(`E=t2Z9gxnyO<<(@h3GH>%4P7w&5#kXirn&=Kq74PR z;%V#(^oPz*DK?DXj-f-6jB~7b%lrBYeG-&-+!wMt9oD`#bSw6;Ds&RR;Vd=`d=4%8 zpvm_($Bf)K>t)_!_p%n7-|gW#D~!%T&uKlHcZ5t`(veW3)@Qsf%d{QtiQ%k!;-EEl zAhwfw1;!3&-nzCwzH;o)cp1nEEyL^uD8{S(k$XhrH1qyo1RoN6qSFy$w~dj9+7|EF zUtB!}y;I$mdzt#C!{Me^a_$KW`cNIJO)^+Ab}iruf8CZi58M7Vt-l0z=j`i-qVY$3 z3Hxt%N9tbnCCSe}yw;b{w!PfuOT4^oLSOmRUq;+#@NHy^WvpcH>5q~TV-GsCld?>| zftTEtJWt!h9L;|k-A~$?driM3ke}Bpt#KVjt~9~N2TEft__a93nsUjj@G;}3slwQ) zISXLL-_*d=t$LfTFVZh0ZDg)6w^A5Oy=tOX_oSq{=TgQ~L+lk|KhoEcwEWT}VBP{)s7+M*__2zrM}3;H?G^%b)DZRtm{)JW9rbF#YwTDG(&u3m1np{2VzNmRVar>dZgI^iZO&a)# z>`uiy9sc00>F}g|#r5W6>2Pb>jDOJ|Pkdw~JpAIwn&B%)+DxCkf25Oq{zTKMy{A9g zkADo=sQ=)ek=FEL#R0oQ<66w?J)t_fDCb2ghMyi; zqhZJmhdd|5b6v$fHH-SZGSZcvTZO*Ik<9b(Z${P}{OU+3jpu53*GNzE-%_OJq2op^ zZ%MD6UXImth99JE*bmJQb!(DSVGQ{7`QXn-vgyd* z2TMft<4YZ7wY$! zbolWrn`d&bw#r2YzsB^^;`M^!^}WUG;^H+~xR&Q6p634QoQ`W%w+zL#PIa&1vv|@y znZ}VvIQjw_4+l0uuap-|AN=x2X!vIM;ffLTMC)ZLqtXk=^F02P?&%|FU-`hP?Q@eV zH<>ehCu=PYCNgt3*49wm)Z;&?$Kp+R_}!mrU9^797q;jHOpAR!4}O~$a1=k{?N$sm zJe?@g9Q1hf!V%{!;%`rfomJz(U({p_XiE1zoYRU_Hfuv$9!DP z*iAO2=jwP5Iyr5;2hT*p&NB2F^y#E^yw_G5?*-%yxG)wnxS(H}p~Q*$Mb@3+cQ3Su; ziH9C&-;zGSXpT!*QiQ@yA9bS8huDx?3AH67$%P}? z=556^3)dqH)?1oyAx)Q{TV6;n`b1-T(d5g>lQKJHnOL9EJkpDrOkBO!JPy8iIFO#( znMyBO-+1sRhl2-ONBR#gMkY=j*-yT9dPzHJ+U<0*-7?q3#pm|4x}2 ze^I*}yyH+mWj8iQsgt(-d{f7X?f;;zkrO9eX#b8bt&XJ6FSg$&lQo^Tzj<4iU;`A| zFV;=k_VmI}WcVwz*~IHEmj5^C|Mlpi0j`ei7j@WA42&exi`zXATr0=Zb>|G(cz@F_lenu6Fdi$*PfVhxPkksC#D~ExCYY~w_8ut9B$!0q;&a2_2E|T z=R6TSyoPIS`Xca|c4)tbrI!}@`GO)pzqk1Od-(k{TGu>ix6(SkzcQ6Mr`lKGG5pPC z^n2J~;5&Cr=SB5dd3yNf!)@Hh)Rq~(>2PN{QtVsRXXk?>cE0-mul>Zy2UQlZHmV0Z zCxaih8$WCpe%Ma@uxPBo5xsjZG6}h{pOpnDc_mj{Jj5?-#l~5Z$8l;RsZ=!pGR6L2fMUn zlQt7>Z1vK)cN2zvOftG&bdm+p)DzpU-5z410V0437G$40|1h55y+j&zVz_iB;ZxfOS#yI_qeD z%9_9fTpwmFD`);mO*Mx?Aam?$higRqJvs5;Cw7eSgpEg5!(?lXy`w*h|1M)skx(IQ ze@12yZ%2PbKCquPu|qitPS45g{f^9d&=JYj*cXw9Kbo^AljiUDN9==|$7jX2bbrLR zaDT*lE)$X2Kf+h5b>qy3PvCoI{?cASy>19ad}VzR>ya?t+&RA3PO-MJY-*cn#nu|R zhPtkzZp*pawLaFxKAPQN#_I2AF45q+GQY!ISLFLrlf@TbZ=H-Uw@zA@uZdV;$Jk@K z9#hvA>UzsB!$<5@F>47~4D-NBwkdqkOKg&UCy^%@gXhjXJGo-3S<(`jDU2srFa+*VfzYzo1U+ zrZ9Ok$8W`2%o#aaml*4nS%W#2efHEjJd|eE6>B?m9=-RPe2;0<<@p^-hi05uBR1nt zb0$oz+C3Qyj?i)?EQ zbi-C>FtTb?&ZnKWFVGNSZU->JaqF|J>wMaMy>FR&GDG`I*vTW%4zB82<|$-zSod_4 z%~-^oG;I|Be4ig%3Mm#(Y`1i6y85bJJ&Ow8uJvWGoH5&dUe~{Bxmph`SHtw(jH&M7 zFJl+i1;G#8X4berVLrxsi}~uXrq^_u9=Z8LO)KSVa#G%!v99O*svkhVl`R?hYzyo1 z9m8)H^E6VNd4G#FS&X^2A_w{ybN4Xj&M@ZgrZ2j{ zzZ2Zs7-z5PRm*-obn>~@$Q?nE-cPZi0Nd}5+xc6;^A`SSv-K$_>YK_wm)3C9`b9Eo zeG2;jMR&wEwJU<&j#xj=N9FQd)cP^cm-D=*8@}#$b`4=nD*sO#av19eckPl|dsL>& zj6`mW`jaa9bGw++-|}^-m5RTI#lI)Jz9z9<8a8XI(mukKG~~wmV8#bk|p9j{V9|WY>KX-t~}Fk*A+>=15Jqqh(vK zbVqm&+kd?^f=>=vHI(7G%G58A)i{3p;O?lD-5tky*$8EUdv!+ZEw$Mk`J4Bo z3rR~by6c}M1%G9>utvc=?~)|t>Nx@pGUJ;xDpx$N>^;lQb{?opQu0u3=kguHcsF<>l-K>?(VAJJHpG@uIq%F#> zxl5#MzC+z2n$~ye{^)p~&E&baE2{OC&50=W%-Atxy7bgguLkNGrY~a>{EQ=0yPREv zw6%I`Zp$E8&)%Qe!Z{AQUY7jO1=PtGif(y8Y8h`;X*tjft5J8W-qE^b4Bcd)LETTY zAAX^IzDsvUwtN#>nARP&&+m!ItW1VH=Fmrmrf7>}-_xgb6|p|r8rgEcETqq4?0wDA zS2=Ve0yGQ8H##!=QAa9QIND}A4Nfm`uGy?y1bT?rw`L<&e`l1lnW9_zCE>dW9H3Ro z@o%|Dg7#D}W(~0FKx9}AZ5CapBh|MvqqotAK6NNl`_mct3B2{4{^$dFX!(UugggS; zX4BSf$c@@8t=#HJFSt@iYdLcJX6T_4j7HG4L(mpM!z9r zPGr~`WLPUQtfhA^c|Mwq^zJWS_Z4K;9V)x-M2_9e-%7#)Dp!YcRww1!;hg>aP{ewS z{TZW~xaI7RTaVF4qxm>%pOIG|NB&^1OzzI@`lk4;7odyhhoUm+Ms!3NdPm-D2uF8) z1^t6O^Rd2h*EhhP&70bn650(7WF32Zf5iU5Zk0g`?ayYx7k-kK8ta>#tfAeX*~NHg z*LRQ)4GmJ@sq6|+mXJ2ItKeg+ zlJZ|?eEag`9nSEs-q6rzspof)KTyLK%G$DneHhpFN6CwGt=n?e#;e3?$$k7Ky z`IS5q*2V_v(Vvko_OrjNOT!Zq?1BC=8M`7ARodV35c2ey29=Ao@;UO{Vd_}F>s~Fp zp~mpXO!5RH| z);HiCZM(Lc`dITCB9xgVt{;q&$~UH6~%5N(7c<+F3&id1Q#K`S%YuvgI-|&Of zKk&VbeJlI=2PHun;tS-WyiaKTSof*ClhJoZey!_?N@zYZA*c0^8GO(k_I>>ko)?P! zk&M_|9sAMU`s}whv_;LHG~^DvsB+Lz8C!a9ujo@ea+cE)r`{^Vq7vwK&MYZ8R~+~jhqS?S-v_OB`@mF!kP10qsER` z`9l9%ebfgYM*bvZ(ltKc63!GW&qeGnY(=+zjoo(L=x5~CtRctuedN&d-L!MI*e?u4 z*mJKKfom>f{{R{JgWcMWFzbz@ayz`LI%FD{%o@tbP$DXSJrt3$1;{4oLUjUT4C|+T z5$j`XBK9xCQM;VB&bS{B_J@433_IZtWM0h|xKF5F4BB5!M(iK9MC`BjMeQF_7PN~z z4_Uj3H~k-s%(Jg?BJx;U#J-09c`OO$?7hAT`F9U=_XFhKbNnF+9{5kLJ^Wp5uoBGG2933MDMD+S%25tn9sQ1u-DA{h9$eU+2$>kWvyp}NXhU3@`gLCjN9{3$wPDF+rGhKd;1aAp4MMb-$q<-?qgb?Tc^O$(4Br#>1QoG-EA$ zk2tHv7S=C?$tyhbFWgrtSLS^cSvQ)E``DkW<=Pq6inf__0n)Ipf5uw&c=|cV@V9k$ zqd)JpLpJq-W_5j+vxz?Hy_JN%;9mCRIN1GXN!EH}k`E!dS)@ zacXw7Xo>v#k%_vG`bPJ~fer3h-OCd8W&6Qst67J8dVOa8A5)(%P@k_0-X6x!#M5@! zZPYziYV*whqirT`c-Bwd6SlKk`zK)5^n!5_ckYs=-Lbvy36_TQ3+s9rc47wi5+~z_ zgA>abqhuMQaFhdkX54HCeL_1{?{#0+H1tCtwvXLXoL_4*Z^{_sMX(BMZuB>Hqg{({ zF~YqcU*au1D?RXy0BX;8b{F^Tf%C0i>vrk;qtM(@XzwVrc@)|^3hf<*_Kre(N1?r= z(C|^{&u8?#-KFIe$Fc>##ryN0M!%1yCTWBA$I;&bbh+7YEs~4ZPs>RB4$eiJEcVis z8L1EEM5bNglggEjPn@_}YhF?7lNCX9c+!zQ$%u6;@^C-$iajWOpJzOFSGn|U6kp$l zav%L_$zLEx)DPebHGy3d&*~S5&~9vNo@sw}8|K@?oLRv5SB^JCI3M_V#$>*E7h7T- z?~cq$x<8v2@(DK08_xJJ?SqGDS01cvov*BK92Fhg_CA>Jj`0Zw+9|P0t4r4ABJ!#I zk=P{mFivG$9pr4v%C@Lft#y1=$lkCM_bqHgMz4v-7t1u(_f50lN9%stmnqW2Huu-` zPMK9fJNQ8hRxmAPdT)FL?UwT5YuS#Inbu!Oxqf>?*UFf6{=cM+(1X64@#pR@;(-ur z+T8rLN%yGM1AFZa=O3E(U70uQhkLIm`kyY(-(=pe%y*@xawbf<_^@l@?0wb!j@%db zzlPa^ddy+WQSd`p8;d@O<;dIiL_}#-`Ag%QUyt$|b}zmOrE#(6`c(G9zdrUn{k!&4 z?0Wn#jG1d6caJf4_bIJuT2r6P^BwfV0mi$9b^K@ez2Z0f{Z?BrzQYmO;rLZ{+1K(t z1bsH?-(U3itjV`2-Nzci>IeLNjPnoJ2P^E^imwj@tZXoTH}X%@X&&m2+0pd+Yw_hL z#TpI8eN|3m6TFCjNBL2|yudUuy!?*5eDzWYYMpScfPYW|K@PZ#lBkHxm)V_(}Bk>O-SzJ|S0 zbC-{OU#z5#_!yV?>{UGxIY8Loha<{2JIHUx^?upW;kS3J_4}dqgX({X{nC~h?{^>7 z{U6xzzi*v=l3Et__;W`ZtU(2A!ej5f^E1jmEQza zihQfNHs`O|(VrXO*S`%0t!$f~Z!#v5Ck*}J48^VLT*Tg=iNOCp z!J@4Gpj4D%vne1ZNEFV>ptuj>}Py1oT-W*&k4m_ zZWD-pzVP?Ao39^3Fp}eSOHto4ZzseEzwD-%8go zS=etRf=UUIBjHk5>>ZaeKv8-G^UVFQH z6u*$))TOO8j@>NSqkhs<={YUq{RVPC*HH&!n}Sle zBq+6?bO+%zhcW3I#-y!`Nn4<0hcRgbW6~t@JAsS~AEhsRg)u374lP`sPnqk=e1o}O znXfn3mb{~K9iqghK#rIEl~XOo;0;Zje8M*{tvsD&~SZmQqN=tpv7bbWg zJjXf8nRyqxnYtz9SDnKj5ptOKa^zF^vFTU!qegXFeedheNN{#e z<5|D$j<7EpzfKq#0uRC0J{vlp_9=Qcl##8Zxr%4b!UE%r%26=Ns0?L~mQVb9e5&UX zt(GJ{WyykSOZ*vufxF{1VPq9D`~v>6KNj&j`aZ0CzfRd04~yk*)Azynbd3)z$+g7a zgO~k2_dDvVH~lqOE;H_QA5wqzo9Pwho#HuXm9C!6{T+CvxRw=udBa*)-A4rv@eKsz za#>LHc^5oy)*o#be{Xv}%XkqsYK^#eXH! zK0Q?5e#b@YUgC^1&eloII`1o~*`HsLs(EN~s`@{^so|E>6_?T7^ld+T1E)<9$;K~y z^(*(^aB=JlvFYdTsJpuD8y7BlV#U&zI3F!&T^BpY$}nyX?D`6Hn`w*N-;hw<2AA>4 z;=OgZ%U2k`i=D%^mR9A->+Ecn`_3Rfqm9nZO?T~;N$+j4 zvMowudgk~asUJx;5f?iUTQ+Lm4<#ar{&RO!bCiSjx4jOY#8#2A<$KY~K^EQ{-jxSA zd#Q5%UhFCSI*F)N*%FmUu%Q-dcz;l4e5FzCpZKXYsy6^F0%>8}Qz4837 z%k2PN!&&eZGqbWHUWS~~eB#u5St8=ksXfvXPwc@yb)b<-Xyj37Ylfrs$5%w11CAq{ zgQE2?80owVG;is-Z^#ZZRkI_chx;n zu6~3n^*^vi!kqarz+4ctZVld+?`JPrR%r@3(~6vVCu=|37-v$ByH~NhItztUPtNLi zk+bx5Zs9D&2K)S5a3y_R*biIe4^F?_x1V?Tpv0B5DPiQ^ann4_4suROxx~vjj(KfE#BNMR`UY?t ze3N`0K%Q6Oqp8Z_J7w--pflpD%3&u{j>C%fVRNukPQ^FmQFg!O*~}yfX-XCA`or<7a6PwC3y4FQv8rdY<+u zdc*8Zrytqdcy>+CQ(O)Hjrn?=x5GDW_C!!lh=x-UZW zMxJtdX>EhApu@tNPW2S|GcG>6L!UKYz3M#m_e7&wgHFMQvtoJJCn-#^ix^jIo zQd+00XTqpWgs;h*-9i7=@VydqE*NzwwGC6EFwduSrgW(7gGSFdPp}YHpuqx-vRAf9 zvlD1?3N*Pb-)v}7&$ir??Ydkpkp1|)7RM@jPq{fJS+5{>LuoL%{=D}+kGi^V}WMr<10)b z-<5A2q&}tcu^o!%8s=N*11Vwu#{Qi%_4NLoz5h-6x8?2${VUa+A+oCQEg>yW`x*JB zdtCG^?_z&f50%O+;$H0UIaBp{YGC$$y`I+Jo%xpW{w}9KwXX~NTTWYl{6#2H_53^7 z-(u_j7L89m&!mH5eULHyN_(GU3|W-^^tp7dB=YmarFW{&l{R>{M)TD7+^;USwXhBd zyj?@vbNU27Mvd^jJnS~50ovWG>(n@h;CDsdS#5YHSXf)6d17aE>-r+v-!P%?O7vU8 zu}|x*`RVmE9NY72#&Lvi434aCk6q6>dd0Qv$}{?V1K;c|@Qt<$8)Y>(DBm$Y;|w0< zIo8F$DZdrd{XU$bvk5o{j6TqECSclO337n3Vqwj`U4BXqeBQ5i z>*;dfx^nk*eXsSJh^65J<$q+8>KW>0>Z5&2eJZv8OZCz^6`zZ`i8C9N|F`EmjqNs- z`@Q*32D{`k$_bUurRDs=Ha(NJW9Kcen)7MP7Iy1%YSOCXHaegMJc)~t5ZBjb_IR29Yz2BrVN9Fq9f&3?tJDp%#;;*pA(J*}nZ`@Smjj&e1 z$o+Btcn^AJB7fwRKg!u3YRwFd^T$p9HU1!Pewv?N&)^SQ`aiB$#^njLG*0JAL#Jpu z^f)xlTH!bDK`}h`7BsE(e^+!3_WIpG7g{`%w(DXJrf}gbm#Kl!-%=m;)SpowWbcJ> z;uYnuU6v^5TF&59Uee!qeTwIBURO5GXK(O1YW%n1w{mj^#RQ&vU~YcXsd~;jLOnBY zsAowwy#;RLb>uuRV-p-G)bY+lev|sN@$;0ceqme@G3S$3h*QulTH|-EU*4+S?|?35 z)Y3lI9G%e*Z__@l<(X~#J@Rh69-2<&Sy9&McWs?Bse#&u2i~9WeY?6@ z`%cN>yB@~hqQ0F8_Sm)i+}HKJ%Hg6sF#Kucfzg$x>tW>^t-xt?D;h4L`*; zImp*>49daa(^?0WwePEkx^J86H+A257y2G3wqomQW=LVafQeWp|N{|D-i@;BjE z&UyWPt8PpjC?yq7X_QGB;($s9>}=v0Yl!iMttTQ^Sc=vofvo|^s>GiTZ>J!H*{j4t>CB8 zdJSyO?~b(}9*Q;QbD0igGjn8Wm)GJW&o<+G$JQc`Pmi&tPI+s<2{Ct9*f+@ii+Ug1 z_v>KR3H?IL*XL}O?;VfkM|t-`Y*Q#4+sk*zADMS(0JLLkUHxvk`BB#RIrgz8*6MtM4(rA?dZhTOS$Wc zb;Or~Pt?(KEV9qJ?#Atm#Vh2p*1fK)-vFp?lTY$aEas4WoY8^;E1&m3p`D+3M5Hw< zH+|+A&Ys3M&DH3@;CA=7@e`uwI=kAYJIcyGeDncn8T?b(s^j|KO)YO5wY?8gF7pq)-=y3j%H1Wg-Up}~dtnEzBM<60Kph9J;Tzjm zOK@P7V9Q8o0Dthn8VOTIY~cNp7@$q+`wimPj7^0vrqVy00VtnMENIQTU(!C=vW-;) zj74I8yvu8Y=P7SwJ3=p*>kXk_@V?COOGhZ*06m%ewV@YzKSg;2`l$HAHf)S{vBnP{ z=0EJpiUS^sO#p1{)?Mr)LeJ8lWxNkG=p0>HVgdbS>vIvq8Zg&y{{29)L`cMS+@vx@ zZO3<85BRHSpPjoybDJ7C_XvKc7fN=QwpDG2FWnHzn{hC_VC-`}#||H?8Q1GMcIEKX zy%Y9lZ)AL@b7VT7cn|wu2-CdC=SAfwa}J)Se-~-{v@gcfvR}TGR=<6+jq&LA343mC z<@{XUYnh~b;a#UI@HGCfP*{Bs_@MWf_U8Yi8QF#7x(8qB#(Ym;!v1`---j^#)qeE@ zX?&FvYWv4mA{W%Rw1WH?*Th%OJEVJF;t99mAIr1{$oK8fPbuU}d}w+9j1}&$HNCdA zw`J-nZ2>lr%4_QKY~Hw!H?K37__!9(M{*;xgO7Bc#OnKN4K zpRv*w|2w=;-}s%Eki-6XV1FA9Psh+kS#4wCTJ0q7r9Pao7#tP5VO>pqY@PU$O{}uL zQRu!7-0h7TP1*@nP8WZRBXh!h5BHuyf(b4E~S^vIvK`(l|f+Je#=81ufpI zXJ4uxm9;XX+(%WPF~?q~(cs%x(IfM?zJh+5%=I^1r*r*vLp$DU&fvrU@*D9bGTgst z$}gn*Sqz!ZyJLj^B1L;OUu~=Ady0=E9g-=YBRZ*YjvG3>WUoxvFBs{F`NQ}e8T%E# zr(wROf&3AhGhEc~=2Jgt694!c&!%`Yx_`5CDNO%N5^LSczy80e@*Fz&pDvO~tA|pP zFWaYcUpS%T_51$4I^O%K2bZpV{^t1l_p=h2-w(0P%R`0pmwoSP(DOqvaB*oCe?KA| z*<$p&!<-c|-PkL8^)R~0(J>o^CpE+MiYY4A@Ps%TB z=%8)CFCO?@%y$-Isladcr7B)}B4xA3th{V)M|bsVeyw~odY|F?eMD64q* zzDb~YNWbl=V-zW`AL?j-di~PCcXxIiTYhmzQHyD`>PV6D`z5g}y&U`mmpRX!B`>x;& zQ_ecR5uI~=Xc6B|d8xh~emMh+63r#z0I`e|K$Um9>}PnECJbF@ZN6`#C1 z6$QF|m|(r^KkQ*`P@N1wTRIo^7;~h) zFYJ->>3diYSEn+}x_J$2if`XzeS8gc5v;MUY>mjIjeF!K)@rr3)|olt${+5rE^eu_ zrZCSN*t>^sSJcU`8aUJWZ}u2$UuvjBE;U-DVICNnF&(=jRA+n_vZh8RPj9r=uBo%G zP#SNo;Vkt=tC74H>T^qtRPsER{HydCo(S-4Q^t$>+`xKr@_U*xex%RI8tUIT<6-yv zMh0((o-gNbC4VdUYvE7NmIk}h+>IFnr-OA$&+JDCD?Em_u{BPY({n$la{5g9%zCQs zV%iDvW-e!GFwgpjmp6-3KImdHbTMi3@l@rZ1*yQEucj(yUYD{zlvf=QYp3tm)AuWX zus%M&_9sgN4=jxxyWzoDtFPm-RK>HTF zK$`iE3GYjDDMN?T zx-XS!%n4ql^H|jlZ=}QMIfc2lKdjz6O{SM4)BR`2^nJ+mO=rmT@_NptkDV^l_aW0a zogvc;@w#GM@)^Ei)uToKd}$xNu`T)Tffk$GggvbO*9qT6V6B7sE<&jfH6A{2 zYu$Wof--CZc(pJ;b=kgHxtddL@%o+E1naR0=3^7A#3uL(d{s8V9#DNX(H>xLg0`<@ z518{Q8Jo$rqCL>eeNjgldqC}+x3UL1um>`%g{+8l%WdJstQ}n>SEwFi&Fu!{rP>2I zwFlbx&h#mK)ZQ>-BtLnVB`!5M@y!l+sLo);cRLJLk4zXh@ZFBn>~O+P!^uXLO+uDc zBFh5EvI=BbIkK$mLwS{_u|{OsN@Ur5WLcp0C-ujWW&4n2JCSAjTN%WDJ$sMobM}L? zo})+yXE&O6oQ)2RAfu{HoO;H$W?hgy+cDv+K`zkE9NRpb^+IRmSk0^Hv9*zB8RQw? zq)3&4#ZL20jqZ+}#cyfY^cmwF&XLD%2wi#_&Af|!{-!PbqxwhbzA#QObEX;fZ}sk- zt^1J8+5NrG%=QP@$`m6H9L7_~JLW`jn~02UR9ggjw~p_MZNh)>^lbGxSp$1!zhOLb zMma$pXWZeN9Wdo^Hsl@A*;SMi)^bw*`yWe{+pFt0y<~Ol<4k>jtYDvQiZ1Kecg?@) zcYR9l%wCb@oB_Y-hR@dT<1Bss`8hM6vrxoW*Im56#=6E59~m8_YlhS>H8^*6mDYLC zo^8H^Q<8C-joM3EZ+ycJ7!~K4Py1e~@fYhHS1F$QDs^2{v3;guAN;=yBx*W_OY%ciu`U0>w5LT%irT%+$?jsH@LqY%b?c^ z_0==V-!Na*+cA62>OS{5Er*^p-v~Y3&WQD^oq_Rxp7B585A1EB4tiZq9rU_vw%Px~ z+0*|oac=@%Rdx0M?{lXNByfQMVLp(X5CkM(#3&$F2$KUw21|Xl5RyPrLJ~>9=<5&_ zHC6<)R;gNz3W`bQ_vRFxsVXCwa2-{*Mh#GC$XuFI$aE_SAw&;kIIIRW>@M$w*#D`uzOjw*AP=_XxZ_*_G2f z)*Cyg-BpfL7U#X~TqkiofqGDHCns*(#@4%>dPDE%zpU@VzbDG-_Xa-Rap&hcIm!0> zu<~EKFH;G~`XBZw?EVL|+?5yYos+?PUP&*z*-?ru={g@f^6kdSVIDDw=Qj9e)#2<$ zeJjwZO8pOZk!ZJzVst9ol}$|SolTi*JX+x%9aT&fxQo=#FjsAKKNnqixG#&(6_l)_-hI z)t55#-JcF^x&QyCw#-5a9rjr|^k?7q&=<2zi<>Jy;kdM8tiZ8tC>!(oaQO2k#vs~( zKKzyRr*D~HAv$vCT3Lw3)-RP>wtI;5J(*Y4!Ji&~A%qw`$sr zJYKu)TzgP^-E-LXYUQ3+s%hbSq`X&JV~J^M$qi8+>u~SA^xOw;fJ*8ARWwnT6al(4X!b+=VZrzgb^a zNxh(HuinmEwQr_x2y47r*J0ki(C(GlJhQdFfkwR!I_qcoBQs;cpDaHR_C6_Xke-_N z$Ha=>t`A3ixIl0s>KY$T9;W>whqNCXJhuJ#ra|w5|A*~&ke#UxhsdeTTiIC>Ig|e{ z<+Ss9o>9^n5<5@U*$&$u>^#|Uh9q5Q1*kyoXgy=0c4I!Na{_!vV_gWJ!}*ajWO@1e=QgZVaPdEZ-OzNmS)fnOq~XLUmBv_Hf5I@<7W>jmer zmIU4le1-=f=AFrQ&)z$ing<`d#^Su}wPF99mjC`}*|++w@Q2ze@@@0=V$Uw27Z*9( zpQo@s9RK0h!S&V*YxOKtTA(ZG2R*rZjvHCu3-lzsuc!a%J8*%n3Z!r3X${Ik4xZWf z)-v*88cQ_hSeY1Wf^?otoH2~Ce$lthUg+1qT;Df~74A2Dbm_076Z6$q>7O}v9v$U=L>l^A zU-|2bq4xW5_WW@9`u7&=Jn_mCdvjkJ{pn0`#IO&^cj4B*`kBrX7ux*=3TXcZT+c=; zXP7g?75Lh_(I0bc44-{##>{fQIUmk$br?5yY0k2xAg^ai+rDp!%SJoSul>HnMB|`& z*m;8O9C32*4EEjh&Fb$Gw^PO)tVcCQ@z-3;8$F6SS259~WZlkK&inCxpl|tnKDiaF zee68J`1ihoF@SZA__g2v(m3GHgYEq<=#o2-^$-9BFZgRw^t9hQyYF4C?m~SU*`u-V3&l4D z{p(Qu%~yS_{-*Wjc|h%x@Gd>p&<=8NZU}$Em|Kia86)k!rsh`M&$06>&E1$&>AsGg zW9i-#FyFE1!d%1A{-owuN1w<%Dct=yyN49wn}NC?YWE`PV_Hktc@#q#{);`U1oE~z zq4=O(w41ry)%P3seM|gsZAMv5N1yo3{-E8Q^pbZ(jprsW$v+4+E}R6WuSZQm@{W>&XvuZ9Ioff_ zb@Yi2f5}Ta07MyXLIFm8h-b<^DKB%4>`i#to94#q+y3}#$C^aMBsV6F!SK@C?*Wjc z6`f+gj-EN+GsT%sd6_ z&lf{oH{-9Htj(vOpF0oM5BG5gCf6UVqw>i-&$nLqLv(d173%PJkgjOE@)um~M}3Sv zH(|$3k`G|Et}_erEL>U25J`86q>_q;vNP}3izBh8DYFVSti54$(i zwecO=BkauuY-MOCyHBU*Ks2V=xk!9q8)Z3d{#;~CoIb-CVM>}3>PzE7EzA+jzEH@H zarjX7-FObgj=}g~jbZkTklqI{1^EJBF{JHl(P+;D^AyTOCf-1M;H<0ahg{w~fct^= zTLrvH!JfUwW(y* z*=^qmrXGrKiQ@_Zd{-%eZv0zuuqWBWe?j;l=XKLFZ8cuU6V8Xd2=yq;ED8? zOUFf<$XS$2&*?EPqBwu+*vnezz0b{#r|%UE+qi+fl~(o@I5(zr^y?;T$Go#@WhHfA zPd~r3Xpp&`w0mvZuhB;&5e-?pvUzOW>F1ZzWn8xl9~d+9-U0WaOx4NSAHHl{C}dXB z!LnxRK9Y3Iw@;Z1vIpK|X_HCM`*9AnL(u|Vp!?CP%0YICK6!@6uA_J^dB5Lh>^iA) zYHV9##oNh4^J~rI)9PriB<-`A_Nn8%=q8?(-@v;i^d#|G>~{@&6V2FjQ|Hvka@w>J z8E%vevYABjR^hkZf4eF6ct$Ai)$9)Oxt&wPXWI0%b)s(9hLx4OHmpi9S*%fc=AQOX zGJjw3YF+2l=yKj$ms}O%o(oU0#V19Van0dcX}{IM`)?BKUzyLlz>>;Gp0%quF9t7H zdErMJs9sivR<4DsA|}C_IN!rZmy-v(viRZP?b=a!!S6kgcdJ2$!mMQUgw?luWqCZe zT6z8WT+wLHa?s>icvk`bsS9g; z)?6mcJ**_{{V4o9*uZQ1R^QH9Ok^%eMqUsPugq7Pr7vq62g=xgpp0!rDr4oW)|PJb zBl~$?KHtLAcO_RY+?~)lZps;+__&o`?9!k3#PEBJH^ypy;oU*P&7R!$rP6T*(dk*)_BVV4p4=TLJ=ndHb{V&+?S~`# zo|VPBl9DM=ysc7oL7p=O-(0-Q<^#6-?9k~qmOkI77l|+Qj1Ki}wt9)0KIriJ#ENZe ze374j4>a$bb@4XYv&uag9X*7vSY1Bx%%Pgsl~4TSGJS{582meZPv5Trzdvtg z>xTO|^PTtQ?q=S-ulJx_=qtL}o?8_xr^WMbJKFYXb(;)h7=)N)Rzquyzy=zhlaY8o1CFBzd z?#nSntA5LQ^WInQ`G1}d*R%g4k_x$JU z{YS3er<=HW@-FUK!*8^^0U(2A?v;}^#C_5Rx{=QqVZ{Cv> zZspymyi>@SP2Dh*BWbUK_H&If>a;L>Bx~}`-nDlTSI!7#@AqKNL|mFZlK#Ony*AB= zT`SOGn6+kbZh~*Z%*3P;%<)!TY7(sfhs~bP9`+?B(XG|QB=etne^UrMiYHB)WEC!S zRCCVvYrs!jGxGai(9fCF!7QW+LJYixy(B&J1#W##=?m& z>3a>G*bUJ-O6!qss-yk0=SJOb8b9@oP5Je_72Az`$gTw@yCfNEwf;PWJzc^Xn~1wv zS@jJK*Uf#`FC6B9k_JA#KGtuxlFQ#Fg#{)&-H!h7BY5^pvA_jfH=^Pji4I z-=F{7=4lGO+P2P)c_*vhjNjXQ*|#NDygv6GXyc+|Z=?*pGt}z6aQ#cBLz=poE$pfK za~;cL#p{j!T*>AM=1SV@?2yln`F{EhiBNcN=yucZo%M`MltsO)eW=V1d0G`mu6@d9 z25cI~ySk6>wkLFJXb)@ho^GSL-W?&kuvhNs`EuJcSH8_`KEQ7~>(yvN?-ez@ymh5$ z>gS0S+wldFZTN%&n+83P(|2H@=hg3?=E=Q1JCAXW_KJt*$K@O1-zK%T;r=Z58U=SK zt3d6jHYB}0kHXc?`@l65w{yP4#O-y&r`fbwCPw_^)v|>tyPIgs7vR}1G#`1H1M zHav)J|3q3_+nUQ5n_GA<9Coy_!sM6S#XBR2cN=OV^dI9yONO16^z6*EBy;88L{G5C z!&+<~_L)qC+SVqR`oKzBzJbQ=)u(rR-mIZkc3lC7&_#P(Oyx(Q_I29DIAI zq=OF&c|MGK)P7>JeYN$ zXD3lM(d8q&HutWZl}CBu>D%=7#-2@W_gg*Ui|Mn=)GnGUCdziVFc+}zQB+@&p3tE~ zzvZk8{bW+xx5DA47yY1(7aZEgZ>;g!czfmjst>-+!t%4%*)}e+GI(t{OZP&k7i;9A zT;5|!Te7#yyEgQ0(>07ujOo4PGge+gTb^XH6yMcs%xQW@nc`Q{Ptur&O!wy@)BX9! zG&7L5t}Oqa>zQoZD4llbXHBF#)Am)`m-UmS8RoC_ zW7fah*Q!nEPucc5dAGlowzN&~Di<%(ZCN?xlk5C6S<0(12D)9_Lf^A;^6q$E{Tn+) z%bWSe-UDml?fX~*XV~(xOoX`a2UNxzj33d&EfLD|W=y4l9dyTV3JrhgrD~%DY2z!`a}qcD4Hhe%=D`bH$UjYs+EssgJg= zd5NEC^RezFp78UHBaY7Oc-cr?Z7qK2AN?IH$2916J&E5LKaPD1owvw~@1`fO$;}(} zY?I~==pxA+!1~yg#opWnALOnIwY7>TxCmD}?^9W~MB1M32>&_Kw&f6b+ad7IjxhQ6 z90GsP0S>`$?;-HM07fR(GJHjMaNM>>2YJ(jx`rldP15#~>UO|pcB~OS^!?@>PhYz` z*!_+T!ZF_{=#YN1a2lUG;Half*O4!EC0&cF=XHyb6W`F$j&|%h@;y_5ed<|l+rMp^ z6+8BsUI+F)I_8D`Gk}mtYqi=CSGVI{1r9IFNuNQzjQS%Y>mN9NX5`Hbr33e4}UD+@BZMD{vYVTnELm92wLy4Pj z#wAv~vWaVE;)PclxklLUtg}8FAIn~5QtOwz`mN2#QOF)$hjO=@yqR0q^3LA~wk4m_ zN%ar82Ye3sWdk-Jeh?oJT`vElbUh2({z+exTn8&}zNG0# zW!Dtlkp{d?@WTFUMjm%_+c@Aof`|9}KK^{&&&~zjCwN74>XW~`t)+Lf`$vK1!x!dc ze382ICE&Q=U!8I7+Do?V_z>6?eAdnH&Ny~g+jqbPf*bl=-E05fc0B}~6nw?*b0VY8 zcwh=}iQuiBrhWVMKkf(t&lfy!SINblt-gE?aHZf2 zKg|B{rw`mf{oia9{NCbc9~e0HMD+P)v*24YM&@s<{N{7uwSw(Ng&&F(R+hoMtXk&>NfGK^IR!g&cUNgZ3Fa60TgZ7!`(a$Ytb}v+A4yDia=#3ki z-T&Tq7{fj{yEvr?)i0xp*MT<-mS1Z?i{Vy1&_7Dt#YXzb(*_{cMGz!_aqn`D@X4?c77@G=C>{W8>^z=bNLj%g2rMzQ^XZqA*dFZDIxGQ%W`jhrYoOM3lbOXu-Uu>3eZs`>GyH~Y%=4o{n(9x6VR}~P*mLwcRM^B=!%5Qd? zDdPw9Z61B~v}X5;0^{{RwMoE#k4L{8K9|#8E~JC_@AK$aZ*F$4Zua|Ehx~t3o!n;k z63V%Z_KF{p|83;@eG^Fb&{h9GHj$pX+&sV8{Z%$7m!Ll7%k#%IKD~rAbv~^ly~U>| zo822XL6BuSnbSpZ!W#?h&TRi$LMU>xq82V0+zScCmH&CA+@Zamvucr>bX!;@j zP4cPy>!AO<@-XylPyXAQo88-Kk3;#tnGYS=ZbweNjg=G!)$i>M%mcT9^XG|gZz6rG z5;eQ)>8Cb>3+U_fZTP&c^w96x$3?nw38FjDsPEf*iHp9*Z$B3>pzmvTcZm%B&OKZx zqdh;kU8$-sIMJtZ>BwJn>Ko)=*W%l!OFAoG`Z={r7tbrLuC6W1&s{d$&0RLsEp2dX z7cQ)?XaFL{Z@N)3w{&Smsaw%lS-Pygp`siX#w@QWYp7l7HY_b&Qde7DRa4Fx{>d zc_`ZcjsL&+lc?_WnbT*v3o2^LDwmYfRQ1cM8Y&=A?xd1AUg@rzTaTd2Y0x8SS=Y@k znK`}O&22y0Qj8h3u!Iw4ptLcfyRe zOB*UiT7@lIhQ>z7Nr4vYT-PlvD@!*c{;&T$LF6@;l-8VkpbLBc%gVuzwXoV6+8^`6 zFb?>cipH|zB=0#manE;odAtg_ep9-xt%B!;Z33Tu3Vox#Vd=86hN{|{j_qEtWEn0E z-(Fu=LAP^*^wL4T$E;&cx2|HTXGirFWwkZs?RJNnmMtwU^ExIa`+QKKY)9jZFda;P zNofQ9z-NG&|ERJ82dnYITIe}+1vkICbm<})aebw{FH8q~DQ5{TO~Y4N-Lmnx$-1Rg<>T=P?!v02TwSU}=NHENbE`*>&!sCP1O13- zfR~KwL_W%`KeyiFrG{$xV*uN~=dzd4}o|n;5zEh?v&8d?4YduW)$^7t@X^Nei{s5f`fYD=jn(xosx6m}^A}c}qpR2G>I8xH zQNBPF==;<55(3z&0{L?Sy`V`(;CYqjd6LLu*>O0K*G?0HT;l6hCTHognvQ)ur>qul zT2qS4QkTWsRV>Bv$yMP1t85`&QA}9M+O5m=9c_I>X~QyEdyZ8M9$O=4yVfYzy5b6} zd$kpv(^_S_{%Sl8`$Yxlj(%$LjN)|D(cX$roiW*6T4vMEEG#Ur^A|4mI*KcinT1FB z@MsSnHO9Ys31=2rkaXaYYi8jw{&mdJ7S?=7_*1}(-zi7K$e2@4QY|#knsv$vvnOeu zX#eKmQ72D2X>zd-&pK&(iK10C4I@jdmj%~(Rn>Jig^wRo=%*KqqbRRh?jZy(SXzoQ zJHS;9wRN65v;4{{m!4bS;7#xr)-4-ZUsg(Yo@cvI2iWEi4DS7Aw4!v$Jey2>eOT4=XSp69AK$RFs?^q5Jd}P}4KtQSHFjn#m}-{S`Vv$v zSu}F#q6MSREn85wl=On}Ny}@imscFsc$9_Z3z;h%-FUP|FE48-YAo_3wD~JEo9Y1L z6B;Ux#X16h%;D?lNj_X&h0|4GKFrY5P`k)e%JQo6kxObdX3WDiunqt|ciHmM$Y|4- zG50AOS+=z7D3r>d2bY#tS1h#{4_?}?7!R(t&!QbjU%71ILPkXp{<6TNmz*$T`lRDr zrO}_w$JJjJ$Q~>{Fu3a4T5GrqtG(;-O_ppX#Jy`W)ra>TxRzY;=C%oYJb2Q*&n{Us zV~d%v-=}}na?8St#=bh?4|D9_gl#^))xW~0&zt|e>G-ZkcG{cw$6rkNtgY>(l0Gj_ zT+-*d+ez}Ae;t4J4Zpm2#9zBURvaWc0qRXvON~j8eDz+gm>C* zNHok^J>lQJ93T4E-##$5`vYUMCcHTL^@g*0O_}h@*)R6vf5P_ZiEH_v@TXtu@5Tuq z9Xav)uaEs~!k$pZ^v8d4^rQl>+)2K@OgO@6>BIko)1R&G`D}Hs2^V*@e-qaG<^0aS ze%5yW-#(sk-}lFSFs|7wyE)l=&^NP-M`Ta<%DX1hyz4VQ-0H(mpYhS;MgJ~-|M(aC zzrX(U7YiqpEI4KRKkoi^!ae`I{K?zy&!2GL0Q)!L0l(ab{CYj+*Xv2YUeEdU`n_MT z5?_Bm_O26-j{NKRd+xb+LX%)#*@TlwTGT8grmGj)w# zz6I5u|ETx7Xz{d}Kb|#P4!mUMsZ&|T4tHmrddg&X*6f+nXG}HbF!WJFX3hSe=%a_s zC~l|YH!Bv=+2k7~0sWuFj4{lftU$}^JXf)_LSBcVxkg>7eX;E>WTIB;`m6E!>RQiR zvVJg$hV5`0kZ@>-SyTDmMLt+NxXr4@_l(JGL9w%i~;Yb^^m;yitN zJKq3ipslErO8`OKrdNJD#hJITdRcv?@p_>r*RmQ*u@zIg>gswglg)YHC;IPq1%wDn zJ3+vKzjy;*DUJxnghQ|PX9J`=A=>OZ;L_HJl99@0M`%b!M3+$+4xK*KDI+9kdGBR7 zB;9`-k>F=yu6}wiV}Oej%gb=s<_TqHgrY$6de@T{?xt~6BubOP7`#&*=4WPTV21J% zH~$Q|K|J!K(4Zh5dM-3Lh?@&S;}mZseP~*^Ul5Os3Fj+*y7<(FN9p#&N1gd8R3THt zqb&nD)d7{^8>xu%g)0BWO}foE(HzBm)r4 zHbY$a(9PMxX9?f(D~r!29(m|j-GuKaeD7ZkQrs1O?JYJEH>Y?~-rMY5* z6fvTl&1OWQz*wQ+usOQ&9235mgyu_WDIR{>YDp@5CHzc#{Frd7jl(5$g4eQHw0G#Z zzGS+|NfWVg385nCfGgmZ zl-?HTC*Yox!4}8{2u(@#@??qfT$%^OMTw<%R1g z0hq0wJ)l6qTk*JMuui}yT|G(*LymK#)6F)G6(e6iTsDpQnKLZFhfa5f;W@(Ji6urA z`7g^da+mYhohK6V_}^`S}lgviA-ROo0k7Rpq_(Eq!vm2`< zmharmp6xT5^~3Sw}Grg$k-Z#%8Qt(oDS`~(??5T`+$ z!ubKn8;Me%=uj`okAo6U2VY`SY$ldUij%w)X9Ou4z}Fd!rt_W6Zpv^Y$9aOY2mzw2 z)hkZ`(RmY*%LtbWc!J1k!qozvC-O_e4R(l4<(_s*p?0?UWngripGI*zori#Evgjtz z&pXxZpdX>^%))ttj4`+2*gKCSax#>jqDW_A-YQ@!kv|dM=ntb-s(K&EVyVs?r_gi* z^{r4IC^!|hMQT%>*)pP+_i=|G9<#vn5(BuDI!mZt$~~%Hso0-I1aCp>ozMRM~!cYNgh#W&WicZ`4#?Un-185e`JEYdB()=3JZ#N=; zngV{Oe%_-?`zazp3OIGy371m{Ckc3u$d3qfaQJ|=BwT5@?awb2{xfxO@lIVX2r$ZX zA?ZsA^A68>@@>z#QpNj*iJuK$6MB#MaO33d_=9*F<6@jy7~>*E=n|c`Dbj?@|1c+z zY^&o;ia8VUCRaMn2~z1qdNcjh#GRXmfqu|QHoMXN9-=Ebb~n$oosys)YH|_pRu$%RB-ynY0k^Ei4JE~M3a;85A^cmrA0=5!qBYYzO6@~DGp>C4^ zU>yBacbD{QG92f~%wo{%n)qw9>}^4)Dg8P9Y}LjKsz=H-~6MaRF1zzXlOd`pX5(9BE;dB(ODx#_Mza2`@+b$#TNm1hV z{$Qr%dZNut99+M&%#5%&ZC{L6EMCs~*(sn?rSFZ`7-tl-6ElKuDL8zynV3B%cAAhU ztL^(cc{H@V$y~L4rxzKYoPADSp5n-iM+m*utGWVCc?G1BU422=#}Hw&f$Vf40|0JG z<7}r_8TPQBa93~@9dIxVCm41jYv2-rSn3nELl{jslFX1dL5_R;hBBa7e$+oaDQ@|B zxFDg=$0EgtjvhV*22`)pD^vIl9XBPJ230`hKVFlvx6`KN+oSg&L{m+fXLZS z)OHD?8wq!YxaDk|6gel!hABsWNAC-Zcrk23!BDO=}Mqu)fK%x7Tf zJpB+Xw}Ey3{fL+89P>yQn}^#}<`JcwXr}Yq#W^R&>b04q4@-qzj?S(!=h8XkIwIIn zy0vPjlvEfNjJ>b*Z&d z{m7H5)XY`n^w>VMDiwk+`3#fJf^wq}p942W_NBknL zMOOTmva4&&@xHK8Y&tq3yRyn0Zxse~4B=?8Hi=Qnj6{Enk>w~)Ad(*CB_*o{enj?J z$B)I{4xUUoaSc`a<%1F;!h8Z@yny$JOeIhQ@v0?Z9SO4}X1sDBDCSWXm7(M~l<291 zQ`Cmc;Te{*Nq(}Do6zOnzC&a?z0>YHMAc2%cSv6YVh#*xQc?d%F{1Z>u33k_P9cc^eF2Z-YU7FV9P$Wm%+csuGnpyO{-ClUAT_nDb4Q ze9T;<_{Zo1XMveI#7EHaqK!_CnM%7jR4#fgVO59+qxyhgwh`SxSQk3iOlLs0h^Ik3 z9;z}YGB(*P(&aWhk#YFQoR!@!^+!meFUM0$=gMX`V-F>kNFwJ-M`PP~Mi))(Vzfyt z-O&&@KF^aXMtjGGBGq`6iBz#45l#!0n+dYIT7Y`Ol2E-#`eMee1F;f!q?$WdnT!af zM|Q(QQR>I>H-`y`hWfzE@l$I5g_AQ9Lp>s?R^$OILdIn)OapOGiSz>epB+nA5q4bj3 zJ>*$n$TPlJu-7#vS6xoiIf#4{*gFO6!xFK2XQ*l2z;kbxNi|KIwONV;3VOda?H2w0>*y% zU2o8nC19lBd*vCEsxlnyZkSydcJvHNc9M$yLNeRf1r+1O#*oL0ZR!%l>f!Gs1-1~b zkBe*6i0kEWb&`tRr2chl)*vO&dbD+e5XA$Pc)UfB*W^6zwPWmjhHl%AG0a2l7!E<^ zFSDV$*FxRBX6^1Z>&T#4CwKRn74By8$e>yCx_j-}-D}Tdy!PzwwP!jq`CFG`i3eMk ziU(VliU(VlHjTe^StY!`b*Xr;b$J@`VC(WtHP#3a!PeziMp?Uc3F<5KY`zHFt;?|{ z^T5{S*a#Ho#jxGG9P8~}hVAC%Sa0((Yo!W^5B1e3#d4|A-yhZ(k;m}9+7%&^_X9P903hV3?HmA8!CNBFw2JpPGi}`8%#=RqkYRdc=+~5za!goVlC=v!v?i=R8G2Cq?EZ9Eoig-l zYG!+<481FTMt>Hzz0&G4Iy-E;rPXKj-mvYLR-e(zuC#X!M(+CxBbBDF23eP1hp)fOfh**Zt z>W8-SBN>B*S0{XS%{VZ5kufsSlk>&@$)xsmBmFZM7$?WdX4SgdN!SLLpr+Qz6>wT= zCNo1P&&Fk#<`!p&1!Rk{vz>gK#m&^-tuxdD2`j2|giR*%OUZVISwI%Y%5=DZsv;}z zHB=(>7}AWkl?V$N8LUbaIE4&SkdJ zlc$-X{3e+9nbQ18ze>c1{)QI!eJmrf=#Q}{XI zimU}fsU)Gc6SiatBWb~;q07`Zln|R5Dpg~g25tzU7r2zGL#1kV;cg*Zt90*#7Wf6E zeFyFz!li~5^!50qk7I3Bki~>Aw1Boy5$R_WYA4Z;@eY|ovKE=pPwBPVWJo`sa2^Ax z%ZcV+e@=SiRD_l=_@-Y*3%H1#evN>*2vIN;ll}ssRRAs4iOii+uyFUHX7_&H4@AP1 zkCQz0&tk9&Dyr_pgNopl=LVpY2%1g1w-cF9_{ji(dr5K+hT(41&!eKZLaQPCRKQC_ zP-&;DDQ)n2O8dZ;_Ud&^*HYdu26?|lvat(^=cydtM}aHR6I1smwHeDVOO(|J$8-H= z@b#?TkT*k*0p33itGH5;lZjlU2+k|IterunWl$*wXe6~FV1SA;;!0i@FlY!EG+73c zwgP&ymG+ZsX}r{jmHPs6B3_0ofJ_;q2?eA4^v9?% zC+KIoa?T>*KEhrBKfxYxl$l)xpxVq~GEk(G!XEfk3JXoyg2K{1l#Zx<<_U!H0#I0H zDd8+%VVibJVfr;C2Pq5{%K{#_Sq;c9ig_Lb&JzDzylkI$O0bN zM5NsU9(dj|Ag{@O0S1c#2G~>P)k?l4V6ZJgn@M%;HjHJx!m{RoEPY{owfvQA0zW40o&gJJR3(Bx?N<& zeh7l1D5@O%19U)HCo z>j5uq|Yt)83$2py?f9FTeSW6y<*QJ1r8R$;q9FASN>ik>_m3*|4 zA7}CB`^hgjIQc6+`nm&XRr6D!VfQeViQY#ko1lA~o*wR(u;pNZolWp^1KpbM;FP?K1oxUJ|$zH1=+LL0}~w(aiR)*0MG z*nra=8yJWu?=q2~4a$M}L~!_BG!zXyibxHil3`-tc+w9Rmhqy?Dnhfc)4(1n>~t`9 z6V_9pq`UhAUplEkn%QdWa<{MkF7Nv2S8ZMH{?<>uKS0~Yy0^irdAy5{ey!2AO0$p7 z3(%V^dZUjXAE39WvF?2m^cQMllp%%1&kE=}Z8qOf;tc_Mk41m(_2Kvpv|b7)5hf-* z|8kTZ#waDBJddG#v+DTTJ{7nzA++ejeL8nCJ^Fr|t;k1z=%cq4*=*1Ia&Fw_(Yw+O zZMJQ5EqxdCr^WVDpPn1gD=i%z+Is#v!>3OU=#8TPx?e!2k)?;O6#<=D4|K$mgYJ59 zfX3+OI44>f?QChk3TQ2&jj^=ZKD&nl+6LL-ugirFP*h2-2s3=-bD3f2vXER+Ruggz z=n3%|!V}nJ=rYg)7$CJL-DQUWv>3lcqQWZX``1ff-sh zAk_&mFn1p-8rB=1LO5Okwi;hfsP;$cNB3(e)h|QoXK65f6tOf)$8RRwD1cEqju(yJ z2?Gsx@mC4Ypw`DyQLBd1#|jjo!giEKp^tvxDW==2z81DySM%6hI522++jodzjgny< zE0sQGb%*TIU*wwRi+I5m*p*(!?nB5Jz1rNFiv6GN*lb5j<&UlHpz`gFo*L$$hE1?K zT{S<0=4GxC|HhyL>GCmTWSigl9%}cqnE;~2Yjfw}S2UoKT-Z$4gn$MjSVPx| zCjzjX*jx+;QgnYz)Fvnh-*tg*?PxBsd7pHvU*6-Cfo`+3YLsrf))JQZy6wuev1>5& zcWnW<7Tx}aTvoTgq4v-MYfym8*6~Dy8i3nalZC0LpwujsAnIooBF?jm@wYH z{ZXXJjv`Nx&vx}E5`Ge#l132>BjhKc*F>LITlW8o0+?;~yjBH#4f7;H?}mSpcC?P_ z$r2LZM@e#lGG7tT42USz60hk%tOh5a`XuIHiGkrO=lKC~g-?9BLr#p?%Xyto+}nY8 zb3k0@5qru71_Esih!6Y3!5xTC2E^xm;)D*wSHM+B+8sXe%nrm|0r3N$s3|~D=6)#w z3wjDeum-5*Ih=k&)nq+A?{!_Shl4>wmO?l4Pdi< z>eGWk@Qr3m-|y3(@U@kF+d*ufpaWU9n*+A{mF~VU_$uV?g(nlpV}} zJZYa;>1FF3%zt)R-9I6fX(sT5ucY211GZN8PizX<`bz43>cP@J@3UPPWV223#I}Gf zT|%|#eYsEHW+xa=>y6o74!l>!puYeq>$W9{p)a_xvf%>NSa&5CEJxl{hfW8BK^y_maz{9kQM!mk) z)ctt@Q`WIJzGS>9t6k$3*|dJs^$qO*^*W&+@ms8KW5Bu5qG=k-`F6i*hs;ZkH ztbbnLXVL3C-u>pmyB7`3wm^c*;G_KOXs+$^CaaW3Hu;>E`%-RCwVXD44cqT3(1FQ3 zvcpd+l~~6gw`u7zGJ<}O*|d6>JzCx8rw!)++p~p!Dk$zlpT!O<_*3*BvFINGow>eO zUaE_v`r#um)+SUg+-~GAW!r&(J`JV6N~;u&_h?)QL5F~T>qf|dv?@{eanve zFZ5Y?cGSAU(_DY%{j5WB1cVtt5C4Ry=KwofC@?szxW zkv7h*c=scSPga`LP??N3=_b-X6V4#B7_+VepjevUB< zb8Y=tMGZVXZ4n#>TMlWe-e4F<1J z`Eq82HZ&jT_Q=5Z^C@Jp6QYcP@5{hYANTU*oY9jxwBmT?pTe0pI{;e2V z*xrX5_%Ei#DL-xjyR7d5(=t@`=(N<_P7T~;op_6Zxg$+nX<7I9PK(SYMb0xlI&oI* zsa_~xdMD1RJ%yZ+>3JlqeOf}C)`@$QPj^=gWXh$jR%&Z*;cKwD@LXClcEM#t^+bdF zCS_lS4TlGkJi7V{6*;z3m1(wR+rqADFYGE?m`Ye_%OXpxe5LAinoebyl%>mQ8H8k| zgps5ivz#JA55lg|>bgg#QgfbFUZ(}-JlE4AYKYxdp`O~e-c-<>p$C_7V|eS?}M?2an(ZI zM-T>*IAw}6V~k1wUPh>zxnLRR5=+v>HGECWMMG-nh<*avvirs%RlMihLu8F)oz;Cv zxj9XmhDFw|2sh0l*}Kk6<)VbF#|fK;EH^W0QXzU^%RSO@ZCyS^*Y{CSce%DySAI`7 zlB4%QjFv7E7332w}0S?8Kt4sfV35TkVSa=K`EB_2*>9 zK+*kskf_Iz0$2z4AYG3cQ2=z^qmfWe{W`CZxX9gkW4qk&pChk9Wwya{jdL?B2K*LSl|TzzsKZIiM;TV`MnK76uP z)XSHp%CUX2hB3$NCG1iM8WGohee;SZL6+k0D*J`2WaGP(bsIlKMr4xqxx#`aMH5%=ev1F&EuE2J(1cItXDyDH?AjTnxa>zmGdpK#{NKtHwdrOf<3PT`-w`t*xFA|3P`E(VS|0dZsf(3 z@&e)VRyc&JRm3_F;c~it?AP#lpRgO6>K<}6H4mz1RK zg!l?`c_ep2pG(c)>zrwcX?c{P@F{`n{s!> zf&>_AuROorz4zl-NJYKx#tDT!_v_tjG>HnRcOOh7^uFqu$`@>a=@(wTfi0qtq9&6d zvc7wsyn$rn3~jG1l5O>~lI>FI;tJ_vwREut-V+JqpjlnqdLyK{DjHq%`U&9->0%cM ztBY*1T3z&ApOtrcWWxw%iW4^E^&w&oRm)?|Zgzo+zW+%T{aC;iL+P~?uGuGxmm>2m zdtVXRYcYWAwPY;XLKVZ`1&X+keCMNx>TWIr4z0)sFCmu;vsdmD(tEp*Dr zq=3uM;L5Xkfw1DxA zvG@FdO9xqct46(Tjb5fkk=Ft)x5K4HT&Pjx=?+{rb>!kVW1kYl2tWQi3@UnJdow;B zG~a-Yre%OoT`=s7$d`zF`g)#P=*OpG4yy$h85b4X! zUD;Hp!GMc?bPKp8;BFZVF0ZSmA1%y2VoSQ#nA^}iZoUt{?}EC(I_O@?m(uwL5I2(V z2AbQ<=~`ue=_MBNdWrBD#5E$G1o7H-Vyy3#5MNHOH&?~74QY+@SKg!@yFpER4%BwHpE?$3Nx=`}6V5YlZE@s;A6dWvsz*~3VjgUxl@0d}Zt zuE`E--S&z-xbZ#?8|&UrIACYI#}iN1Jqb^Hwm6k! znr-QSSITvyB0=^~sFenv>`Mt3s8fq=QjTDsNS7LCo-PZ{n`ZQ8@dnRw}L@2)g|(3q%6ZJIW)Lp)-P zz&*zKEhpG|haM5-H$d`NJ?g+#IyXzau}E1;q%mSO%6pzHcn}F^=t$2)M>=~K6!IU@j1l%?JTBsb^6lI zfjd6~pWHt;yZzm1q@YPVFiJglq$Mar&o})HA0N$3LzCvTW0jZ$sI3QfmRFSNO`0>k zNpq$*S7r)o3NyV4^U(GQ^U(GQbEY?8&h#eCW4sA-X8VzMQ-~{yc+WBXMIO?7j^Xdt z`FYPV{9PxDz2_L5Na`^aH9K$K_@}Fa{0{YJ}%#?usa||O= zRff}FW&9;akJ!j{o}=0MzDSOq!a=n@Y;u-1=(eQ?e)cWCGxbp|giMvvf3_d=)wG;`SV$d@jWAe~%+~-m)q7NscP8X?Oo#a(UQKvp~G` z63I*zwv~2;(~q$`lxef*th<>i^ixj%cHuwyK$s7l0S6N9!Ti%!I|FU;5*2d|&9tIM zq=e8!vc!JroX=<$nknATw;qQsH#yDT>FIQzek0hLYNR+RQjc-=8QPf*p+>1)n3D<9 zIOdwb0YoqAMPwtp0?P*>)WzXsGC@y;bGlRQan6~G;C7geEbPqT&zdyh;jw^lX6IVt z)CwccR}wC!>RI&4oDCo&H#$ophUI{=N}+=6bEkkG5!p=GlwwnXSq!v?@Mm_}0daph zyXdBo`<;5R_=j>0f&K~MF9NWtKKlt@2%us5qPV{5sFC~0kTnW7kcHFhn@4bsp79gY zL_Q9fAB}Deq}4=OeEZHKoaXcGdltZd1$>dLZ^c0@uLN_kL5DfG7pudZ^I@|*!oA@u zO$SYm$4eUa7BeVsDgs5%OjOlK6e2ZxCx@Aus9n~eM_A1{GOj+(7)pwkDn7_LM)5ku zvm86OsOmyX7GcY#s>@G_$ylpIzuh^Ov}S4-+T&OT5;IlUN1fw@t?CDxlrB6(JsDdj zE4!4Fm8eSV>u8RWKJU!(Xp4&3ijvc+o1COWIwmQxr_Ra?GB|)Q1H#Vd*`&~Ke0C4SlKjH4q zybytp$)Fyf*9gDw%%cQ)rKU~u8JNFxZFb!Vh`T86;|Lq;=@Fx*NKNv2S=S}ENU@M5 zIUPwDyCrc$pkppt)*a41ooig=yu*<`=w$Yj2rR;pwNfeSwMNn1rq zj!*d2MBswYt-!4I+)-4k$K8~c`qTCfo%(>c?XHhI(4q5Hi_{tF&|PFAbm)Ey2@BMr zKK~Uu zJU==Ik4UM7$~c51oe4VjE=SUz7VfWYA~}-&c#QroCARyJ{>5O%NwxX2rCMpdN&P%f zFmt4|Mb6t9(i=smW%7nDZd{B&tTUsEF~u-bqw4C;+f#|36UJJ0TRn%-_->msOyAQs zjhPw`(=hL=YfVMwvCLI%D^=Uxu7bn1TmHq1 zFDY&0p%-slfIkI%@7u4D_-{Bn+QlClWvgn(P)V<;WZHDpmo`l@ZKdNmZ-pVhW@DPU zrbQm5F#3o6q=8X~pOhzlLU~U`0dbh4OmG;0jST4Pa~LoTKo9*! z^~x2?VG@`VJPwR8!|O1?=-Us9!`;d>1o8}mcHLxd1OAKfV*#}GfF*=QV)r%~+HG#Y zm7p)BEvXX@!Bf)*(gtQYya+#VeuUQ$*a4eHNd4KZQc$xy)P-ap6VGUh@vc#C;XV~+ z^vnSdhjr({o=-JXQCVhxa?r#kT6B9SV>;c3P?<~#2mFz+O@ONQ4dLGc=u-n3a|Y^> z<7g4J&pZ_f;~qFdz>kT{BWM80EF;NGVavg+Bitx#9oXfRI z1Ivu3f&QDYSilw{t%S`8m4O`5^}@aZ<}ZYI;F<9i*iBq#i}E!$4;ZfpjRwe!5SdO; zU(bvaIg@aPp%NLpk+Kmfrzz8WpsNYx0zM@|8x8suRN4~@&Udb&39QG+>~B5BRl_OM zdW@?IMe#hwRa1zc6D`vQ@>M9Dj*a)Y8pRRAdt8kO@)e~gX z_XV>2msy7u4_hJ5w*^yT%0i{Ha zBNSmJu@h;gF3W&UBuo)-5s{w|&Jb`Vk)?zhOs31vNk2!}>%goaTq2-_$S(-15rmFA za=tpQDwX;toS-T-yp>AT$*pxM@g*F-6H@e}pW=7AL;?w)fy+~Ygs%X+Bs=~{g-AlO z4BjC?5`Ie9D*y?llfmEm5=Ore7726Ec#fWXmxMW^2t!Fn`Eqz8J|`ZF|8z8D?T*Od z<|oO;4yX-F*O@N#Y<{LTd+4FwqC$O~k+-0MNke^VBXU;z<5FyMI zbbgwRF$JA(twfeHAD#H6Q)i?+PdsBVsaI_akes<6qL zNl_w8=tf8!L(sCKTPGr?5l$6A>n0Wu5K!EoYIaBK0V!C}na(JM`|apadx42d2u%XE z62V3iSHqy|dqi*yiJJt_mI;Q_#3tF#s5H3sknIn@FcVrTJ z<&TGgAAI=A966g5v)e24XA0R#iMBHP0c0zxOfK1TiPMy-%#j-qIF|e4nSf_Yb}Euf zF}Xwhisa%Ja|=_HNEM(qxwwSfi2|ri?pXwk2zD~Ik!tkV(LWvs^&vRO*m8eHXcj=a z+#3nkirvw0kg?@HM`*Rirdpy9nlf#fGZU89*t&ixW1EROJY$>5Fom(<4rfyr&s)y! zqnKthsqZ6RH&=Ey%_Fmtnr3+^%d{>wVUi-&I(=B5Xg4Ti*C*ZIk`@FoWs@*8qy zfB}&>3`kH^l4O-ENE!i=%!q;mih!bmN{$A0b=QEdyJ8NjF6$aFF0SjEcQLI2!~gTG zu72+gxaapj|8xFx&Yf3vtLoORo2t6HyShbthd2!4KFqzPi{Z`x4B|OYq|btMGa9!G z8pCMwcWEqJ615iNfT0~2@v!!;P774hqRKYMuJeb{yfi4@y_{zsJD2mB6wCh0`A{4dEp13X68xd^uk;T{0L(_RXc>4GMlp5vEo zab=I#vOcjg7C`myRvGx@3@^tk3tCyrcq=p~AL>I9fBsL&^S#iVVx_`7Z&%?fB3;EY zqNSnXLDho($Ou=i^~bn{#s64sP$peA_Ve2@QyO5wk48k@lz)< zH*%|Zt`a*)TnswtE{gHemifv&NCNI!>Ut;XF%oY9uPTH3Tk+fSCuLqI(*RKTPI`~T zTwpRrm2@Qwouto!sqj*wVieSh)mp3DmHCl`C_SM)yksen`KE9*MabQ2bprTInIg`; zVo}tZr8PNZn1o>$R;maQw5IRYMas}+YdpVo7p;|&vF=7#hgYE@MAI5mYpvB*@_8gy z0V_POFCKdyTQ?F z-EhX6`YM!OVMBSjO5OxJ3z)79E0cDsfu+iPK>~%fyGyY!jmXk;8G}#y>U|um<_^Y^ zhMd~dw*93NPwLNHt6xVAZ4jiwVEZM&0$5Pn=GTEu`uXQNK|AC!P<+Z`)eQ#1NAOR`f@So);wB>l9@_P4gPL+CiPusGP&9-Qs(d>03s+ZSg0 zJL1`<1{E{2{XOpD;B4w;Znrzz-|a^CN&ZInw~WzkbpPm)b_e$SXycweu*c*jZge-x zP$mDsp7I=}{f+MO0;Tyk^P{SQMnrU#wJ1 z1Y1`x+?uW0_dU@>4J|hMORX3-O2Ce=cM-2w4@^ebc<||37J*4WV2tW~U}@>X(bG0_ zV^jJCenJ_l_EbeT#(k49uW3BvENVgr6V`(|dU;#}xaZJ0ev zv0`9J2xI-ZClwx{=%-}%vN%;Qnb+M)xB&%;AIviWi6bC?^?^;L#1LMc@&F+R zL%R|lPG9mO@LW3UjzeNpnD>BpA}o3jWed|AmC-CIoIY^|NL{6Bg4HHp$(LpD{jM<& zHH7u?e4m#{Q1TN_9@%CdX8bTuWG%(Eb!L6iZRv>3K6D9t9T;wh1_bE3+)*zb;f3hu zN%L5He(P?gs1wjZnRO(30XkQx#*;*ax`*-jx=+$-v`2?E-Uv<9?pfWxQ)wWeD~Gy7 z!VbfLA!_vo)#`psPCSl<8^RnI*0r5@_?rsXntORr7E zyXyF?imIa(F#Jd|7hWSbnp>TX*$>CsUE$>4$;fZTwxsqB`+=QOyMNq0xhH@}8}O~! zGAP`t#01!0X^nQ9zZrnn^g0Rwdq2O%O6ksC=u3BR0Dlv~>zy5S_yNp!5-Qs4T~9(y zJ=@&a1fsJf9W>f88PFMC(rCvVAPZ5Yo}H-C-ce~Z_2RfjJ2HljMQVjcJ7P&48>y8R zAkt_@`0t2yuq_0;$AiIq6bvR(Xs(hOH>-+vo4~yNtUNI7V;GHWv?wU2%rbfpwZ`kH zqT62N-G$6k$!28I)jNq`XrbF9!h#8>hob}xJR2RPtTAg~Wgh&btYH-C4PYu=EFz+g zgOuq=Vg;}S2hlk&DLaMj3FbyX*IelXNud9Z_rO#7h^lD!R8Iu&G2`*A1q~qPvH;}+UI~A$>+aUM6ut=a&cYqmLZjkiDDM=|6|SG!S^BEiyI-6%@$BcmNaJBmli{E=Q$a}r0%kBIQnUJ#M$o6E!OpR&3u#A^Jkl_ork&bbWV5@SO`>-<>;=X|KMcmSnlcLymVN ze_ed4+S|#nb3Yo>LVrz=%0Wb5k93i#%0Le3FY4dp`gd4DJ4WsEfph68{U6--;HRl! z3QEt{%&~xq-m(Zma)F&yBFsUTE7i=_Txmv9b3}w$XS83FHlzKTv>BbP)=eW5!H#9o zW45aOIBB>U#z505GsxB^c`%DWH%NhQS=JP8D-2?}kvzUWW1ncwv0d_ny%}t~0n7U# zgKg8M5q*Ckf#xvNNBz=6`sbqAhee8`*{6+YW=-M!kv_zlKSkKswt(_6t&V3X6ltzrSVoWUB^HvO7aQl z_AC>&y6tITJF-z^TUe|_+ro~MF{gfLT3C_l3MRrMSPe4`>W-{4CM%O!^dl(R+Z&mA zT6N^O6-DPsD0sA19k~JKIxuU}Czg07FF+jxwL6{tF{5`tv|8!>B=D#73glZL)CXjb z18>T~rc^~c?^S51@s+7f(9IZ@WYZMQr4F9i`3IRKF|7-#@1mn(B3F@U1Rxtiw2WR8 zX~rC;#iU9?bbKc5+`^TuwNqv|no!i$6&;|cMx~eDeP?k(>0np-_~{MDyP|h&k=Yn; z)uoambdw1Sdag|+tNA~YYf{sFr>2fB8&xGXbzpGmrT>;@m|pvB;dh{Bq|DLH=6L@` zm0r0SkvGr#H{*~gb{PothfF`@ocXhQSwM-XJn&gSMRn~yZsyO(TN7}<(V)DvUgopr zH2XXRb~A*SmbxgMYXa_aAutZ%r}e^zV@oPp3O6V|0e7jO;vBVll|&`5$ksqHnt;2& zSeO0kPF!Ng)w%4l%Q_Wnf(fjzp43=#-Z%^P3(YP}(++o02XBwYqVe7j+?%(kqqm1K zTFjeZUd!X-ZhHzG<`*z0^A?qQd$0u6Iim>>&0FN{kpzV}7dR`7zwTk?C)$g(g|XrFbNygA;!JfXNyeGGWCKfG@-`_-VF z{4wD9Jbj<)fEz+L%)DC`im3g5Rg^0Hr7kufprmX)Szh~Viq8iIAVmtmVD#PQ0or`f zS%ckH085mi=Wd694azXK9tfpJsxl0+$55b;G9@G!OOK5;$ZW5Q_Wh?kEcHP6Vof1+ zeoN6Gfd|#&0X2At+&91%u&D;+T`ik`4tIt5Y~5^Aqn-_bnrifCOhk=D|7lBK*dyWY zmj3HhW1u~I*uiS#knEFcdHBG5&>3l8r;&@~5@4Yke*nqa59!|%`u7rS)kc^-Hvrdm zAhR7aOq&<9+YNdr?b6az5mtI4m!5Yi^8|^*zyr#>L_+*L=IHghozJl#8@s!xmteTn z^~c6=bO;URP7}oz^q-57YjFQS7T;<9P3wfps8Z)ERp|t^IzK__H^AKxYXzW!oR96r zZIZUz)6a(W-1?~)nAw1?%+^#aYXc?TvQ#W<1B)C|Sk?w4f#2#YDJ(q zuZ&-NJ3RLo?;B2{vaGikR?+K0!i_zbkn!|F1HE3ukS~YVYGT zvwxUj#7qo`ZvR@SZZnpI)B>pCa5yofI&^ zB^@n=YZ{L~y`j|WXSI5jub$C!Z*3Pvw8O9X)4MA`2)6O(yxT#FZGD;ISNMkG^e4@* zY-%^7ecrKJAZ83h^j?*vL2_`BUUr@vf0vKI z9aCh>zZc=2rx_gK_C6Qr6XyNU^N_694D%}ZnEBu%yV6cI=e1%?I?%5-HqiS@G6g@7 z_!DrL0j9{!3pL!g0o`CLgm76tpa;GSp;0mvm00I1imnD$sRP|7mr<8tw`J>Lkw5y2Ogq;Ntlx1kHe?rkjAF$-DCneJWX^30 zLf)wnl-&w!aU&=@L`ElUQHx?ImNBL>EgS6!FoLqbE29zo4E*RukTyeDM$iW#_epUh zIIBP-NLvvfK_BLUK6&b31tl6mA1L&Z5}8dg01WV~!5&XP@Ly(sZc|v30csuBr-d>Y zU>|0oK7A8y?<{WJKy_{0_WF#b&PX-D_WH~KCaT}HRFm!X*#sax)*={0x#=AkR$)(T zSiQA`^bM=GjTm8;SA8U1XIOpYh}N*mtzk8lx%5M1Hn^aRySuA49(=|TLV%neHSJPeQRSRsNoq#S@YMvzVXW%KuKn;p>$#Ff==RIX0 zSq`c4ddi^Q@`=DGW!jU#yvuJ;=1dZA0WZ?4%;i7b9E<0JFgD24`~-6mH=yV^(#>>( zaLvmcM|}}a-+aeW-%>Iqa6~Pr!%^R%z#wJdsP773p)zpP_byTTmNy^t%R7l&KB$!z26ID5J+>Y8R1v7f_U`-H2CYi-B6ov>`DC7^@6X z_N)WIHf734{2Mq7G~fSQW#DK4BnPzc&44H{w2kd| zH~rFuBEWz%fKJK~0S1f)Myk)aD$Y45 zVFopQGoW_*Z^PI4RiV6{MMLmoPs{bVpdaDvLZA z?i1_N%A$|oC$jLb%)^Ueyl<@2z6r8$7wjUFX_vfq{B@pI7hQ=nWGBr)X5C=a%(bk% z8txeD?jNex(Td)dRfMBKS>05XMSZoPfkt-ts+H#M1rHBnH0@H|z2M|gtTp_5!Ncva zMujGeQCn*n^F#%SJS%-CIk zs4_2-AifOTaWQp_5I+f!H9Km9*r*k0Q4@L5Ho`&)`G1ty8^aW#PZ z!7OqH-z;Wmpl4wz`^8#MeG@c#j4FRnqhAOp2^UPzNA2ka@e8KGpHTH*BBGbyb*VV; zgo0IiOEa4BHP9=hx>P)uc7yeW2nxXb45kO5xl)&k=MDt=DTAq=dj&8?878vx-UMD& z1`(W(TF;LtQ$k`AFxHLyJS-gI=f*D;&tFaPmFn?aHDKiDZvi&Dkq=%Vp3^7!BiHV& z#C853frr#+QuUETU)WLck)MB*sxOF$9XY}rbVQl-*=CGJ{-8!Kk{<(qSL4}`{FO_^ zA^ln0m3N?QJC#-imx@EifF7lGdJayPibJLVlazUa#9UyuGRz`FZU7c3bBw1WW&oIo z<*QD2lzhx;~7$CWiZGKo&o;g)H1*tWcty#L0*9BFZfP$wdRppMnZR_cGz@D zP6JukC&(FIuMsn^Q#>D-Yp-mP^FV8~=`vcCBjb^409_2E2L`L*sDABps-kDVqD&;+ zjLo5b?eWyT4Y-N=Ai+!{_W+=YuyzFr)H>2m%u79QRBbY!s|14@!^&-pR&KQ!>6pI) ze->@_np?452Eker+p<>F^8s0VF&Ul@7`l@OAx`j|*fYV`Ui=3D+(%!sXT&hlqAybow)j`?c?vnb;wX9|51UVIZFGF3Jfh!+J97Fs9xlEj|vQ}=c)Ib=+@-B zuZMsz86{x#a*HX#yuEQGbhng;v4m>`gYHp*Y0&Z?74R8!j|$8bgYkFbM6!!ohH??_ z1NB&lcpL3a`hy@cud*+i`G~zj{Ifv@HI~~NJB1iJEZ*dH5E+eR(~ia#AIjDClsMQn1&i^e(}Al9>OdNU4L(^pF`1-AB*ks-kmQ(R)+06_a#h0nf|b2NgZSIOZhqjbV{;Ut=FL zmML;0Sdr&_;8SLL7 z)eF#>+KmbSW6uHfX@$lwl3ad`V&}a3lOe`DnlilM$Q{ z*D@Q0c2q^De}^nA7d7gc#a?I^A>La~yJvwXdG9i`%gUMDap0IbyVxw|$-zCyx&|ZF zKL0pw!GYQvpjZNqYiSmO+83Z0#JC6fknyTn*cro_oeaaHE7$H>Q@M3U0)6DonbNS_bq$WVM0oB4RWOceOK=lUI z-OPiTYXj5>Q0FzSiq63Z4NX!)67Kctml2My0Om-z*XKA46aVj zTo}?YH|uJKrWeA}g|#%y%?%xPLH!U*X%ON{eqhfgA@|^^ z%8t0QGgX#;#G_0v6RPN_D;jZn(Q#K)p`!BMF^X8(Y%Gws5-^y|8$yd<2DiDaE5Fvj zQl6LFEMfq1n+utFF1$c)vqUB*3{G<40?1r87~Y>$Vm#~oi$pIg6q@DeF(P*i#6^ay zg*j?1TDyVUMbEydD^WcGJcesE!TDJ$@t2hon%@DM=ioq1BuUn37rvkjdbsdy;58(k zrTgnm2m%*=4d6&(Y=>2@B z_lscWqEvdw_yV2O2#1C_!@LMxUX*Vp&8GdZCVu;iE(A9m90Z27b+~ol));OUhlV*% zxQKCIbgSX^bZA&NSL-=YPnfx&?01@A=AsXP_loAy&_4UClPWUnn0e$caF%t^&s6!Z z@TF!RO-D;nG`$%6y11d4Kap#@(WEVI0=V%-v>sg|j9Lvc6TQ&5cD~_WynqT7GRpA* z<^Y&IviS?^J0dTGc|lD_7-(PYoLz58Opw6%L&{mhLmjK46n`~J)@GUH<0tTFu91Cg2z)9w9;di{| zmh}I~NgCH9k4sF%mL`T})Qy`uOe-*rAT#eEX4FB6M_4kyXXlpwA?Zx?zT@|9(Tlt% zC#mb4Wa1E7pN$o2w&C?9W}B>5p$3#j7U2b&Z6;m@=!=k=Z6>Y&mMQZhi5q}*=qsxg z{-N1s;={mUWpI*-F91(70j_RaR(3lVdouBGPtoN=MF~*VkW+1~sMhe%;4eBNkZsTr4Ar6vVRRT0ri z`p#Hwi%d`yp{PvYal*8N^Ozfn_H()rEH0S>EDT*iqYlP))OD z^pW|&XKemKe%WVHiR%n8|-2b3fg|*+1=&u#>Dz z^L|xy3JjWAb~t;n0QR*%z$z3WCVvF{6M?1u-RN~=i~AAW4}=BtFK16vs68c#sAud4 zQWfB+pN3s1%0MrFmVNTD5^rM=tEI%uv&>(7 zdJAzZe*(^4?L@wiCu->VxcN&t_2)^jht|e0 zp1J}2dNYH~>}QH;ifpHD1@w(y^9;l-<^Y&oYW#xXDX!<7bf>-w`Z=m7LKK?4RnY}y zXx=VDa_6b$e*8lVylDr4olIONSd&g;+|!<8W7h1K+*Rj0;LAd&UCIsdx`xNI?iQ2d zJnxd4K%Rzk$gzD%YcNe!`9Vhduc8Xf=RUID@}dgt=Dwy%YO*%>TP(HD0{Uu7H}|(v zDcLc=C^Mxa2Pp=%==+lC2D5rFz^nsP<#k4PTTpxptI_L?xwnm!wx7byzztH`_0rRC z$uSRJvWq@Y)Pf<-YZr`{YcjIl@>+jA=T?CWX1SuV)uR_tWWh2ORc~RVV0Z@$Cv1%YnsROiKFQ?5qq}hd}aDTF#VjbOrJ;w z(^2<0I(#cUwKwc)PwooOZ)*hksaiIg8=Q|!XX2f{PK!%(gR`^gPlI}#RgAg88U6Hk z!Mvt$bwdGkFryg+GEPmwIWtb0>l9;p>cLezMWRT$W(<{Fffe-&d`7AR-gmOzB%_9Da1a;kuG3i?YqS=X91e(zTl@w-_A zE$HtkgF-ocmud~l$$yuISF~};xdO3Rk(#{N&J7cfY&WtC>Xd^VcsngDk&%u1`v~wvXm%^O-M{6)H?QAyFtD=)egFh~& z3wJgL+1#k&)g&|8o0&>lAD-oOc4-GFwYQ$GlS-wJOM3v_=pkdinUCh?2KWKs&oYY% z$`t|X0#N6-m`1G%t6#OMGV{^yP1OG?&P`(&7BM?`)$vHRD1vp4PbS@=AW~N~mU; z>~eff@40qb7?_tNF6 zY*QsQ6}F6*+IVuC4p$drSKPzz*V2Bir7Wn;xl%K3Jd@3wnTN#K`A+dO zX+860#~96EEW4Z0smwM1BC|T;7`x&H$Jm25xF#}o#qnQZ42lXzSA1;^{wy<;{%Wi+?~XK0s-hQHu$KC##8$hD zjbF{2lxE#coBcM*HFGvU>wQq~NtDf;J}eED(QyX)IBL-xbD3w(2+A&_Y_8_vt^sB#n8hAfmlLr$uLmvmi&~$c9C^*Y zN{rg+yRUOc?K#c9o&{%BbiyKTm_WDa1uP-7N|z2vn>Mbj5z;5QLiQ7Js&${MbwahS zS|&Tbnz|!RSJmk%N)uVb*FG1}vftHNomdB}XOTr6m9J4xp{s1U>xY}4jDcuTTVG4? z+V-}y$GfHuLd}|eqAEIeI@^ayQfQq^YH|3_398k(grR_$y``46_f9kiMJKZ#0Dco? z;9s0T0irtlE#P&bhMnl`P+x)iJUq3;`_h?)-fEtcS(is0zp@xJEj+cO_jOQuIi;7? zo!ZI!#xETko>=028F&r-uI5e+~>eNQ+I9$@1#%vg1U1%dO!K;abaFk{w1jSH){UXZYA1#0Lzh4M%uBe z=mk~yf}O|f-YXd$b26HXlg@q1%IKJr(cEuAeG!$6jyV}sU^f-HLJd3SWK_YTvZ8r- ze#sf$o38VUKA?Jr=W~ndtpLR&R56uDDmr^_2dGa$eGs1CsnmPNp)T9Q-HO}7Q=s*( zL(RL%^X6>|Pl49o1JoZuJrte-t>X?gAHL@2hNnR5y#Tcd)OyMPeTP}lhD(hWVWj=O zv-1Ub^@2Af^Y>f&GGE9_bm1=P=>0Q&%%TsQ)Finhm}Lb~;|?XFucc7eByNO0N$PZdh3d0ahb z6>#;Vo);bDw@MaX@3^B2irFvw(kB~&Tuie?_@bq|=*X|orIjM}qE~FBzWm8vw0bSs z57dkP0>3XR^BReFfj5xS1bmmoTs;-FUEF-;S%ll+0WJE3`f_ z3!}|kvJcT_9NGyE(XPnh*DAF^U6i`B9$A#SWHWF*O5NGiA?^op2r)L?Z5yG~oQ$0< zxPPCE@K;3L47vhi)6`!7d!kvw<_bY>McJ?DMQz7JHBGleDp zBJ&St3VMxx$@eHD`zB`!`dq})=FSwB_9KJr#n>!69z~wtuC0m=N5H0sOrha!OyOE* z3QMn~!E8*yp2LJGEW@&v730tmx@|Z#tO;~|17EAlkjJt%VNI6T7a^LBWcvf<9#$~? zSukbciQdgY?TaWH9iHUf5+H5^aZ7lzw)Zh*j}x z7azr4!_&Q)YCDd06M->KGfQ?_&D=SM!st?`ndNJ#agEc=@|($QcA8m!Cz;*S%$-g% z%U=d_)M;kblB#U z6o)VICc24u<#`~^#!@C0yBaemrF(r9G+rrLC)^H12bbPgT?h8YiAU5&*9bZcXEd>v8`@HVs;09H(kahr4P&@${3uZ#Aw?g{LEH8vj&hj%QD-8Cs83B!;T|s>7lwp=vT$6Th2GgML|F9)J1uhE^1T1FP@PXoxoptW zLOL#APYbO;!+B_K`TDF>L=bA;z@<GnrbwCTq3>*_o~I$KV*n=C z=du8VNk0RuF7r%HMCpws(Vocb4bs+@(UYh=D-=2@PG>+4PJfJX%A*%Q5!%fhahg;T z`Zz0xvlzEgAvy-|xBNe}ME;9N*P4}{L=7ja1m%1?U?3W+yMa}sn z^UggYa?42O)_W{S=KS|@kxCbK;S&KiyZw{G{-dFafG_0{!yD+zM)v|kA)D_I%NO~1 zhi%&81?mi747s0}7YNs8r}U7w)VzS1n7cH(?9t|hzGVK9>NRchBGn~~yng1zc=bBw z#dfNZq~t9xmXREw z#Sx zvhtJjk0PVcrx-)_e_tjrgii8!OmUH{Bm0Y&5tT!HEyT*dDRk&4mg1rlLfrcb2=jtP zd}&fx;o~kv3r>e8lRhhRiB_4+7)_n8)f0uXfLF?171nHRjLMGWP8CXDeI7&OY-; z8jr1tt{%;Asb+$tlAdfh8GYal`}VHB3RvET8q?Vw`FjqD3M@P+mkZ}gs)))aa~CP7 ztU3x~_L?CjsV=)ihZI!CLf{_Y_PT6o{6-k8Z8avp+N;a{z;Ezei=o5Sg}6{nUYB0= zBgmXLj27`pk2AoUEhuGYV|7#+>}fT@Z*?oEtoBJCiBl@{ za@(5+8Ay#00SQFDW^_PeI+;l#p~rMBiApdl;v`OpL~nCgByI^vAo4YO*UJuME14Tb zqIW=o8Gg-g*@0oZ`A^thJ*;^(1LL@ghoJH?4GZ6=3Q}M5urhCuK)h=t-on3-cn5er zlZ@WbF8r3<_rO;kCcWyoGwD_TbSBNwBOh^O7k-&!7FI=9R^UMsK|UQgnHM{6t_VxV&~|ksMRFL%=R@IiHzDtsIU%u6b9uj%Kmi5{|1d z|EgSYxy{TH8?DHn33S4lv;Sb03io<=Q{_tXxnG;5qEG&~Hkb1*GrF=DUsZTO)i9Kn zxd8qbfXr!SRuD#0WG$Ihl!C0%YtWgfrfMTl<@I*;k5a#$F)OK`+AOGlA64}9v8{iE z%p)?2jCobj%EMI5>1=L7qp2IIlCg~9zX5+$<{A?J0FEnj1Bowy|0uJK#E-xa=sKq} zipeKZFglZCNTw=;FZy?1EbK_0V`)Q^i*SNW{SUaDPIKWrMQef zB(4Tlsh=?j43GRkhz&=JmdrNewHLe$#r@sH|G%aBae~84%RhRo0qDh zSL*eIoWrIBeb^InBKX0&u0WS4eEvr@)cCOBU?e)trlcGwQ-*?dBYyWz?n9d;EQsX%sXS8SkvF~j<(c@6He3v?x1A1{!Co8X84XpHB|EsD%=R9FL zr}1n1-%asOWz_#~fV;`q{vRTv_vmw;2>O2#%;WU$K`nAQqqz@x8%D%`{8d+fSJCy03vR zHA_|GP%g|bz)xaBs$FqZJvKIztWZ&_HWsecOT|*_wM~F}yfaCiqlOPIm|{}TnZa~( zc^+Qfg^!8i#V>fm|G^NPBm464=BcJT<{Z&>UiJ{v6J?nPND~=)X<50`t~wu}QSKyz z6+~X5QSKS0Ib4NKQZZ+RX-UPg1SWo3=}zlO@!wV{#c8EO;4hRO4ln z+rXu>j+@+4rF}N?RqrmPTbP0Z(vMj-(nW>W1%%m2?RlcI#iT0KJw`g~Ta!9KX|;UU zq}!IH^O~B7Po)tAKKOyAk7B&JPyV3(mCSni2PUPACrM`SIawt6lgvpz*;+~Dn$!I! zD;kp7Lw>?ciMJ^#|=3tuk++ zWxiJ`o5|o2i?`_1Bxmqa;#(b*^2ldby-D$n8B;iJ9|9R|GI+e=O-4#FaUgdL4x$y- zdW?*@?{+G`+<1=no7!Xt@!>2=HgoCLNXxPX_WFKGaz^CPk#~ui@Othc*ZeTQb}u7K zFREy%wf-|udHoJytXXZ~QLBq<%YiawkjJ&dfU}cSDFmh?xh23N$kO`B|IsTW&%4^} z$guoD`XCZYpRt3hNl#C0N8aDjQpqs)Xm`OODSHaQE=-0rKDY}pc!OGo=S}4%(;i81 z5Etua53_*gIceY~b4!f=lNnUI#frlU><>3m4}q=O!fy)+EISE--53|xn!SvsXG~z( zx75h}&rc70v)oE7S8pxZ_DA@&63f;UEQzi88dUE2Hz2WWy?$>cwi=OK6-ewRGV768 z?)ibl{s`s|R$^;j_+Q;hWL?eTU`W|dSdlHZajB0$5Elm*F(wNAW+NyT=zq^d8 z4F4M3)19Bas>2-bGG9R6wJ*WX(=0i1nZ>Uufy>+}mo-0La$2KYjmt9Zf~U%gIe z$mLE^t*zv+%Ik`i*+XJ7Fh&_H>^jDM-L|@o^H!)+I9`umuGb6D;&?sgyncu>he*Kw`f-lqyluk5@%r_^T4msP{awHTu@`Wh z_dDg{93w`V#YVmDpZC{_Cj7`G)}!Eg`VOZ(xnj&>(NjFf^C<7uU}Qey{(<^|OJB8+1TLo~%#(Y=Dgox_VJR8?FXc@w&Zq zL!~)pSYvvr*@EWM#|FZ~2Dxb79Ezn6=^;DIlm~jq?_%}9Nwa|-$|2y5RmAFNB_2i( zXIYKd(H!XIkLFr_{1BagIT5E<9kPCiz7|c?TpNox1u4lHd8H#4Sz%;P?CF}9@M{B3 z{-=ylmUZnEP`SVF*_5$K7_zPncL8?98{P-rvjOMY9{;0Ph0r@p z>mc3~%(eq=ZBL1F~g?&S9sSzf{0h^@Je^JWA#SGY>0C^q-m=5-ly z;daJ$0?^vN5Jl=loQ-K3{w=JA-Ur*U!dRXsdO)?FXT5e2;$e{^k1F#P3G!9;ly9(|{?=hrZS}mC;k) z7tM-kEsbp=ByXCl4EDF_N?@@_eL-t!Y!hs6IzSWDlKZoNa={>V{3ojx9qZ!afev*+ z@YrE3D@NI2t$g9sHaO&bY#6$s3?6#78^&WZc%4Q&#^89c-fkHWLZNocxMnIe{4Ha< zrCBpY56O5vMb7OTTXH=`&Tbx$WxDmxJ0Gj2cK3Lh2l?!!T%d0ZWLtA>D^S<5DB))} z1N;-9a~!kp_PiV5aMM@H940dm=+&RhE&ePFt^7}`qHCu^%RYKts}Z#Ff3&m;@~FKG zkjnI>@EZuWH{7cX!nxsR;7eujr5nqDQf0c4SOqLprXPtXfy2t2$DL1fapMolj3&bh zakEaaD40ZI7BHz3nH1}-13WKlU{&-w{MO`{+hHnejAbf|K(`s&+`O5VS&K;=171*O zH7}sDa=a-SGFel!E+f2WZ9vn1Q!(2Nx>&+zx@GyJ{$LK>H5_vpu6to`8HWkSp<=33ApN`7&6-@p;_WOaM zOKhDaaX%Dv-jn{3UOq9eGw3g{y6d};UlIPt_@V)7m2od%&VFrM~gl=v^EJ-GO6?|pP z0@NKoUK>PI{|abP@D*jrZK!xJa0m?~cOfn%mr?Yyiu!=YNpC))%sC{86gR)poM?Z4 zRdkQO08rQnpDujb^KQXfZb^n{VIwLR9wmpM?5DTwv!gX<(L&B%c#gTe1e zQ5B=+@n%8>NNrOkkD*RMYxfNz(vkPggeEF1;EH>aReJ&Io@~|Ky?_O@-tnM#hTc|~ zrZi$`3Dx~(q#s4~&1ERx72&zc(dn2PGTPDu=qkgpi*}l984jG2PLqwYQ@)iPF6-OG zxc5*c{en)0dXoOGDt-@&YK=d%BD;lbDqZ|5a`DAH(0Mj`_O5{3)xZibO5r$a?L;NJ zc2d;FoXyB;%n4DjkScqD-O8*Y@q0isC8s2MTaEyaAidP7T4NHhkJGl5Fhxa$NJ3*I$nBl8;SN!GZvEznvStZ^%o z-PX?1+`EvFHEx{>OtyN{U{fBm)_7EVV%9x1VEd$e%44#|?ou+c+PVOk6PbbFdpiHz zinVQBAF;a$J)O3;J_PQ51i5x;m1fgpEJL6EzrP!u)g&8e+p zNhHXv{{W6Fvx)?vZR>xOL6BR21b$#J66CGez^yfuL6En?>aBI0Aa_?F$lVoC4g`5C z^lxpdx(M>t4nTX~C(Eo9WM{n$o79^=qS1Y_h}3Zr9;51nV~O)y5#_CC>a65a31BF> zk-!jHUS}tgTbF{l9PKE&+%OuJ^YhFZgs{|d(sHF+cL3W^NPZry$dzvW9dI|Z2wi^{ zv)8h?y`t~@AB;USZGeH)zQ%CX6y{U+HL^2b(=PBcpz+*kTn#ksqjZ5)N^{GVe5^UE zIJBsIcgW@4Zt|Fr`1TG&j*46!YERu)z|M|c=?>7{-2BVGr4Bx_{TRR3Ng`=iajLU@ z&=r%o^(Elhq>0}5frLH<@nO;=PuG@KWIu!WAqk^7=>&9Jq37L}nF>wR7^eHQLDoI{ z#D+_m_Ofi*)Kkh3(T+BF{bBGtwv0#fFtO3c8v8WC0~pw$fBG!%n9eLSUMxpa;qYIM zq1(=+`_6FqVSpG6VqnsI?;|mYPH&qCj7yr~eH?I%ncY@N0F+91e1sYcMt_39(+iwN zZ#zzfcjaQEoknl_3e4v$u<^JSgvaq{tzg;Lj4EBL=$hwx-t7@U$IaI4OQHzSAvQH( z@AeizROFUHuI4CmLxA&?fw$W)0!E`OnAzxyDgC-r#H6yFuI%9HWjmvY@B^56HYP?> zA}OX*K--;V5;5hUsO{>?MUwNCR?fOV{lXm2Dk1-j*Gb53y7IeIO2|k{ zs_@=$GQWKdLy$xzob6XjIDPe<*2wxmIJ>B@BM{C#WbO`x!!&mL;}TBaKsaB6IpKt} z4Jz9(0|v1j#ma^vcUS?GC)(Hc3BCX#&q)Acp6{br0Wbo!t#IWMK&aAa{HgnFh43#L z|Mp5%5aG&7-T|SMJO28b0-=NM3+mt*62P`2{4TNrSc~K6*$c(eZK!!$CnR#zNn{&= zWZN+L$zPq-ZJP~dD!gnv?vEqmvQx$(Hi)m54eJYdLAmb*`ARTaHScgOR)JkgHms9S7>|E2J4;7w&dBk>{d0ZPc% zvtKpq)!`qL(M`G=lYfqX&Y&iuZG`P@3fr29w*3NpuSUVLXgiI!6Ao+;-du*eJZ_^- z8>US$ei8kxL8#ueF-CP5mMUJg#Pjxw zYSuug?m{G%SIl+$`>jo47MW`s<*(jlRz;OtEk>A_4zyM$*AuZ z(I73z#n`UMnuEqFeYesS-){<1lZZ z3>Nbb0QbQBbpi9QgLu_3zvZBq-@Fm#v$bDfng0hBv=p$+e@^C8@v%9P`7OGmnSVzD z^WRcWU)B}#x*4@YZ%gLi;h5jawd78zUd(UFvCOw&`uKVY)Aml#*6wmUXgHqntd+2kUt@k0*&Me z;E&2QB5@peTk`n{`4n^|hqvzhUYT;w+l9UDDpUqX+f@d1Q)Uc_3BYJ(3OYFoY6`4! zwNhL=N>J3A+nu)ey&{kZ)>@F;Pm!<$QyBlTh&QJzscb@@taLn!=Mdp0V?Cl#Vk1RvX|hwHwlls(0Sb69w0iR9Zg8L-7f8<>Pj>JY1CKqDzu__uouSnPLWnp z5x0Wb-p#-b_FcM~6c#kHSEc!1S8KJ0I$F6GG^*vd!t8HDnpQdfGJ>zIUBH?ab|=@P z*AM*F!ut_wYbVsbkHX+XR;aCgq3(Sb%rRt`e~Z1ZfSmXK1bmw` z>GEk_JGc7S7xlb-^;iSjdj*JfUnihFVl8oE-G`F)l|!c>s{VjxLg4$mPvF0B` z8M+|a-yUem2t=L8TX+>crwt1(G4~Im_^hM_bG=@!$Nl5MjMd6*E50IsD%_6oJC*38 zr)X+xm5Zf((!?&=%6I?^x(OdOEqp829b_B2gwJpI4~Gv`t>E<8q0p|`BA31S3yWcJ zXfCG+>GWCiZSU03#SEfmz4re3v*z3O?)=%IUfyK3@GnNnHT^CVK8{**mWT~+Y>B!1 z2BBlQ!F_`PdG5YJ>OSnizi%+WgY)2K!O*bU+ZzQ#bAuZNn2)7u%Rl^*5$d z{f+6o;OX-D&HOFuy!l;9J=xgRss3g)X6}_D%_RTRl|LabcOGTPBkj(k3?-3v=TR1= zHrGaVSl3RbO@=B3pRQCo_;jVx!KW*gj(xf^pB^{LyRv*JW7%5prQTL3iMwFhD`(fGn8IKl}tlRm~?2X zi7Kr}qs7jap-h5`7 zDIHbYCroptTd3`u#*V8*__vK6Sy!dsGVtXME%VTxnb~^kqRC#^w4t9+IswJ(2^;$w z@?bPo@OVQ)n|G~}-$#+TH@^Y@%Bg#HTipXmU94}{hkBDNICNPwNKT)py5*`%cnK}V zaXnrXM>3(AI5WG@#?X6cskSnoKU#u>=NpqwNH7yE>Ih;o^|iGhdIVM2f4)9Kk9Ft> zwSA$!j3x96g(l=h`G3rm>gOfXNo`k~B8tCDXxoXr7cRqs!|)N#r^qX}|2%ulV!=C~ zf?@95yOZB3il*V;XU_3Hl>`{^{*}OD4lejy&3{kkR$z;n;r-jE;_bf^#BMWpruT0< zd~z=V&u2lCFy;|=!ncPPdOA0rW9{cno$q}C&JsUP#dp1IY91Efyvf250q&pxnW3l8 zdKLWp9a6Bke;~hhQ!Z&bEWTs?5DNbWK7z%+%ge6yyaTj6kYZ+b@QypC51^9+bqv$V zdyeY^ox$kPs`o5gWCj3d8SNP9F)OimU<5FPpt*mRxWl+c>Rp?%9Gyq zh368VsGIX=$M11|@0LiDzK7pGJ0m)9DP*U>?~4Jy*MPW6+`SZVw;jyQaQ9Nc-P0ic z1S<#RRAS{l%Zfd{z`ouCD+h-7thjgC{l%OZN_yC5=)gvP?c>GC{bA@~eB6%nB*b-) zri@OAgILkQf)pt3gyLID98*H0E7-i6N@(N22@K9&LZ%Fb*lMSZgC(?WfpiO=HrKcT z9%SM?$mu2vysDsw3NW+ivdRtc;3^Q6R{T{tj*Cs8uEmTFmVtm&^2>!6U%b$}p11>- z@L2dJUe(LYEb(r^TxFdH4*_?Wg%iCk0pjQeZ_R4^|bID?}{w?X&av5U4cXK{H{#q{7EDk{yvP? ze@!-RJ|;7XKqqs%V-7IgT;WZTbuba#ffL_x4Kho6SdGF6YUNa9mc|@T5(%X}NsR}A zdzE>S#LK`-%3!H?d;q-Xw6agnM-^`F#<)`pomTEl_q;n(_3B;P`xNe=sO1@CkS5jA zzNXg+ureCFo}o(np6ZahGY@s8>yBSK7GbN1>P)KW)1i0v1$v7@dP9mUpin^ZIu$pe zXa+FR7MDHreOB&o4^|hOZdGl(v3lC+ei83rWeVl#jIBhv*HKC?J&P`)TdA|}`9NXm>RqzfYw!_49GCHe*Y*1k`{MUST`$Ii z#vxXdNAZmr4gmL=vj1J@=7LC1F(sax;3fM0GO-TCXCEs?& zmmch8Hm@BmHlov{v!}wQ&Fl}S2WhZkG*DY}qN~>;*%r67CWV$;oyJ#j2e{`&zINWP zD~a355khv`xF(7-W0u%1u3DH&ZeDv(zGJU_J2%Wg(IvU2X;wbQQ#VVa&%xF@ID29q z>X^?MBN5-IHM~xCK+eWxg{8%eq&#62p2cE^k#@9BKKQmYRdyNd=U}W;w@w(S^sO8 zv^PPhv;&ZhhT`0ZeHM}u+=iIqn__ZoWHnBWJSonSZ6q&F)%a55By6}BTWRqz%aK^m z*uab<@lEmg2*guHp(lwEvI~t8BAMk!h3p{Ap@v$hB-*)XdyvICf2uQ;#JTYo9^$3LX_8=OW)apg=)w+$2`JE? zlR#;^m_&**9BWKQF%CHTBJPwI)XFt-%roMSsre@sjC`9vGjPprao6l_Wol!15H5N+ zlx3tIZr4*Yd#`+xIA8XTiP!~qjkOGwFva-GuTv+r@)*9_bg<0MczuI;t>3Bhnq8Nj z6~hLT8B>i!!DJ)(TCB6Mfz|8Ud2x^`#pf_-@9bb*q|7;1*s+<6ou@>E<{&=9<)$Ua zy+19T=a9%sI!c@;_&Arco;M`1PGXIrR^=|fIhPuis5Y7S@I-C6N+V8P>|BtD&{Po% zs|isu97M^aAi#3U08hd){u+UiM*PU;VgVnZ^%oftD?5Lctoo_(*hZ|4#pLwBrkxE( zg6O8J0y&~vtYsv!D|*55-se!5>{FX23NDDaR#x%VdfurHY1jC{i2OBkLExhSO95RL z`+mTyX$4-afNbYK8FsDM49bGO+GS2{zqY;XVVR1;;*0rULa67@O7*Kr$Eh;5a8DF5 z!Pl;_u)3&3;ua0XafcVza`;AZhi@#Wmgt)J^H0;5zggpCaTyF%m72*_{U{q4Aea_f zmgst`Wi`{Jq}8vOAy={S5Tl{{(bPGne(X0*DxjRi`sYy;OxP8{8-*RxK0 zDgM--8aR}72G&P0bJ6mMD^X`mc72p6fw)Iy#yu*_GIYvRn_UgbMXy#5xp9Bt3aFpWn3-44|c19g+XoKoa+R{t)EE!bx#?Z3&{2S zDZhSv1=Ao{QL)BIShR>nG+>Dt)J}VeEENx8eo%&5mZL@N1`<<}svtb~^*|m=?hl)zS|$cFAEc-&+NXXuF8B%eB^l@8c5w zb*?Q-(Inw7@nTC1JO4X-in~Y9IoJAjLDk+8EdC{I-NY9fF*Uya>kx=CafBT;n1B@P z4ldd_NEFjYNToK!6jxRr!HGJb9$LD@L(3VbE{QpXS0mw~HolVRW*LaBB-BNBUH0@) zOkok>47A$x*((v5sOjEfjs=A)w-GF{yia`lhjm5#;!ld|n}|?=_LqZRQ zhn_Myx4`LT4b_DZ36T9Px0_{U8~^pae&Dp*|GE8GrrZ`*LifGrmL@ z71yMOAEE3{x&BY~6qvB)kDPM>QzG{Z+Y9WC3{77U%LU$X>j zrBXj$bWSuO=jvUmHoC?s6JdjsCG=8sbqF+K5|Q^1{VdVyLIQdTl2-~xGl;9_&$g4 z)?MRneWoAIx+R%jkqF|6-(%};R~OcS)M>EKXhT`=+7e?=XY7g5jobN1q59O{TQYNV zuk*Fo{Vsna#f67H!DgTw)8WeFf%f!;f(YBHHjK7T#xf1W52euZwq)42QVz)J_ zZ~o=@ddzhyu`RfNZinQ8Aa~J&7~vB{h1KtMu&d@r1h=b}u>P;*N3x(3hRnPw;R5W=ezqby40hv7`89Pdw9{EFMf7O3rlb(ufa$Cpn3$9?xxBS@zpf|SN1NarBf zcL`<>-CvKz06$dls+YDE&Xk9Ci+QaJ2Hh2hd&IDALQ`~!J;%Zw4M)^jcJJ!ZsoK_q ze+MAGkWaL^|FMu~Coy4#k?hYP3D3*;tBGn)P^1KInz+60d!ci6Syv?H|FQM^uZ0k{ z)K8B+S#ev)4uS_eiGC~<@mzne?)05>--}OsKE_6aU^1;&j4on32i2nq-X2&{Y*(Ef zMSpEy+YgC#q_w*Kb0+nIwXa<#+dZ}V)^@5%EWVK#c(i>YC3YV*8tnGR(WgB9;s>Gs z@yJUc8J6Ow&x_6C^J4Q;t-3{E)h**z9qba?nM|^7ZG{(G|LQ(N8)uSkN))QCgv#+& z{=+EjmB$@HGiFI5!uO00dIp0j-tIfPSQvY>EWq282qhhk z{1`FAlYR_n6!|oUtI$h20@{r9Q6riDlkU!aBv*}1(zChWDP_E*?{cg1NU={Vd}Zc+ znMnitHQAALU*`YA+?xQod0c6NI1)$Q2kT;0u~;_`RguM;6e*LEs1{{Ow!~Yu?4jK9 z(Hq`gwM0p@EQ_#&okmBaqoFY(8ash8v0-orZo@WgqixJI#zwEPVeA;a#>`-1uxs=f z5ytc)LYQsz7!!jSBdq|DXAeeEIU_^)mm=Zw*?nsN`T44=q!2 z;QIrRxIXZ6Jcv(<)IehZi8P6RS)(H@i}DX8b!Jq3s4=wdc}r|Pw(6dbtuDVPuM615 zYEkW>>M*{(f*c{fUFKV5$Tw`gV8QH}Nb;TZ5h;-L;SYO3K{SwIroeNLkUBF2@U&gR8ex1BlkEj0Gzs z-f(%`wvAiL;|{o?-GDpfwqL(_ONHFPyGYg%chl|sqViz3CV6)U!8!$KH@}R#DEx}< ziW|03ao64U7r?JhJgQYS-Ps=DXBB?kLGbhUWP7^#1$$yWlrPqk=mEb(Pr66=rHN-) zTEppeL3Xbz=Tcv?US!eory9PF0iM1q%NR@zf!j&Rzis0)4Mr6ntcOo+GY&f^b5psqpN7xzFazr(P zDM#FyBw5l~b80!rz?38D%pg`}05s(&5z&yN&ZXJHrYn2!KLU*gGB8~(Vm~d3Oa-?G~nX#s_BaRre7t^?xZ)vl-i-Q=O z#CCH)OK$5xsDgwe<#6H10P97xR4Ej5j}9zZZ=tx09T+mhA9UPgnKacc7Qrn?q)XsU z1XLZA5_A-D0Pm8CXJi2trhpPbCK@c-%lH$)LlQs|#dk?Umye0?-btvIHNbB*0=9w@ zg^O%0Ye+$w5DE^KbP1gW1c9BBv+~%wDX1igd}j*X?U)h?E?Pw)h}fvth%HSkO4~9N zBV!BM5enI3cKh|~P_(_oaLwN4qiy#p(s)%s5lcms>tz(lB@{_SMU@+`^DkQ#wY_(nlCqR=ScocF8;rh;R@@2#t!g#4$v|>~R$YF^6L3m( z7ACE(KJ9$j@;rO9r#(3HTt%!YK62I~A0=zqir!A(WoyOimUo&^#ain?n6wCQwZaC~ z)RYVPa=$%&3$K0pwbyE$Ll7deqZxRiyOJ0=vN(P7U;UJ_oS|%I$^*DH^`6~fCawG~Dfb(<_m9*3?7o@)2o-z>kT+Wo!(WTQIYUX*ZXB?_|gYmy`g zl8-6N1_``VMH1>!hba&;1Dr2Ml%+i0`$c$!^-q<@!xJc>@I-df_6O)wcIVf+!TGh) z8o-sc_LgYhdSbmBEwS#|w;8r>N%~!q%}!cB{?5sL{2h|(W=|{H{B_uiD#yY0lfsrk zk^YmyjzJ%!q+;Hi{I|D~|Mp$v*zx3>{}-_{)=xqTynZ!4g-f2he~{^`O=#L-{RH|O zzZ(}Y`8nYD1cnz0hEfZK1%$kd$wliY3`!7vZ9`!%$MZLj@*d#7@S!dZ9~$AS$9Z6O zGqOD&T(~>hFdy7k<(p5Wlx3b-p3m7RgRRa-akeqb{Il5r(%11#QPht3+RnyK5kqLd zJ`2fh5YlT$dw}mnn8&Zr3MI!>q{P=oS78=cZw%j7tD^^Es}p1Q?8XgM+k`o4C}lXO zEMr%0J64ZiSc=E&LF0K?KNk-f;Sa`E=M325F~ngc`)q7=CboLpzB-4$TT>9oEp_TD zBuwl`?SO<umUi*K*IiCS_Z~GK?Thn#?S*G<{x0}_L)qU5 zi?ZiB;p~)2Ph%5lzPYR{P5WjmVNvx--}dfsWP7nYtjqD6%Tw=Q>tdAO+&E`@zb+}{ z_M50Y{nn4E{tf5rmd~RjVgXq3WBT}OqOe-@_gkM){ZsbOW9UA$f%n75@#pv%i8yo4 z`V2^5L*MkCwH@6|z-uJ1_U;J~Wix>>;WxGh-W!LR$|;FBbruPTQGUyFjtNjl5bs+a zP&;9O`4&XnZT%Px%#>Y#!df`pDe8BxP65Az{oShy_oFb4Nk@l!_kvEf0-rs})eyVzDpU5Y<9#&?m1EVkfc&T8OxQjb4}b_^yqeAf1| zMV)=j_VTM{>Tzgj8KC7g`}5bp9zucPpP;q$x!3{4#*$-rcKVA^K%Pg7 z#z5ZQltFUSjuGDl6~!Ec(b;WQi|oHWuPnzIU)6&coR5pr{gWn(s*@7&m`1ln z?!F3DJpG4M2wCcxo3<$7OMH9J+Yk!6#P|b}ap>`k#x|_z+pV)*X8P?`mlb*5(M$z} z;=)X?ss1s$2WAQ@x<)IyMk`_jtmv8+UMuK3QM1iw(_F&fjVxNj-VrmG$YE;v&K!Qv zD$@gUGT}N$m;_u?J}X)oCIfw?f>WXZzBanElxi?bs8DOp&s*L(OM>#NWy#L8{@JVO zE`J*qC+`Ut`r6I-k8i%yRFHkMlBB0h9ju`xSgTv@uR~LHtH*}QVZ%8n$l^b5!{#Xp zhNjm{vrHdH7W!VdY<9B!6>Hz?n?12q6i^Sqv}KgF@v5}QkZ+)cAu44sr5KPRzM%St zF}WCChFvZ{W`EhXX=yw;q5wlUnR6e_ z&fw$2#rMjm8C)@Pa0Lu7hxj~()-lk76ClFkiqVEhy%#4%hSj{rn~|jMuh(n`ZmG3X zwu84+)9RBp#b@)wT$JAyvWfYGY!QZIwu%ikf-0h{9=q3$pw}L-OLYuzv~&%VCMfKh zYEX`YM)Nt8AM5)(HZ)@z@e(V%GJ7SOBMW1Gv(2%p`L2<6FGSmfJ&VK}w#~3rlGzMD z3yn%;Am5>D3TN@2SI5ud&)FwngcX!hXPU{BuPR))c2Qy4|BTvw&aAte#S8duSlRAR zBln+v6~&0Dv5iK~7Et%7QE)!tSw!jZnvpoBw%u_3$124fyd4FQlEiVX`OHk=w} zcsMYk5l1Ter81Ho6^iU=eiRt_(UK0AMjH$_M*Z6a?%x*NhH!9OQHP7$$_S$nRiwAn zYj>|mn}J-YN?)@NL-i(n()NF-;Oy)3_hZb?;#P^ibRUNerQ^3$y06q{%tlnCKh1wntiRmDWejyZ}tZUzzYHu210}S zEj8E}6j8yqF&G`PyFX3Chl3b;Vixe88gA%!$jj_RMw6pRFA1-&q?e|mjPx2K?GZCO z?UB$ZQUT}Sw){4c81Feu*jdZ2Klht~wFZ2is{!kTT& zQE-@S3-n)VS#{l131^uu%d*o)Pi~MC7Wi3R1%?VPzuQd64jt*l_z*HSZ)IoQDNS+R z3ChbV3znkQ~epT7S!8Ll#gDwZ3{k8rmQJr*PG~xU{0`{a*TVA_E0HD5m^S z<1+nJbANIGL&oGlY5?tLYQQ_lffJT<`s@9j(UydBTe#->OC&G#SNqXkSBcHVBhck! zsA|VrOMhfQ;euj}#&v)`-?yq_5#U&7{XDM%10gOZg$5!N591xCPMMiA?|^@x_XUg4 zIpHkx0~uGo8Q}x7tXEoSjve&dZ;=Il2-TQCjSEv~Z<#eNF5pLVO+g<_qA*aQuc`tp znaVZ>S_l&hKe&wlDgCnlln(ii!9mh+NG56;jIwilhw$&fd5l_7G2vAco^pJT@b8Ju z!)I^MS4?_oST1STR|5{*nGm`GwGJpl!$i7h5VJY)+h&`Ci~IEzEYSb~ZvUk&4WwW;<0!Dpg@NxuaWk$sN@lp!dv$=I}_(W%y47syz$8 z4i^5bcuUHgPM_hx7Dj;`5i*MGi0^>V@g2avhLiIIPt;*mr5zs47CacLlc2t%wFCKR z5q6Cgyab6zfLi65OU)^qk=O~)IvWwTV8 zs$pRjivqd76J`~YDuc4xUY~}s_+|<-cr<6)c(5knr!te}?dr@KEEm_OeKY#4G}F{? z$V&&w`eDJqgJ3Jp_346Pi5k>&uM@@vBmr8FS zF^Nsg)Kq%Pe#4GI263;d`1aIx>r*Pfy|8`oc?*`2KVw;Z#j;43y0E#*(V|X{xoJ*v ztlgaSP0)R0*U``BU37+nuj4 zdE{k#cDw5(%RHpeLmQ~ELKVYhHiLsa}B;b6k3e!x2F*HUP@4{O+mHQk2|@qmHZ z1%JS5iWJT5o@wa9GhLX5G7HmHhI}*7`~^HxGxpD50GXP}%pjUF(AIQbw=e-2bTgA> zc1<)%NT=qR_D?%sw8(mde>#BooAMr*4o&xc)&l(ptr8KMj!X}H*7Aa7@xZJRbaLAD8OuD9Kxa9fo5p&| zXDqV^Q3^RI!30r9B!RDU^QS6HKU+fe``J=>6;&y`8e5fSM8#Iqj7YEMbc9xs)CbjE z@-H)C|FU=0`n2+{dROhQVs`HZHNr8_Fc)UIbg+YL+D#~`w|}{{B9)2l3jMT~9s?C+=wL~Ce!1@$3&iD2lr3vgr95+*#abPQPTotq#c!*2Q1Mf1W z#)t*9it?xeV&zS@QZ^K#Fcv1EojuR9+@$)1c7}KMqWU2GJd2f-)F~9>T}Uo;x&a9X z7rI;Y;6h{pdX6l_7f`TqVoSf|T?j4+i%#jQ#;!MZ1Luw1l|3ly%AWcjUHJ7qp1p|h z?Dg-}or;j?!msW2@1ef^dvbfw&gJm1e-?$G12vn!J^sB+MHfDD-`Hj1XZN7i3)+V| zXkbiJj_oe)Ms+OiF738|2FyykYX}1fNP@Fk5rBUr0;q}I>D{Q%x!qMhs=%jfllHJD zvqu-9RR;{Cfk4*<$q+`Xj9GZB?vEHCz6mp)CgPb)7r7W+G488%u~WU08+Z6=YW#8S7JB$hHu z5Gk{iXZ_C;TlS_9Z!;%q`5qwMgXQOE;IeHg4vplej-LkLG#S}uTb2;K%f<#Xu6*Y6mMG5W)Bmi$_ z+v<#1ESIroS!x=is!kp`UVkqnltlc!kzUtl_0U7`ppRxR25_z3VxOLDN#Sws zj;!Z;ec0cre0}~tl;DdL)6Wf{@=h^BeaQ|a*_R`7uCLr@f1X*u3z{hKVM<^Ko5=On zdoki@k?+&O7qyHm?JvLxLwzxB8%-iVP(O-RC`FUfy2f2WY$=2)t5ju`I=pIF0BR5w zZlHzS){g)UI}EwP)F2`jtMP-z;4Q*T9#2|9!^PQt%QtsBDF-t zUN86Cdi#8R!v^|i&B3T(fLCL%jE$YDJet|2JN(Qx-#99R`2#{6J4#i2s05#%Drr1y zfD*jl)M!v9_Ns!>N5QBz>UzQ06;>N>mq`^}US7e)cs~3Gfb%g~xi#2)NkBN(=%B6pT&A zDqGNX3nj*-{KLUvOnIb`?p=PxCNB%GSmk+Sc~|cc_b&y9Bf~Q{E$@yH8TO6-ayGTG z+So70YL9uwe<_w(@D`OG_KcuQFO397Q8h%bS5IV}l3 zr3pTzYGds&_{LBGBn28MZTP2uTWLR{X`caYZ4AQ<(xSR;5=vr?VJ2oql8-7-)Q_mf zh-Vblk<9ftKmf${jRYxLaHPQp@fcehYmTAPaZ{NT8tu3Nr?~6PnPrqQYzBO1bYK)a zofnahi7~#c>y0(S_dOAUP{w;AWVTht-V=dJcrOI@aDHwSm#r_&Ab59dpCqQjQy9gu~_X$FFc0U2_IMftlO!pQtEs3&VNs3xM^~8k2JO$ zUQ|;N1wUg%Z>a2o&7!fHjbRX}V&-N`a7L~UpG37j8CJ4tg7Dc#5q>nz5GZ^hmM^(i06nPg>Mdgzl_rVVl+7{R7Pq*&*sPwh~{Kr0|ijsaW5 zZvC*#&qz#z3E08}fd?GGVPp7^w-t>@?0|@1aJDEGh(O!s{5<+Lv`TFNdgUDlgUcKq zt$dNYfzM+TCx&j~SSw;zr|mil)P%u|v1tHXJu&w*-sm|-T$zI%PBVShhzwaiilc`y zpkPFR84DmD(;BNAd8>CYoI2Q<4XD`U_vv{2{-k#b70wH{lP)UOA(fphh&QS*>6t>M z*R@0yro!A2)N#9qw@2v@%F;(fKgt@9=TDfAe!}qIo@V-t=i*0ZbtP@1SjMxjUb)f7qvR1HCNdQ|75S0 z4&i`cZr({h?2oC+51n~Sn&#GMWE)!Q2sZf0{JO9$Jg&#^?a??tVxe-u z&x`Jzr4t^Dju~rJo^ju}eg%?nz}@s(S@Teq89DTl#r=u7O(mM~6x*PKR_KD!dJK=1 z$7E(cuOtO#c-{m>#|B@*$@9DCIz>X9JK{Ex@0L`Gib){TB-MG1j-_Oi7Y59c@g&U> zqM1K@uJ9EuxqQWua$=id%9-YxVA>UPBb4PxR&P2)W|EFSCFY+x;%@7+8uqNgM&%qo zBJ4+wG?(&(?wScJIetRePaK$Nd_u#_ow2IQWX~hnH{7C@F%9Pk3n`|Y8JWbSU0Kdm zSZkur6r(nW)r?TRj&)REXO-)sO{ejNOB$aJ;WP{>tN3zdSu&0W5dK3dM!$8nLD;=v z0*y%(!?i75j@nX-UXIq1mz#gQ%V2e${Q~hQ7VD*IP5Vf68Y>I@so+wyJB$+CMEc$x zNM=|vXmTY^Ho?7_)Lw{DQQY6dvmqRZ+3oYF0(m4InnSXmQ$%ZwTRr&agCDc;(Qc z1O_yz>U@13-;>nwu>UTcBXuxBn0yFsEHoDoY~mr{Cir2eDoUBnALVGmN>o#lKLN5j zzxt?4swXrKObOOCj7+i2|8n00zxb~w{%VHC9Daw`FiaI@M}MKvAu|siXHqmn-G7*2 z9ZNuJXBD4!DE}gs5k8?Zi$y+)@aEeBd!%nLnEjeUFpHhe!t5LrE!?-GF00@YiCT@Q z$PV8eI}h}yp*cVU2l$$h*Nu0sF$5>(SX|%##>s8pI5~!JdT#dihG@!&6U+L+L;vdf zM<#y*HkqEw%$eE9|IQC)@jqfS*jT^{nx89lp#S=j<$r|#0WHjx1zMh~bfEwD|H9ut z2DCC)6=-#?)`9+qPW`D#1{PHvPb1wk$S!{}`a>Qg>`q}TDJVr+40Nm>cB~rBqB6(+ z%`pr}{>`!0F*IhaV}ax53|18pFvcU9-tKU$H{Kn_x><(|?1+sAkuXYUUSLaP8WE<(Cu{I?rh3?l z=X$F0G?|*8w22ZrWQHg(Ky31)4H)QVys=o@kt8C1)d>BkaN+h8_1DmZqOtDpcEAivNX@`P8CAVC*o1wvS;K^vA=aTQoqexh;r=oIwv3@>_{S|q3vo-99Dl|zm zkRsL>2k8Px;~-fuNEakQvImY)JYu2->HGfpkG}%a0!WvzBR31uB@vZRBUR!EsbN!Q zsz=n%K$-{XqCvW7kS>~}r-7plQo9scr$`P>f;7Wbg-l<#U$$k1`=O@`BDaJqde0z* zFcfc5o)hJJOts`Uz^+b8q=xK8cQ}sUC9D z$Qck;Pt<_CU?49T$P0HNizcfleBijG!oqpUKwdJnx5=5htDf)z8MALeUNn#wSpg`3 zt^iP2`QFOC!?)+xZ!fGb-d@7L#dX-Z=VV%hhEbRAO`q(tVOmBQP|ov}Rsh1`ZWM%L z^a#dICQriSk|Z`ZHz>jDGV{Pda0r|`!;9!pa!WWnlOeofcdo$bX5^E1yj?> zQzZ~Ask)@FWDs;!6O+KyYw}bP1Q?YF!JZ0^}hR7|M0Bt}%JQ4>d1?`zzLy3)9>eIM#d8|b=AARbfh{$X2Ieiv*$ zzbkbJD}Jd%`9pfhkv|kYhKT5~1cye6V;O{~effc^%&+uLSB!9Zcl4N8G0?l?Nr(6j zp+-0Vnru4Z^6thOOR%w)-e(JJsPw-4A%^pZVuu;Vc7&r0$Bt(4+)&x0EeW@dGWiXW z7Rh(`>EtoGHDV+|8ZPCAlF@cR%!(4}J1KpZ%$XXK$}_EN;eeEdIV9?D*;J%Re}bCYKaz z?|}i3jNXlTFucR0IQ45#m>kTRB{^jFKK8fZU~)9(qL7Kox+u!JxQ$4Ur^`2>LA>EP-&FAk&CB7q( zBLk3Y=k2vs>VPW-Fwn;($|g~xckAZ%0W@7S=Q+#`XXkG3H-pg2X;a@M(=0JzvfxWl z{!qqeJr3J8h)8G+ZTR*@$CB4 zjNXd~{3x3wb5bsKy(?_C48c9Z_C5K@q`$tqg+qFz1wgY(7qR}K(F&~F0{hWemC$JI zSEWPcL#D6jiH|-0umAeXZB%#NRP7frfCX9=TQ9GJ6&ndvT2Gfojngr$gCdluBv;sYm}*U|*C6t8_*iQ4w_=h>4@!@d+VZ z*Hq{a{K_rdOx zlKdLT;xzoYq`D?J%5x088WvaCS##?zs2J5{?eA#)DzF}d0jlPq$YB{N7<8MgD1iQ{ z%)UBGYE9Mm<@cL4&VZqm^ZQd9GPT{uaZG9>vvD_09i8@Fv0B>9zKnd?O%&(^d{tpz zZa>nK*Zt`Y^qzce?gwJvUEEe84p5r=Sw5!gs=2?t|GiyT?fpJ5J)nFWeu#ZQs>Y@Z z%fAs4XlNtcfo{66!W&V6MmJ&|=%x!Rwh)LWZjGI`I(3h=0#v zl(0?n18;NCgk;idzhocJ$!w(%VtJ+MHG*~?)}LaMHT)UQsurT5T$x>WKvF_{huD@}*(4sa04 zSWLQ$%3joPv@X=IexizFt0b#9!&1lblseuSm8tsP)IBKwJX!!eN=K{cn$UxG`q27b z8DV$EU_|Whm4fKSy2B;R=eon}(YnIuEU+W1zPH7RnNC#NE)~2-^o=Hv&<$sH&jU=Le7P&+V;VK^=S8=bYr+JS~Z8wZ*l=(qIh z3}~ANS^{kyXm_CB>8#DP4|tHVHRUopqCdJp;p>O1J~K>zp78jb&8K%jwx z!4CBQ)N3@L4IT^$G;}cBff~*ebYbCx5rIYyMmtbr1P0pZ!I(f}2NMUwR@gB!Y^ZCC zYG~`-$}q!b_RX-FVKZ!I(F~iJGQ(yz%&;lA8Qv6?7V$N!RQGrK^KOD~m0IRWL^b6W za7XA4*c4m99ho~|Q)&TsMDBn$g-#!=!Tx1{MLz}VLq!Hzm5s_l3^;>FBQpMspkAP= zqdnjV09gJ5tnvN>pvJ-ip;F=ZdM#TIdQP4E(Me$Z=w$08HWju`hEH`i6~c**1Pdoi zC&{XWC)U?WCo3mq4?*Q*4Pm;pQm1_PsRvFk!%?CGvd8GK9{F-KkUCYT%dg|&s-J3| zLY;1%^4y1{J@@(G@M4>gIu*Rnpw*k(gp!HZPc={JPwpFs`R!B_r23P4hWYK3G=Qaj zf&D}e?9U(2-wq&o4&wJmBtEjgumPu9VIz9L{vwv6qX&SecQ=A2aiITI%L&=3cg6|B zPXOBg>dnm;xe17RT{P)l3 zq~piG#i(x`kDN5O_ok13pHbgG9zALOm|#)F<3D8N5058kHi?s|lSa`3Ry*#M1qZ>U z!Rl{nEXsSrPk{2D2%Q*sUcQx!I*o5DPlQiklioac?IBLSugsn(2zuc}^Y5$o! z>Q~})>NK;Q!V`*0oz9$=`jt7ILs)uYSa9G>`Al~eDxayIfp)$3hwn$J0U-tiuwiwp zoblez9j*K^)F5W#Oq<6cwh5EURsjbgfEVYU#yxyR74+OwmRBmo{Hex0!8K?9&9~pi zXEZ?$7C(u}L}}grX7nap9+h?LKTs7o7bM(T$3$1Qa2Vu5%?t`9D&JZGm~m2)1TU@i z(mt~mGAWIm*eB(rxnJ-EK@6{$#% zIcI;*GK+*3mxiH7cn$6b3~jbksHP+yHolAbUEMlC#Nc`eu{XuwH)BKV;mueL1a5de zx{fLpUC-VP6#Gz_`+mUXjvw6TL;cWEh5P;;qyF7}-qV}SN$I|yF!Cq&1z1Z0r$eWu zIXMRsERuxZhs+8!EI$0c7=tSIzRdfEU-;ZjgP7kX6jON z0(%;gVq;GTYX)YV$-Brn{Zoa@Z(T-S-@2T-g7!LfrE*1E00o!vQ1D9Ris%$?S1dsx9&QD@jX4k;WW}&>w63FZ-?H+u!(-q4&h{P(p3(Jw1 z)a|P68hKea0x!H4w4GT#1BW6Ja=RK>ujE^PR~U|XQsJjU=e}|dIsVGI;yI|Qc&@1r zBcr`&t;f%$&QYoqp7>xbbuM$x?nAxHoXa6htp^^dJklA3G9ae>X~Xc4vA`qQN4k}r zeWdURge^Q$dIU+AK%uKt;E}pXYv`DiOg#HY{t=@}eVWFW2dQD|2pgu3762J)Eyv|- z^7x8xwO6;l&ymEp8d&Y`ggG_|tVX&qNSs`WtjcE?D$O@E3+SdSHiXi`G`&()DNnF?5H2*y0d~XRT{k$+Rj#)9qFoMEI=Z9I!J@g~ij~_kMe3&)r;r7E)$k$c- zVIK@<6S4HrPl))FhiVU_7S~L~=YZ%vXY@1RS9_@XFwm+d{L_F3&xu*$YlF;hlz#)A zm+#@m!+JQyfk@^zd=Im3KAe46K8neLRz?^qiMPJ%H+!vbPwTGOhLS{_`})ljtL_Y% zBDB|xnA}7foil)=C=ax(fM*^?V}wt^d=L8`c1A3g-yg!6r8;O1>M+i^@YV%L@z#ad zMdUAbv4GRqjIZ`|eeqoMUzA#mCsgIX7`iB}ROn&^Vb+)M`)lv-YNf*O&%GarImF)C zN)gmuadPi3y&qaDy}$B)$WQ@=*-CYov`i*O4I+Tzna5ec*EFvM!{{5rkPt@kd*ast4FVOIH$qwDc$t zONe#6=<&SV8Lbhs^k{b>l^(6}qs!W(jYlCv0~BT!oDGYa`a){~sQf0UwcuOiD|FRbXfI-TD+2@xEjVsk&fp;3Hjh=o z;ZBR~MID3gM)?-&i)iH}afbYhj+X`7zZl@LD*?cDj79c@;B!grz@~sv9LO?!-GaV| z1UR(m&>RXgwOCmkL9etK$AP*w0a8E>sm0I|Yb{C$r@-LjhmVyEKVgB z(>TCIrNPmCU4N4+H1S3M(q`)s$MG;d*Wc(jh?4l)0b*xB)N}oHBbF)Uk?Q6EqjKfv zS0KvIujF{-b?$28>K#@Qy_&enTqf{DVJEJp^hvO(s~Lo)qlT^nkA)xaF7NQ;*~gK3 z^>Ob9r2SIi$5W3Res_^qA*yF!nqietc_H*Pw2$p2z*<>VG`=IAjij!nC3e zlc>qWup$B|zUOh@!*&fWgL*>~*1v~KW z2rxyYuaZoJQ*`Wn$JO)~PKhrT3po1Vcf-`$B;jjdJ zfR9Tk#Ds+KG>U7W^e<;7|5 z%qniJIkb#FhnE-8k1aY6%^{)YYtF?Z__%+OZ?he`{B(O7IckIM(O*TfUx_!$jz37# zqEM~BQ|4v|Rh%wOqohh;ex9}oHR3N6)fJ#lb>t~iE6~RQDu)`vI~8l8X5RGQbRJrs zMF7f?G;joP(ImuC+d7T}?Squw7J{*Cv-+6rT0mlqF`!yVj=egFj=?y2>oKd<2! z*n?ZC4=>MdrGq#fe$YF-G?-Nn1c(2JUyp7pwv?nHV;BE!c~J`>2}@efUd1rOyx{I1 z;DS4U?{;uB*$DGle9G-aH;74FZ?vD;x*NPTQtJrDLrTy^p=h5~nBk9<8qAIQcu%c# z($TecE4@y`l!RsSp83`3>0FC5AC|)H$??H=u_;(QM;a(m<0##~BU=Gqc4gpO9D={F zIP~AJ#E~|4k0R|V?u%4ar2XGZu}A|0(PBN#epK})Ear=_m^8tfGZRL+pSG$9i6lr>DI6V6Ndm(Oxy=OI{QG=;8AUuEi z##Y*`0&Io-QodW!-=&68fKKz3xTBu`w@H4hoWF8u>h{1a#ZFn)df)`A^)z~T=Ud(v3Es&MhGix{lDbuo5{v*t^MOBj9VLx&94#wGtHP5}IPVjAPW6uP8m&6gqw zbJm>vfaimq`3{4X4*;=(*sb#&4eFlnR6fwa^hq^7(E0#mXo13x-SeF~cx;*PAagePAD+Cn!!Ji}_<85+4IBJ|xDY^IBCaT2;1?{qDmtbjFF3Wv zbdEqt2}j$(MB5IWcK3PE$6ZOh;ArV2E~%B6qN^q^B*Tmc7ZaqM_MTs0p6tvy;kASl z1#_F+8JJGaS#S>Mqlk?tnKg4~k@DkC1*VjcTV-*R8m>ctPYlcvt0&jIDPc&@li&)* z34=$#v+NAGjLDOZ?6o;i+|U4|2SLwg=u*zT4}%Y)8o?9BI2=fgt8YAoGI-;u!c!=` z!c&c>?y!USQ>mvYSqe{dNU5hXPsxg4=BXUQw1dJ2Lr-+sLE(e#4+61`*gNfjpl&;8 zf6(^??7;U#;0eeOcp`+5C=xskCQ*}#VFv_Ak$lkmMDJ^xj(9J47-4b5i!YacDXS)w z!inL&AMVtO*>iTBV9G-tVFEecV5|%@@5aaKT z!~Hgft;BeMtj-J8T5)86ikpZC>l3G&>WHq&81(g~XB_%0-oBJib&VhQ#(}-XCob~~d7tlhy?kXgJ z*SY`zUB|Utn1~Ap!>?#?(5q#7nydl`|L7p{{YM8Ahq~|bQ}Kf-#-|XE>X$eaI3~Ls zGRNxjB4)xPU~q^YD`XsausVz7qyKUGw}%Em&^(M8RRr6}PvLIRQ#c}c^Rep@TphY= zGi*RR(Yqc6I^j6uIddd?1dQ@W+x+OEc&u{lZ^y5K4458L)GY{FJH*nc9m*V4*Z&wz zVdiKaZ(rsca3xNZEecN@DjvdFS;a%OqZ|*`js}h)NJMH64Dl0Q{qU8XSmycAnrGnH zW{k;uGX^-?JTx&1E(6Z8>O!3D-#;*mtnfTXMcsL0{OVHj2dJ8@muf`LQf z_|a76D7KJ{&Pj;8u~cM+kU$3}U)5U7Bco``k;AzoFy_p$;xRmmc-Y+yj2VuVj;R}U zXsL9pd<-7X`VB^1bkiL41vMDa@ggyV5J2IsZdkZ z?Olv>(+aT%bLFAtn;-o2U+kDOvvL?Ez_k7z_=;{&e)+?^10o5<nSG5%JGLzg43-LVffH}GR#i1g!u<^bB*CcdxNU9H|x!#d2JHTpl5 zamRV4NoPZ8yeCJeVy(y9i$|n4%}1%X%}2NoXIkU)C>&IQM_%rP!j(!?B61Rzb8ZqLOoHQ{Sp-?Cx=bMGgu^h zFl*(}tHOY1qky2uyYsM!N|@~9OCl=B*^v_dQlS#gh>i9sK-d5)f?_ZA84c?=W%0D(ih}O8NoilWC5lHMH&PJ z-lEspT3G0-P_>o5Itl882|)oWHwYsA`F>g6KpIvh7wVz^;@KOQRF*Iyi1gzW5v}M- zzkgtppwi!BskL}p*d@7mOXs12Vk7}}E%^u11GW!#UVH4mg{hWvY1nAfh=8m4&?4b# zKD3odpo51oSfL$&4<^sCf&LdUW+ts{#ESuaGvKKvI~b>r?;qwc10tgju* zbE?S5OBQE8vM6^3E=M3|26Ew2tvUYhlW5G7MsK2cK3rSr);;mFK-! z={7KrQ+!oMhNu}mK46br<;rKY)`oCZ@zw0C0r?&{SWZMf@O?8&x6$Oih%=$G5q6|3)FB6 z28Qd5F~VZJ0fDed%r8Z4d7f^3#WF}(4uy$PI4Xjeg?`m!dE9pZE2Xt?f{kTNM+yZl1SOtdd zc{fvPG4HQ|v+-^NJM$G6fJ2wWK*8Z6Wn=)iF+?nu|82~IJW0@Y@|Io(>3tKMf>xx&x^RSnk`#-&cD-a zNp-l+qt`k;RDG_6OVm_r&Wpum<=qKvY5B@L0F5==s;a|0_uSBigqy?NESfwx{k-yw z_(#CP&r{NcMRmBzQ`SvFi;9eRwxK!l;BHjiJ_WbM7!li+))8nn!9Oz67{wVynYaJH z5nK)$-DbRVu&IF{VLZmUR02-7(17BINmU%d1pt>-ZL~Iup0LXI7~W7 zsfr`@5i|w$k=m%-86;Fs2^BLTlT~O#L@18P>_z7cl(=5tBXVm2)F5;p5xOoU5TdAx zILICQNJqZ$v%mQ3&&0+7|o1g1v3M2aT^OwC$$h;>f~MrHxC%&t=Hh0 zTBt3U`K~Qsd&ni#;x#|GG-ENagiv59w&b{BIj|shPV}z)X!yhXuILv5a7>~*?;dVfBsi0I-c$h zYx~hxQwR+9Lk;4`uRK_?nQfV|ZVs8T))-8>h12tSb=DRLB0+A%+H;b;5AS#&VP?$D36sB=OAjW^w zjcIr*U$mg$8r(*K5}o$UK=GcLIIr=G&m?9BFjqRWfsYZzDOB?w92Bw&#Cuvn8b zNFH|DC6*L`Hk|;j#$%Dn1%X%~olefknd3rrS*Ua_aYdeg#gGvBlHeW1sq=)+8ycxB zg_q6zgmJ?hRG41QEIXgKyukcqmUE2CEf;tobz!;9PrF#T);H8-ZH(FB5u)BMPdsJ;*kZLIrFDHgONcr zP5=I@xRNU}7{i6KDmIuN1YsJtNh1^=YVpw;@(vq#-3+Jzi!h;LBZE1vy61Se?Fm=u?{P|#y6372pytf~3zM3HyC z#wSygc%&wClMp*PRh?q0Q;jLJJW0r}h-F}<8Q51I{w?f5nyB+y*Z5?Tcd(`=^Lz*c zNG}WoPp2$Qkaf*qW8(UU$e=xe?ICg8?TJTuvVwaA?$;bXY;X`n;lR#H#tfRng6rdt zk^`=X{SY|R`KV7eHaUFA;2?;?fgPl9(Ht&6H+qU3!jtGCS3T$$Ke_r$;h{C}lr$!s}*UxX@Xa-0yg-#|s z71<=bVh{?35Ox*c#~)l6B3Weu#hjW%fuU|nIs(z@Xd7WpX=vGr!XyThg~`~IEZf9b z%=oYiR2{2AzL2xpOla2G-}lwbzsLFnPUMDUk=ev7N;Wau*nv<3m(dysMvjC~Q!=g4 zY+(m1sj#EC1C~_WQQ>8`l^q~PO9|pmf?7l-sL6ncWe(umU%_BcC_dDo5Z5vK1Kie2 z?9^*~(RygKT^trznTGy5MnrA+z0BY%aRduhEqVFF(dJ zy+()wuhyb>$@#*oPQH7W(n}Ltdce0A#Q{k?G^uWM@tO-;)?On-O3zAg#rYZGqGe4l z<*=nz6_-+Ik41fGU4yuA02{SaxPst*-83+6__lMk-!k;}+lIRnb4s8LV}P+GmKsPe zRhFC)i{+o=lp5XXBPz>;vNCEydKKBpi$Uaab=e$rF@$~r_J#fFKrRy|&2<_oq;2V+ z%FX?F4n6CS=h{2r<7)5p&TDr8x+ly7c6JXyuolE9>ov&;WGl}_yAf1$XBs<5Rc0rU z%o{NcG?q$qXKp8KDTkx0-dA{+Q6?q%J?Y>(N(Vh97)We^i?&u-9Gr{B% zJ2RZWWPo7gUd>)b&aXbrdE$1NENS!DI7Kl|o!uqdcUg2;SiUYPW& z2uD;%42Ot2UVMX>^x=%=$b5d@DMi)^QnfH%!V>fL7hxy~AUR(23W)|JIjF)Z?s;AS ztK|c_8JnH&VFL5|YhoPYjwM#^d;iaGpGMWPf-WN)aWQ7)+ePwh%wtH_HQsK_dpLIN zjJG`t0gl`P3pjHhT_*={VhqHi#D*~>7QWAs-S-!Qi!g@ZVs=p*gERBu>;;@UDbXD9pVJNb?&{)7=5!&Ms?_JuH6W>L9auz$U zwWD%~XP(TClh~;+Gltq|VAntL0<;$2v<{Tl>W3pn`E3E4<7Hk=C~X=9@mh<##tCx- zE9VNhzZ2Ip;#&mX-vmwahtte~#9S7wEh{|h_+}f80d8*OC9b@V2}{d>HLSG9?}lN& zBx6*Cyehnb6xVk0VX#C`$noe!j7KqmeMS{$s$CnWaYOST@^0uq6zQQ#d^`L* zTpo)y)aZ_c-nzacyTkc;jgZB=Ti)4VW_P6K3pIy}r|3~3opY2&z`yT|yTavm)RI2ggV zan6cQ@C@h%dU(O4&puAkx$Qd-*&Mp-}V5) zVt=rhgv0CV&WP6q4>5-Ft~MTGR$`Y)vzH5;I_l-6u*?$8NiJ{ocMTo1L>=% zGsv>=V8QwjiTbPC8iYdR9m3WlC@D%9J>-aoSGUt*AxAa)&+W+W&+R@3B=H zITDD_?C?3ZJuY76xWdJ@SUChxb5U`(EDvKL#wd?>T!=w=mdBi~F0~f~S_}NQt z6b{I1D!Og+&eH-T3foGCFB8(+*ddsRal`4E*_H*O#j1hvZSd+YY)3_Ayz#CZUedyI z34h*bfZx*y;U#u7gj7eNVpA}Ib%kZ-C1~&^qV#>+zyZthNTQP+>5}Pd1iUrc=8dUs zJoL9lOtIqzV743P4Q|y5(lGjQ0F?Wh4y zO{6(1PvarkP7{AiwnLMNNtAYi6IN{LoQmnI4OMI^#fa2YX7jE?W;<;n20MsxAnI(V zCxcUx)iYS>nu?q6X13FlC3+VslPx}oU}ifFe?eC^1=bZdvrPufc7q%)3)Yrw=M8ve zvP$cl181OJx;H+2)Z8 z=y;(gJRZ1m4zeWa()5_-$|pV#3lJ|b79~WIv13E+ofLNAjP%-c&2}2RFr-4QrAS@T zFg^S@Vj+KzI9WYo-Z8NC;-PN~#QnUIjCQhi~2yQN301zC-zMmkjC6>Cd35GWcGvJ}q9@ z;LgHr#vj9N#?euS$8u~1lP93y6Q)ae*9p{ynslHwheEhm8Hz&NS{telqm66e&R?`~ z$en0I(5W1b=OLI5rHKW5cOo`|EHcq=qxYkS8--)Zro)e%!Ym#@shJ5+CVD z=s;{$X2-xENsl0VX{?-xX6w9wQjd0#2JWPUYY$QOS6-Fr5Q+p0r*4nX)!f-41CBfN zP_ce)#qPLka*MlJepsa|TBS1s z9$V6gH7?VIxHZdxe-;m^TUl!!VU`0|_Dmeaagw4kgqlustjT71*KHOXM3IZQBdtd< zKm=-0RTLCjQU5P+bW{a{G02qwBKHkL%aTk@wPl#oR$+&pTDP%{l_o3TN;lZ7^(x~i zI*@kJQFK~=Qx%syJ3R?OxbX(wRn>GU}S> zbDEpVgS?Dh#@Rt<$cYWbcwa(m7;=8ggh z2y#eh<^ecUm$_=Xu;3l1af??DEz8S>zXv`h=K%X9$Lnn39<4fO%!gu4{#4m z0O_dbIhU|0@NRK@tADs3zUX$mWI2crSc~rOaZ66A_oux&Mx*G8tyNsaRh=w~s=MXJ zz_I1_^vImt!|gv;u>B{(4=Q<;k@)!J9(i1Nf@|iPIs9-#imG9%3+-W+%Mn!D9#z@~ zAR7nIZF|4Rf~vaez7|`BGnNPgXt6h=bn|P$u853{(qyo^1eGC)&ErPKl-KtA9Etrt zH&*BgrbP8UZ#lKnZ?V433(X1aducO*_d|PaUeM{%&)_0tH@s)M*@%Gq7($)(tBOUp zPtqzthKL>A46*2O>&7y6;xP9GY}BB$5KDXuP`Ite-L^GJ$Bc~F6G*3foy@Z+=(?z@ zwz`ccv%1fqz0z#zwxk{P+|C5rwlGx@w#V&k7f>_MXe2S_L}UiCsTXJk{9$)}N%rK( zrOh<%&Q;BKgSCeZJ1#Jy#o9B<$S^08jm@(V<~?kvxLr-nKZn$w^H{g6SwdY}5$cKO zzyn}T)R>h1&FWH>Cf4GcXB*NJFoAdqeE5b5?*+_O>4J}uew$6hyW*D_KlU$8x`F55 z(`)$VzZ|hxCcDry|KmOrZ{(WA(mU0KAbcJ?Yr+N6^3Bh9(EPwJ)A@;<5Bg2K9rJ*k z>p+csJoqme<%DyArpSkW3lY)mP7czk}}~>Ml6=K#+~$E{7V!58uT<;y%YZJ zqzQi${6~X#!vFkn6CRyMhXtI@FJ^XLur;q;g)nu2xkc^xb|a`%1an-NmaFW-MwWVZu!GX|#3J(+pRC&OM%_><;zf@tcJczZo z+MsF-=7&`QBL(!f)nSXO4AqBJeW;DSEUn?thzgDRwxJ;yjg6`#_BEgn!KKWYXCM~# z@r)^}!#6*2Mp^pR@Vo`@UoI(2z@=(?EBC0{J>E4HTT85|6i&TX#kKkxQT`a!I$^{0 z9jSX%28RMF&zf&d1#ncJKsS_SURCa?-osG*l(Kl>9^B>IEy2f>WlGIIyZL_nD&N2Q zoU#n7sfrU{5ruoONisX2A`8hy9GQ{lC7~(Y1-4RI=|VlOp1Y1uNvv=snpA@8!&Qx& zXtTT3+HtfzxS`m;gnNEMxcc!A?NfRU94@aYx-Mfu0n16V~0TfxfgW68N}VP9^#heYma7+rKqjMiq+m$NE)hq%wl) zfLq((<*D^!i)eaaYdDB-ffcKSd#+FiF%DIYk;aJfjAl_$y2D|FTdX+kQU5mVDJrt| zpm{Wo%0*@^mX;Nd8Bdtq42HM{NCmAhgA7Kg7`KLO=#mRIRYHe^4S_AY@~EUzSPJTq9=tVI0LtWWkorX_8HKQ^!nHlW=$b)5LdqGF?wYgp3C}8;9v54x~ z0cU!02j(bP4OXQcr5)ndw!l5@nNz;G)EsPMu7uVN%TKC?n`;@a&8a4~@TtJg#Jt=I ztTNco1OB+7R<*IRqr3~=1>E(WT)->{rwyqrRzk7l>05#Z%~nZ8mLf~CVG9ae4lJv{ zvS$^3I=+UN3oB^ zarD8XYmHhl9m>JHqSiDPleI}!uvOh!M`X|zwyB)0W#e!NZJZ2-^7f(~Zn(Dnu1IMt z$^r>EgHCLmj5*~KQk8Kwur;{muJm9S-|A68TstaPdP2kQP!F8Ai5|K66BQN1VVoBb zhvDKH#(wmDu+A(tf2%6%Z3}lT(n;AGRP90UkO~e3hg1@m&Z!)(oI_7&(0M^pmyWYrvR z!AujF24iyC9x=kr@#gsd&)vJgS9RTYzUOG|bvUvlkT{Y+;w9eixA6<`8zG$|UUp(9 z%|j7FfNUXQNnqo|AdbMsxpeB>xKrBF*3-JZ?v&1WI!=Nm;tY_z5Z+M=t|Y8>bNGwqD1Gb zk5lQ}6W^Ig+~K!3-B)|0Haf`)a44e>7e^PyDx(h^50??qd|1Q!$njbhj^3YY zYedk@+tZ_WL;^agi@8>rW24KwP$+{cJ>^{y_{&o%$(c+>A`k}4=p55_d=BV6`>om0 zxsZNmw!*W>e0R=sbNTSIZY9%Y11SB%XJ6UUYi?8}m76@}2q7hbSb`@deK>;5$`k7e#L@dTS8}hQ70i zomVH;L?_prS`)pp=JcBAy)`FpiQeQ}R4|3`Uf+V~!h-h}*ypPFSk#EeZ>w4!>x)zI zJ8$cqxcHR?Rvbi}_s+cN!jcb`@MWpf%c6691COu%ymJb8<~|dd+)l z>_ZL|Uvz#EUxFz-KF#OA*uFRP_`;(17TE_NqBqvOxrWa%T>az-5(=z*Rf0r+KbHuitm>s>(k=)UT zcl51|j<0=gEri6O+R?jrynjce)4BQH=<&7Db9^W?5naCHiqc2cM#n40^&aHoYee}-y^Xu*Vi}2A1 zH7cn1QMxW#uq0~wLbUsZBHQ^10TiJsy-RRDC`G_}bjH8wqw_4a5deS1M@PgAPo?_C zq1~Z^9UFIY9Gc*Z1NPnWGnD+5yeA|i-9EF1CROZ+c z%A5>k-d!3$vCMZk``ZrZjx(<;d70djTrTdU5kir!xJ&Z_AAsw=CaBdbrYX0zy-TiBWThDexAqG#FYcunc|Rr-WV zzb_(rZq4&+Y?qyF&_j({_5P~p`05j@qf>0^Vi&fWgD5ca|yfSm7+ z^3lhT_ebyC|L*p8#l?fU40`YZL(hxL6MDf80C=+wrSH*)gm8x(x5@wvw6rN);VqgNWw zG)8AR6_=qje$W_QX?#9QMcI?t=v4NVY;-z%HXFUc;h!K<_DVMD%N@x@$8s;`Ai>O-1J;ibh7EKCgf?;+fC7hruUno51OttMIUlJAydtHg`5F+ zc2o2QNB;qz&Br!J&uu=n85-RD%I4^e&2Mdv&TW2Yb9A0#IjC>Tb6cVlTi)dM+?I1& zqIb8vwuYZUA?9|V$_2@aal#~15 zIhJW2?E3__I!g7zC;C3gt&SeK@W~?&ajUaCFFf@8!_bFLgueXni)?!RKxa38_^DH$ zPGAxI?GQh?d0lk(XTd4duIM>DblhTM;Vn4kqT}v4hoyQ7Zvh)^#Aq&aR(sKh_|#>c zo!1_rKAv$T$A_t?5AG6|vE4&gKRM=(%lGhtW2+BUzQZmafUn!rD0bi#YMbrWA(usX z<7L!`{}U%@A4b=U9PWV|`IQtJLomfIr_tg*Oh-pdYkQ^mOtCF(^OkhI58083J;vwj zqgQbH!nN5niDTo+P0@!eYtsyyC!=$G;5&Mo-GIo8(vyU9mA-(>gUx=-eChd8jJ9`6 z(EyK^0a$r9-YGv{&hD2B<5!6XVQR zq{F>cajqgdFYE3EOE|Va1VUqzLi7@=R@|aLMQ_%;RYQQu+cnX7LgK+5g4~TqGqNck zPKl09Jr4u6+v8JDOeF^AZL#Wir$+A)C=*?l?Qo3H%IJAQcA^(x*U`(Z|*pF1kFdFPU|5rS8MJsE-W*;`-}hAB4?Rwv7r02gZliTd+Xh!OBfu z#`t)b!(*cNQ&&>ahp8jukea8!>#H)4Nt7P9C)%qDYF)S~dSCdZhzU?-bmj=8J2L+G zc)ntNV!TDaTc|@Kx+?S3_?O2=uMpD!=9Iru&Kev>?uX^ivGob;8p?b}*oY1M0eh-= ze2gcjE8Y~2z0d926A3Oj%i?})37@KbxiWgC@^mFTOx~-EKB&BmVf z$62Ki+qCo3qW6hfjxH0k9DPWn7R|ANEc#&j<>~A#`*1pT!7)_%8PCr^@qc56#ea@b z=7`EXr!p_hh+YZ>-vSBG*S%Y3CNFj^n0SRqM;Ix%6}>#W5B*%`Y_MhxZB7RK`7`mQ|nN1|g9 zr{z2kztxgd)R#U2?|UVEE_hK`@VNjhJ0}LGOl!o6)~?Q;u8y{4w*PbJ5ogs? z7hB?dk(~T)Wzzdh#uulOKAq)!fNxCpY3_ZhKfdPt)KoI-tFl=?HR+T6siUJ`82qU6 zerlHDeoE?ypO*ApMGeK7QMkW_^X8sgHJT@)Jg-8R%7MYd`6W zm7Vh(u*BOHp@p2EqU6@N86ag1=X_ZnOvw33G_yR*ZND!|Rr*N{%FXyvvXwU_l^hss z1k#OGy2i3|KEt(wOJn@2nkvYvNx8=v5$_KAO46YbZkB@$t~gI~?*C2<_?o1jP3{8F zCiP7)#MhFl==ZgU{o<^zr9WXtlc{oF))dh7HQ`IvPhfIrEeY-vr=H?E!544x$wU0o z-=wR!NPWpvHlNBvOGS|z(2w2OPj)D&yC|=pj8E%AA2j_wby2@69ez}B#9b=ylNqCp z|Nk`Om?@^AjPPvlD>Hs3ut=uw_2~^jJ8N{GNezM)%x>0C0|W~xQ8S>_oGkOMStr-0 zCR542;Y!l!^&@KN_jMGSmbw^Thsv$K(KXaiL+{7cFz~}_s2v;xF~ISZ+dJSZs{K?q ze!x!$;pp_zJZ!GNkXpfH+FTmO>2H=YD%{g$9TF)1Ed;U`{7!6s|r{R62a|MS_@MSCjdsZzkT zS}j5pSzqqD)4hJK%jCI*H_ml8DRX}|mxW>axtsm;dYas9U4$L>Tide$G}Wo+?yZA@ zZl(JV$>b5Aa-UY}UvT&|IV%slaZevo%Nc(YuqkfzOE&qN8vK%@{-%xA(Tb#BlB#SJ z16aauKlI*^AxoSoOaNE0t)z!%id$8r+>_8GWmH&K%5L&YHrgAcRIKo7rtqq;w#qN3 zZ23iB+vLk(?3AjNqzcTYmPe`6 zCHI$CmpsYq=f-YHz5lmVG&SY^^FjJmg}@JO{#r7b@pn;w@nOFpmn~W_(cg8E-ai|U z3WjYXxy$|F;D8d(QjHgtm8RSeo&uDWfLJ~4k0Ihymje56UoOfE)b9T{n6)V%Ry;xp zpE}~l*}FNb10_Y=Vgp&r1Q5+Bs!QH-MVCJHG@P58@f6#{b(+=xvxhF4fZ`4Jz(`2v ztgz;GLdwkta17ghIv`UcC*T4BaG5lottWz4V? zafn5Eza{4v!D1NXb!DK19`uVg`$_e@v&By;qt0iI-IrbR^Qm%f-d7*y*%4n|?#DmF zurs0&jgcXhOQA>(!2*FuT3|C6=php|%kr`zPjcdjl&5YgeipL3DZNgZAn9RfASx)= z8=`=Kq@`SA-Y9)i^B^5p?HJVZzD&ui>5D}vYnEq#UKJ%%hx{bFN)LkilPR+!%2rIK zVay1WyjfH!)zgI$__zfR(vZtVWAN0uNLK_nIg_#k;$U33M3K_ z-yr?$>h9XNw`UCcxym3)B+84df!#OOz^<;29c`V7#GaP+&W~*+6viOaNMaMLY_Iz# zam#8r<0L7-=v!zrQ_EmJpK@(hJEI48 zlBqJENgqJZqW>UF($5%(Z{|uHf)s_*K6TWP_|{8kVp)>~cV+#Gtf5r$j3-1ZO1dZM zt#B;4)l4R-05>TaFb|cAwQQ0oce<#P+}jHPMJW_2Atz`q4yO0*1!Gx|A_fcD&`3P= zm{L<=!ce=6>Wn6;p~!JrKLz6m)r_~qmS)-&wxDYv8Pg79gdH&p{ph6S0KL-8Ke{qq zZGslM&~!5O^hMJP-~j^7Y${$%{_Lz@Ly?H&O1maQx{wTb5dnk1Pm>q@{jQr%TfMGS z_EDi3^j%U(cOdX?L2iXI3}gU#?m@cY-lKCcpD;hzTo`HTrXZoo3pW$h7~-kXuritY zZop-7RUd;i)NY8Mrth#^STbtX>?+f`8P_e03B>fbY%D5=HNseM$U*SP z7cH2S>OX=q_DiOWT|)gYb7xGuWP@F$uXH~~`Jm&%hbE=(LT8zoPo@9;VE?85%fYWv zjD6$3UAX@~CCUcU>#vxEMHNA&n%6{@#56|ckvg(%4yX4wq>iN57tP6_l+g_HOSk>t zVEQiy(|>h#5a-7KW8JFsRk9_%?`!jZHGJGX2@7sAGL)4d(TeJU3kJnRG!IqEEH1aw z8_=ER1Lzu%Gmm9bj@F{qQqQK`_Y)A9p9aD}IU7rPZD^ut{VqXKVFQSz|iAD7z9);K4f8?1Q97UD>W?c~SBHE&cz%_#70(Le}nE}VSNV0R8 ztV_-aij=({x7lQbT_hA1Nf$jE8o@F!YL)k+2ujr@X~fTEaYzXXNm-{dMuntIHU~yh zc!sL4k_P^M`8gm+ae!xxB)}s~3+ZL$f~O1#0o2{ca4~k^bmcJCy6^hZM)3&azLp6W zgu$dBvjw~{AJUddJpekD>&fKQQD5FD){kjp^;99@!8HmqZMR{JWGEzMH1BW5;*X_*MI)V6iRK5~pOt=3)29-TFkg1e~_N$_yNoaY(wEj`o z3GNRQav|l-95Mfq``lB|NYkKIn!*`4+05;QS${{-iiuuV{0saB zgnY~1InF!NdAUNrGS*fWt;0{{E+;Nh@n9vgfzNbmP**f4C19cAU^M&pRKfFScwWwP zNOUj@&lky;Qerw1r?}wsx(=$G z--naQXRP{}P+>6>K*8#~NNJD;D*a+)#73K=8SoS4HJ&cJnkRNe%I!>=iw=o5iPnoR zWhyHX6lLzRSjyNlQ|_JyBQ-PSv>0aQ;zt&@X*mXikcT4e{^=l$28bOY2XxSD!Lpr9 zeRJ^!>LIuWqRDQW>~7EdMebFkT3PC%dp(pFzTnx!{qi3%2}^OfUL;5AVkzLe#8u|2 zd62RoJ9>g3jcNFMkl66czY)KTSLAwLX0xc-mByyOU$`wUFs@=aBb!^QpAJv*5H4*N z&8Zrc%2Vck;VDL3#qdzQa&Z6iFqVla_u^9`FzY)uANqJdKQ+8+-AC0K(fvOw!V7L* zXk^O)!*5(AP=pq_Daa^_%)a>>Y1ItAAiy54x^9fB8W zB7o%6ki#~1cfezE=9dkqTLy)$29xA6f$&#TaHiOvSdMqa14j@^`wH@{U-yIQU-$mZ zrO@Rpz{w2y$;>Bj>caG*CP^FXPu~fI@Fsu_mE6T%LRFl?jfBS|<^BL!A`jR?4S@%* z4@#c{TZ#w#R4m9d>}hh3F$jl?4O3>H36(qi#GEgON!o8&3dNSzYbE~%?G~WiMHlWT z_mkAD>s*x5NmTTjNJ>!YH+z37vJ{qT5a~BLie3f|ejiaM9K0ShqPx$yB$B&mK?cuS zx9OylTvc9BaF57u>8gWcNouO}w!lIFKudnq_}%OQW)~N_0iguRKIn^^0t}|QGWy*# z1{iQZHPPvfnF1W@2mI}VRIpVi!?i|0X7Z~jnRvhjjkCsJU}>NLKmZzKQBnaW#ZOTwlmC-oa~XRLNmNy$@}z$~{g(oI71ZV)W&FIe zocsRQ{A?l8wZE4}X7&5IrpEAFd?q9KBGiNt8uVhYqTeq?|H3v!s2wx-;;TXL(eI^x zyM)BxGb0n>zWp=klDo&=Z?P_764f&qVv1_;wfpEmCMO9zW^ zPWkj-4RNw!_bX4~fx`=A+Q7#$E5w(37jC)4z55ifD0MFs(aB;I7(8foZs$ecH~=&z zxnofLBKLWGgX3Ktk_hFBl;(XFG|aP<`wf@pUGM(w!K8b_M8_8{GF#OUqj-<7ncMnQ z|Deq#iimklAonn<=<3tHZU}sgdcP@#(VoAoTbIq}-5(X%nExJ8FwLAnliV*I!j}AP zeB}e^&@Z92qP?PzVVEF3>;{0P+yuOH<1spogL*d=TolhsRm5((agUmDe(yD2fU#Fx z3u~-bEx3+YVy*asyM=kdC^0`JFoQK*A_4oUFtG;T_NlXQv80^Pt~Xy}P!Nk1x1NsA zIORT{?yYw75E#`al3A{phzB+o*2&_UgJ(v?b17HBa5EBIwyrz=%(abbVbG=0cl#x} zsW68Dq@7Hi$*0V#GqJBq7Tr~~OA$_P3hp2Vrj)@KCCcvOET4Q`FH3I6YsOYl@~}^b z^;8q?1<&dC!%aK_)gySw!P7WQF?a5+YPEcHtJWH=%D%qQRt1y^K7G`$ zWX*&3i&E(eMgy`*3@pLfS!5;;+P%u4ap(fQ*Xo+L@yfAaFR-eh+)L z6L1G`t5*Z!&w>jvj)R-VaWD?5WB0sdyW#PSft2j*z{+ld)< zw7SIX#;%B}&|zi^S1bL@2A_maH)zTYZL8jKD3r0ss%XwcUz+}F?;BBm$S!7vN?Fcs zk_(FlP!llfbc22Z)fBBu(t+QQHJi_;wm* z<@1_`XQ<&bfMpY7#ZQ+~&`7D2P3SzI@!oTf%Jj9BnxWW*VqzDJlT^x@#z!JkMfxEw1QfVODBaJ?8TY%8nOtb% z=J?pPMlj~B=^}Zxf^iB3XVf*bf5yulDq|w&XNWz8#S+A?q-X@Uk|$AdKa}v^sSFK0O`U#A~yXP?0G^eUT@gVM!s;(A612+|+tRg+s%W0yNQ8;*h+Jb;zt= zPuWKqGVU>2tf%JnKsD<-5HY-4WQ&#NyrrOdSgAnE3sO>~b4m9oq~RWYs3b9pRT+xm zm{~Z4WWFlgYWmPN%o_4#jV4aycArk=TCPqqw0VuTFoL}V4@KPr10D9a9+zrWd7dj7 zzyl}WhU{B&SIoj}T|$yq7P7#kMQ2`<^DX@Zc!Xbp0nWjliqoz^H<;MmoA-9g1eL5x zrLlKB#fs5HnFmDB0kV&I)ImHPGT{rBtgcHiwU{qS#7l;iD+x@J;hDbO%%N`?a?81~ zn9=~2$hg_439NdvWHD9x^krclC~Xk`1gf|0S%~-?c#6wvm18UdDpL}|{UAu756%OsLBFa| z%avH0lrd12qc6^&OcRygV?bM!^$%)X>RHo(d}pj^rN*nPnuKGkdQL87%UR+sVEN8K zN5JS!j zWI%d*P=O0lg!?L-(-`~XWIsSfgvXSD`qNWi_IFEvWy#eRV$I8T$;ez(wVLV*?yE~y zDOx4Mhw`;%s%riJsz%M-ag(&cDnHi(*D8$>z`^PJbY2CCC(<^M2rRsIC@;>y5|=e0 zLrq3GOa-XS9{`EuUKSf1Pv_@Z9a>diOU~U~0pDuome}llA`yD(_7p9lL7q>fr`1p{ zQPsHyjlos`bd?AK(q*Yl;EdM77K1WWUI9Q$-9sR#W{25CC#4?`<4hlGcGfxg?o42& zHwXQF6NG_f&+sY}+l}lBb9{s=Q&ug1$3eqrn@%!^tit(M9P6lB9>y~{ct{;!IPNUk zLzTN5qm5g*oV$DADv?4%-=yqI!d z)+!dD5^I8*wp{e%tNg-B6Yu5VAEV#or{?|SCcnPVm%WN^F=(dhy`ai6USis`j>b|+ zv4}>rf!Qs8t=5P}<7mKru;5AFJ7cJ`Q0hs=F<;m3B~+FoKj#hlO&7Uqq=ut@!ZSj| zCt30o9#$ym4feD+j~XJeC;)puWw*~D&7YxW7FP(m$4ksuXK7)Hi`Xc_SOLj-zfqi;x-B$rz_dF&T3C5<;2%u_ zGpc5t!{|kBrLw1e^6Q>>N(&z|!H35+i+rFbf0hu&X57yiS=lOdNve`(W$sTC0qZGz zgUI|Y)TH%JYwPC^fz%+k`!9*Sno7B!K9sFePRd=d)=S*aLz5%q>&FS#zlb~iZw?iN z>z`Q868FpWbx648e~5791+hf@1oBc?EV-?!CrB_J(oB5MvpQnDGx|Eyn>E~*~C(U)v8Ay6#5ACJC7>t+!JB1WAO z!U@e*Rrv{r{Hjg9dLzF!HElwIRc=a{s5UXTIK?b|VFYW#O*1WUwzko@U}O9tijdzf z){SiJ0@AtxF=S&RHQgRD@*y8}{r7xnpRZ?;i?u-FDM|^FkkTsya+29w(-6P`yi=Hp z%7P~A{WBtZ7{Hp8lbkg!P~{(BWuo|N5Z$0DA!wPfPSgkV2ppC*R2J)6eqEIx{|&z` zz+}H*vJNxWAhcgFiBrJjF+<#SC^7*W`$t3LnTjB2W5f>_&PkHg3p`6Y~cAUS35A zJTK+-8I-2y#Sb~f_t)XgXoxe#*J2MMxxWV!sDk_{!aZ>|+^hW1gR2z^h2P5GIjlRZ zO}XEpmYd1kAUK0o@-YyrY$429zbEgHsW)HDaz&G7`StaKC{#G>7Ry-WA_H6m(J?kb zC{E-@pa!8&5Hc;){#olB4o$5AIQImCQa3R!A(^M4`AJ#-XgbOIPiX_p14%wrXESry zW+LfN7N+JW!qmhWiZNu09vJXX@ESfk@@e2Fn3j`FOT20~<*VlR+Vvy)y>{;1H>?|4~$s}nj|-{f${G5%}`wB28#wP{aa>8 zDlIGINjN3JeIX9EI8|>&e($NQILQ?EHOiR3YrKo|5PXBNdH5PJD%b4+^QHB2lcC)d zerX`%&X_ej-uepmmGm~h%g;!8IgPsjPrA)b% z!GfP1Y}+a+ZVc(Dui6NY?uW2gR+s{#)~-sp&TYC-Vvbz_jjqWQVJ{VMa%qnhf|I!4 zWV{edtk}^w*dHQN8YvnsG5y4;_?a~XHd$_HsOVsbmlp2H1Eu)iA{KPS0vVzuzZ;?z znJ4)?7>?a7M7 zd61J)m}?(h$TCBSCDkE;~X&5b#pZ$4Wgej{eHj9HOaR{AND4bD){6+ZFiL4rDhv!CFe}!Y~^h(a<9S9#jeptBh(0# zjl+`ib!>c;9x;`6@F!iesu1UJ!8xhd%}t1kA^#l(3U9&5(T^WtWr1*bUQYJsv)PM= zc@~VSHCVBGd0U0_hOfYL)RqB%{uBT0MTPjq9D6!x`3=Z`vNV4e) z%7>6s&*Bt7v?69@WUWOtf{lMc2!uknRwES}T$P`ZZ`G%^UHgPJEaX}p3W=6a>1Z6j z+UM^ibPP3!33~>IqK%H?{$@Uv?MJNyU&`|~1O_A7Hgb4YV5n)?naj>krnujM-^O2hXQ z9m)q;xKE#A`H`g{=<8OYUbPI4@=d;E8+Q65T3nnUFSdIujd=x#N|T!0*c#wgv^Enu z#c9PIZbC!&9%6$?W>~0_lb$$7kRW{ueph4B)DT{YIET=t5`M^16Je7|$sJcpfN!1A zLW(=cx|&_Y=HHm4{q1Ro17MQ7joLD)8lGT&;LTO{f{V9MkDT{HzDYDE=PSHT&f8Vo zst6%r-#i5OD?T$pJ$fQec`(1Ov0++bg`xe2it8C&lyv6u$T3zc0+k7i?Nh1NoNu3E#Sa&Iu1 zI;o-duxE$CujJ>Oc@mh8>GhrKLzi@y8Eioc9sWc*UUxOyUNl9k+ zA^95xGTgBlKg(_e(@H-}$rAUq{_J((3UXiK{Mur*-__FI6N-{_VpCT&CT z#mX_cgrf5#M#HwovQ(49gor1B@@v%PCi!64!An^d*fa^YIHU;~0iro6w+vy)u!xj2 z?`;Wwo)O)o`)-~IO%?*To=Og;TnA5w z{4};|A z%Il~pLI&;I2aZ2(+J!XBN_M;;&LEXn@YO7F%`$h9Kug>Pye$rvns&SLj5Z0PqU>(* z$*25;^xp!GqT6hz8OjZwXTtlD)Wx;F8X=C~qL6bN{zl@F*AUs*AW!)Oi%SkrVcjl7 zenG*?X$+bwOI1pREh${gY6g)CWlf#Mckn|CXd6wyo9_)*rhH z7NSw$jX{^04aTBLOwxqlX#FV|N6F?W>E6ThM~-_QhF`K7YlG z?Ra!2jc|cPn+AuLkN@7Z8aOz+%hpLm5GGrlO8=_DIJG0V7Sf%M7a3s!`e~0q$q_%H z!H)}ZCry$o^p4(+BX+Sw>fNX=kLxRzatd8H(W-)YcHkrmY5*C043tviG2D3T|orp!89><)^V2v+^|uwyiJDN%4VsH zTHFS(p8bOw^b+^^i+;Y`@eW?CK$Yrn&IM%R9yg9a`~o8`b>B-A*^9)W$rqlX$@Q2w zc`+4qxdCKyo$hEQkI{FN&31|F(k?pG2JVCn^WD#x0~_Y`-LT4ogfe5!)`tnQlJQ1i zw7(lz8TL)KUt|VBdCZy)C0P<2fZM<)6mt2a|NJ286I^bB>jnTr21EIn2E)`X5)_Xc z;*M!G6lR@L(7LP!DcILxHTY-E`vCyN$QkubvKK}JDu7H>I3{c)m5dEZ3Hx~ScI$Wu z>01g)osc@nv~lo^C=UYkxV3BDBu(K2_gO&2$HUnE`2mGyP>AeOc6VBe)#4>upIPtl z&f)@rruEWi0!y1~lC;!)PRS*1H?9O%l5MmirOH!vN|v&#Uoxjb_*1Q*C7TI3G1klY znXFxQ&{_{<4s1psp;#3wXQP=#uc36c9;ExV@Cq|X!@C045{66R5kB}7`y`aX9Hq^_ zKTgqTzGP#}Kmr>X`b`J_Q@?~O9D+y`*5a#uxqL;8W6C!AKThn2S)$fzmZ1r4#rw&+ z+H!5(VGw(45VqS;qmv8tqehq4j9fiDTLGzsKyh1N-x$;T|Cxg{c1qw-@Kn|4K&$e8 zei?0gtDHK;sWQzR8+%1T0w|^JT8sk#5iO1`K6Bk19LY3N==@M#Eva;Ef$QSn^u1iZ zU`6aOn;JyQp{WjyV>~AC0)+yZXbmxT!=m-n2vJ)Qt?mm~V?7qb*ACdgU;6FtW5;86Ipg z29|P5M~t)hFcouH7*kg%C!%y8)EW$Yi9noHr9tr4aG;tU@k4iOOf|@DV|>Kqm^kjQ z-^@mA9ib#$p(@wUaJTvz9fW1dgRdFVBYxCd1b8{94|)`EOefedL}8aL$Pu~+07~3F z1^~+Au~ER3^nK}RQh+9FPPMdvlwwApK%{1IKHUq?a=5mJwUEb#wGbG~0BT63#lEn_ zCUe6632Sh0m4Dk;4I#* zO0@)?!9uiJaD%*q@zD`>&4-TQO%sULyGOCg8NnZ8Gis6voaYxIjjGU+{|HS13B8q< zjomD9kE06}m`|-+iP^@kBY^*rdP23E6{`t14QmMxnv%d~Vd4dc#q(X%@(}pfPN#MN z0Vop^MQl3o%1!VzyUtIzKm=s%0hbna|q?%21 zFylGTs4sU7+2m9rm&*W5JEEvcT$m@-x5le z=&~ui-eCK|7^ot>x!5vXE}WS~K()P~pC^+N*!8WzRH?STS;clF(klgfl$WVL=OFHsy;=-}H z$_y<6%F3kv`@_W+SAxY!=3OCkQ@F$~9P>qllS3Q=HFk1n5LmYI$`sL@I#wj1s#yP` zV{yJoS!IN^Ls<5g;fPSntx$zDB zI?yL0RX?fV+QJFS=M0Om)W&rSlA2&yG#Nh$;~VO$*qqu* zt+QB44CAvA@jRZYSz$;vRbkTsL$XO_NcIp%EoF)hg#(KM=(ULVjIk<3pyFDmgdI{u zOtRpRmeiVUH>CkWI1tIQX90)Onv^ts)rJ0O2go>3Ox+x({_FY;{op3Hyv+vEapS`y z5nkXsGXaXUYEZX<4c7B~Az zvHu;gZ}b(L?BTN-jr8*<^COyWRcfw+$NS7{5Q`0-KT zWdooO%eTL3t6!-wE{bHfZC)RhhFl=yjFhU6~8B*dl27k|1zm=DA_KQXua;W>Ts=TK` zEvQQJx6nH(xMwTDYI?@eQB=;&=Fsh}+d=`mGK`f}`i%Nd>-HwypRMjIBfa5BpZcUK z7&e2aP>#{({3`0>7l7(QE6Vn(RJ&@W^G7%NRcxFCzAD==oZ19nvKzK&o5E1GUuCTT zT)Hwl*SIMzPJUz@R*6SBo%CE*rE+CvzugGrRm;Azjc$pC!QpxY^zNfrj_N% zLRm>co^N2Pj`*!j>c`ei8(FuZ3&4>A4JxemD6)ZvRCs7qF3-}|7uu?+*XaGitpR=u zABeBp{0jQSSevMmVg>7OHUbtSNm@gig}R!0iZuG$Hq&5EqLmhJYqH~-7tqCxY?0gO zZ{3*l598#Nt@JQRDHy8)EyoKz!?lneePuIG`a;YRWg-V^yfMdR@h4QmsUAxuQAzV{A;G{lr zmT)7WqD%v`2{cfH8(~k?IS7kfiWnT8$jVWm@rPcomLrW5*;j7?h{JgpRpjyHS--l0 z2f*e5?pVdIrYDSsb8A%1{%n6^ZtGU-EuGsUAkrfMYR%M#dn(|qFY|7WC);R@R}W;j zY;3Bx-ZvF0V{X(jryo#E3NLaz=<0eu*9M-(QA|jD=nbt3gMb+Q zvNjB)#Tqt_K7$0^Uepu{30<_MrC4u^&C4dkp*DRR6;hX-vO#{O+yE?o*Qj5h+;lr; z$9Bg+W$`oUeUa)Z-lDNUewCF~Je#cXDyL?v#xP8tqsj5(@{_rAFUOZmF-5{oyITvx z;CMztEVlaPgyX38<=gzNx@D8(tzXs^`U&9@Epmkax`BEJS#jQ zJl~%57M`%Nvq?K9r3gs()j4-+_Ug2h29)lg1BOx<%*yN!N=>W_vcpiBamd@DpyVjv#fvsaIy0G7hwhK7By=`^r?8uyJYbuVHU zXM-%J8hXN57ctHS07<7V&9LPoLRqHhMqp%{=OaKgEZbN*UJf##sKJ-l{}{vzqOdLNgV9TPX5*U5F?T7n@cAi>%O|cEyBQG~E>g1aMFwwl@Mk18}lO zKvha7$I1zcL4(I%b`R1Sx?@A>vG}@m=sNGToZkda)iAm9Yy%nxtSmC(DkBg`J^DfD zmRixS=zHvnU8ZYeyi*t!r6gP54zpE@TQ&&mSNjZK74m3MMTbXrl{v-ITLd!2mc}%c zVuUq&do%ytSD|1Z3OjLZ>xsg8Mj3soJrf&M2Itt0j)7>C6~mJ7Kb3BXXRxsQC4S3n zkN#j8WrM+rB4wSopSX?kP)V>?VIU&pcrE}sBbj8%yc}K0!+wemQxkv2fK`yIPFr!X zUbA~Bvf9$HLc8LujMK&*$Jg}4Z0TEvD3SA-SawMA&4zWp;x(ys6WwIu>ML9a@m5=c zJ{%LH?Y`oeCg4S+tM+OxWve;73uEES#x_L!fx!>?Ac$rPdn0QIppYwv!+`iKfNrRR zXDmUo(sr0xJ1QjL+x1P-kI;>za3>{As(pziqILL3FV7t@j62uiVxd|o{i(Hk+nX_Gu^SMVt zd<_E+`5B8WDbfOKV2vSSyTE3Ic(sHDzzuI1fqM&59A}zLcj+u^oD`;q;LVUUF`0)Q zWs*gRyb2wJ_(LJOL?1LhA0A;#mi$fvE}U~H2ZU&=LYBlFsmrxQ+30X>r3#m2A+NzN zqYwl|^lDBa)f6NYcLBeAVW>fyYPqjy3>#~iqnX4WPT!hi1!pQ~qm~qYaP>W;hdL$F zdy2&$uqBCSCgrZ^G;DkfI)`O3iq>Jj7#~7Eyh$-22u=>^;O?L?tN2hMARf+mmj9fs z0c^b!(h_oI>9=fATkvPwhP^_;yW8TD}8G=etAF-`I17;&tlgZ>+B<**AV)M zejzEO^CxTkJU-;K1KI zs1(Up#fX<6(5SVJl>KHRNSCe=i|9!Mi@$*$Pa{_rOYXd zvG{Fmknt}$potQNi{xt)ny!Ux*GMOt3N28XY-cT1R63B7C`6v15Y?p)Gtu&y*6D&a z@1`oi6Pp(vik75sdO8OkQig-*=DF^o73yjFu8$rYI)!4Ez*i)T2FJd3MfEczekYZS z?tU{8*E76BhXn5yaUi3x4l=s-h9|h{<-5XvBrZ;z%_La!z4(5o6j&JUynA^hRn6FV zEa?mSz2yZ-T6EZ?AUtmsf^qSrEcS46lUGW>SQ#~&5pcO8-_w89Wq;fufX3D1V zsOnR?;L5O6xHnkT2Pt{9poWH|xkufQiJ1+V<3mZr{ygHeKrH89l%2?|Ispbjyx$rFzX8!JE0FX%zvK95Eb7aid&4I3YzjMI% zs$OEIuz!i8o0>}ZD4HqWUb~3Wkf4a}g)auGLz#di*;RQy5{WKxFMiRYbk>oflFASX zk4i(0QcTebjDasAC0;P<3=RK?`xD*+mHRjVAVy{4$3f+Vg8_p@GqvG1UU|9}hhKwr z@r8#z(e|VdKtg!C#?4`#HvFW1b3y+*JTz1Bvw-IfDndtf2_Kzvs3S4D+ZLt5y!qnv zJqD{q1Q5tz<1G~RkyBM-Q>BPUW{l(Lq_m1m%e}vha*t|e3PE9xBvv&?qDeF>&%vE+ z0ztZ5^0$RBLh^8^!JyB1sEnXf~AA>{vREKDNNtl=~GP zue4S*pVn^5{m~aeHf2`o2MtmeO?)V+mne)c3O*R;#aGu#@>Y(3U$+O21kmmWr8U4F2;5P?a9{hs%9jfJJU9`+OewN;J;kR8-C{5{QG0c{BBfaFui)OW|jLkqH@-t4db<^?8t>G z>zBP&k=i8XMjE_l@5?NrflSln6lkGpwVze5GqUbRlSoWI*a4be0dACTPFQy%D;`dF9r%&!f3n*Pz4=P$JumXURiMM5Cg@H_*vdcCq000)2L2D zg#SM7g94@}9s2o<9nLc1VRn1v3>(V=u=wrt?J z!v1WZ`9K_YmT`AJXNQ_&w<;1#txJ&Tm{Kd08VM*K@%cg{fUh1Bpca!3fch#%-c{*gK>bjx zd!LXTdixnR&&6G1I>`VBJ%LzEC!%r;?$1t^DeUMC{S3z~6eyQRZ>pc}w0g-sg}A4O z!}r6z8)F*LU(7^nw{sVm(XU-4@?}|b93YmBFw9b1hEvD^X-))4AnLCjPOOdw8b{>^g0O3z6wH+XTV!&q@@GaEYkL4n|&}( zoT~3uvj|q!8qxZ2E&PSx$RY|tqs+Z#>_gc)N6ovtt=X~IGu~<2q&nj8sdxe{P7N5W z52O7A)*n`%nmx)N<{!sVf!6K~H5}9gn?OcOE}-^qyM#g*-qnLe$g}WsUFtVlw^|)m zzwPs4iA3gq`R7G3cJ{XIYU>{BfRQmz6*>5v`(rpehNgC|FW%xatj{jyc%pj0c(dYc zwcu)xNGTwlm_mr@7W-`1Vm{FWLkgb#GIs?zh}gV^iZ%h_4N3co>m8su?gSRckc?mE z+pbdTmSFjf|CXQ7KgdV*q$E>YYzd3QXDxq%HXm~~JUdbGsNgom z{~}pL{2?Qpf%~6ZqUiNWPf4j__Zzg#2cv!QB%0Bue|;4*yAaKMaih-3U${v(_%86o znMSWUj`zXdUNP!MYBH@b*ss`NzY-{2Cbj2GNBLp+ZqgE~9;Qv*MGGp)!Y9k4iJY*A zdsX|UEUzp+wSsQl7oZlOJ7V%z_ET}{tnBQfB@>Hoo`|L5bo?9c#5EPhu?siFwZhdh zPJgp%IGihllxN(ZGI8|wr9)Y2LqQNjnl0xJA2Vi`LdN|jm8;W)HR;SnZr$$!YZ>>* z1jvn}osy}X*4l1&UjtF7?d^mg$MpLW_j4$!H&*9= z8CmB~jjZ!RT<6Krb+X_uuxoD%EnRy!?srC3_^(G+_?K~oKYLvj29^H3f=d50dMSN} z^s2C$=y()s_XZId7f~iZNwG0i`q&N#^)X97O~LDPL~+0!Q;JG2`@1B(nMrr6C*bqn z@JNgn&bn5YFgO!LlpMS23uqTVpUPlie-U!%d8+GqObzR`P=iuwl>8M=%jHiVIgzv9YSCD47Z_@ z9*Napm+Pr^V2F8vRWYmS~p5P**74Zo(G)XpaG}M&h!D^FU#a7`9V}SgmKTUmdVR7Eh3PMIoM! zpQ)I2rRgk>uo_?TbWR5~VR95J+#Tf`-2#R9X{rc87X6x&RHXzf2JqfszU!MJLW=3H zvC4(^q}M(P2I;&Yq=bE-ZhG~oXZV-4+XR-{2Pv!J7JQLH-_~O3w}b%O>$ol5;ET0| z+X}M^BtEMn79{Q#sGEWH>Tsy0u^!B0zDJjp#0)QSmK8iHIlq8MxKr&7!M4EP?l12LF8C^xOi zuK@@g>^D@8pGGJgsp*u5fC)d=+u<;gxh#o6wP7!ZI5(p+!d@MUU^*=oAy(=&T4KK} zRRApN*)p;h_cVlAP#@ONsYb0=7e+!4G*u(IAiHj81ci6e)vxLgeUnBC`mw|SM__v$ z)G!fM#Fm2T4V16--|yK&9#O%W5D0Z)IY#2%J2g+M88{L1MGjB8%TYk zD19ywr$)baGDe(&%<&cvQ91hA-azD#!HozB|0!*6#`1(rfN80Y)q?Aa68H=P;YsX# zFO}z;uRoSF4d+4Bj9hr31?xE)QGm0u0=5?(2m@y5dn40eyjPVhEaj_A%+iOm!Y3Wz z^21q17=c3Ii?UVq=5(Dp!7O-KELl^kCH0I>n#H2_8i|xjqaFPS0=a^|%XkS3dz@`; z>_KMx@3vUlOxi5&W||08T#1DB$m@zf4~__RZt>=ili)o`8)|JcC~Or9`))|Ova1F6Db)U*dA{r;B(<69#Ehqa) z?i84IU~gkE?VvA@@!)!ywnj^^77iCwVZ#qoVe^GE^+YS7kx^f)@H=d&k8^gw@lv-- z=z{QLwj^|!MIDAcLcmVhO4cLUYlLq08lfu@jm!n*U!^<$FNxfXHmTdCgp|AKfmLmn zvZ<7kpu*>D3=Bcp@(srC+$bKvJvl8yhlFEh2C*#_k>edk@TSRViPA$x=80bccK}{s zOC$JIn1jNDe{p^_v&x{cP|SX&dpJOFq2L*(03a@lA!a|5W#9n7h0yQ~n56z8z6ls_ zDz4ma?zrY3aZ~Z0wmn_l2d>*2_mw{PpHcebun&ELh$v~ILXU*S-^Sn#ka5Hpb#zV{ zTOT*aD?lu}kguqkU=%WDr7h9}CZAq7egv3Q!Xj!wQB)PmY#gv+gLR&ONyjHN6T z6pm?B(2guB6@cSRXB%%=UW+55cqIFo02=c+$CE)HLuQCyjbq0td#AFg-Oj>eJW5iz zeePb}$BAbXaga%HH_J7{^w<)PLI#M8larD!8{L=JVgV}8GD%YTb>!BM@S%OquB;{K zn)TX{%w=1+#F_d|1DT2zAAT0B`ZyN@jEA16K3%}7eoh7?FhhORR#HVQd_cR;jCL#u zM8pI6O?khK@9Dr;!y#Q7vf_n>wefBg8~V`R6obc5DqgqD~&t_7)ni;`Uc z+eg{5dgG863fs1ep?%r$*55cY-uxc79Y4FOb`*T$KVLCpqr} zu0vyuQ=>^$m`)L!#TYoi!E%QvbQzJunZq$On7;x_BJsR)HZ3Ks61VA}KkqWbv_sPrZB)QvqjW4qn|Z&B{)vZQ8Uosn zH0vy-45=K8FY73W#T<~r?`1!m2)FNR3wiMeX3s4KhPJdA_6Gz#1Rx@7d`h3-(qpJk zK2o{CN@c-Wv?^3u=-jk;*5b(k_ofJr_#y6;j$P{R4Jyc;Ser)z z;5W9{?vLwTr*xTP{_vlq%GH>=Q`0-g-G_yESQ{^he`i<3oB=ysbY{7E6cCxk)>(@L zC~PsK>W~?Oy&~-QL+4QT#t@~wDLgz}I74QQo*fCW$m<^0fYrjM33UZ<&jJFzh#qx`sWTe}r9wIG`k8gy$&_QZZMRd$`8NO(~S+{^&{20te zu;}Z`wE<&7>WI{r)Dih5ckBD1#je?Mwia^i)7_82(yIgA%C+aeP36`2vAjOV$fS=8 z4(~u71qNJDcH?2Pb1&B@zfz-3>29it7YtIaQu9^fIz@+>aO@SSK!GF3GZFQ|jloNoE*W}! zU#ql3@tSyxkH%qq{}_ky5g|gsB?^52X*|!xy)BK8?c+oHldSQ>t2lIWtnrPA?*Hh4 z#yW4Th9GsW8fcwVaL(RIxt#*-Fc=wZ;o}%u%!$A5^5e1YddYlK2h}wsaL#VL=}ARg)ENzFK)Odm$g9h zGVa+h`0IxC4EJ|&x4v@Fz669c{4N1I82=UV2ZXB?9K{A{K>Q4_c(987^nB!VNT%Bk zh84*?TPkVR5hhk6$J2KK1~SWzq@9#`T2R!MRINqAeiR;wD47{J@==?aQumL^_=LTq z1~bdhun34f2(}egere^OY)(tuQNk!jFzk>0utb_817D85}?J|J7MG}rMiplqBKOV%63w9JiQ8dz8w z)Z{;Z>Y_MrvHLf+I$xp>3}2lF0|?B0sR&}V35uNuLjep#Q~Y0w9MHq7xAm9{f~cNC zfb<_z@VaEWC&=eT9C!Amo-jk4;uIrF!O9plL-HFTnc^>sMSo^q#?OAFmvPy$<*j@7 zE$waV>Frs*yQRCct*0kZxF&iI^aK~|e z?^v)&x()?rgX_~zBv zTFxa+Ypli5*GZ|Pp~)Yn5clXYx_ z)|W>Y+QHJ&2oA4(O`2%jZ7>s6eJSA6;TT1|KbkZ0yqI$L458=}p{QSC+9$UZrR5hZVMdtzoP|Q_=-?oaVVG6=LQz=C92~TpQbJP$+9`V& z=YEWSmk9=Zp4*5C+_Pjn?+vlUDa=Fw_VBzPtJG22=pN%ZV(?~Jp$+xE7# z^!6k=R<<{Hw0E{8+MZ}_-q+LC-MqV{b4N#8ccN!^S9fn+duMMV)Jb!XQhY}!zPn|A zTeCRlj<)@Y-tLw?dz*K)b+&c4bi@s`w|4E*yDgpVtqE8tJ)^Fj@Yjx=iH=oqqut$I z^rSU@r?-7iTl3!5-oz8U(A)MzZz66X3@ZV*Ry8{k``cQ3ySgpi-L}7ZPkSdRW6jOG zy1Vx6?Mbw^wzTeUYX%0*-7THF+7jx%2Dl^c?GwEn+nd`ucZ3Jq573g;_h>tLJFb4a zqpNdQo&5+PVZ5rLQ1bEZaWhB_`e)GTSY<%%ZtK~{u=aHAXw$g&l5V4p?zWa4&E5Mt zn;GiP_FY!AwWF=2+wR&sdwN?sfTgB)&vvdm_P6ZV!H`=RTx$yh2_2_US8HNlXHWaC z&bA$Ot%$D#-@;3Uf}^ zd-wHNH`@2?HJl0X9WE#!^cuY5;cwjh$-4ViuB=|ZkkkuMWbUe|za%m4gB;y)*uZPkA zYw51KrCX>9BwoL(b04iQ-O1Cv?Yz8mXLlRe4J~kdame4kuf3zUy)z_u)cWXBD5KX< zv5?u_-rBpgrK7#2XXv#ZZ5?gBZA&}%b#yG<+Xd*`x|f2Qt&h^}on3p{dzbEneA)`n z$lSSisqnwMrL}i`*Up{f09$kunrmxq?`iMqTyH&N$cg2TFE^+r9&Z7uV&x@db+m6s zfeA2g-Mb>59Gjx%-A^{}X=~}(2N+`V7VQ}VXu5Z{cPHqSVP0Ej>u%_%y*I(Qc6EY$ zRNURt+ZI_=xp8| zGmHQyOBBk4T_1^^9WA?hATb~iUWUfwI~rjn8tUwU)iAMzd?*lTbqrMwObd@%;RCHu zf6u-S2tWki+}rYKTW10Q?P@bNy{oHZM_VVf2NaBT?$axvMDyN#y$ldUZhm6riWMuE z+@9V9pST5|c5sEeL!OT_N1<#Fa|^;XcXuHVf_}SPr2vCU?CjXrvpdn-1*eg82xa%c zH<1U;+u{PvIf#C)*(cJa~ zAQXCag71a)c6asm?By?R9)``h1yrLE+7>s{3yHQ4cpuddbv58aFQ^3`sIlE`Puy~I zLf{)}hVBJ$BK?dlg}Q9`?N3ns>lm3jo`(kAd~HwuX6E zpL<(E#twRF4eV&|Y1!V<)=ViA?*ZzzgJ-i%9MReQUwzC~6)lCNwyPZ3r!M&D(X-M96H+MW~NZQ;1hVNIZ zr(0y+vg1Hv_s-@Zy*i#`<}@B2^|oxMT#Gaifsh;nVf-k5Q)u5zmyx2)w9N~<_wC%d zr=>H|Nvog|>>Y0Scw76f-ISqs07U&k8KC&1ZJ2r;Jxl@FOaSeNj_mC2+LP#Mf08QD zgW@WA+S;{8Vq3X%ME!acnI7{ahJmbjOv4G|(g-!9C}-`x+xMaEJSnNaADTh6Ag@)N zdZ&S8N84Voj{@OUZg#golii)WnmeWcP|P|W8r%nVcMkz4?jR%vq3(}+*3s34g3z(I zxkW$Q^|NDFTTcgE4B5!rkMC}SVuaq^fN+1$W3XlAHSb+vg?Drn(yJ3Yy7qxv37guU z#P;S9g{^vO-f1`OJ6iU%JVA-JCy@7af&QRXTF>Y~``-P%JZwRn@?c~S5CNS%ty05w zqPj>sZQj|ky@yibJn%*U1_XMB<_vDYjP2P28B%EYb#PG#s3AaB=%!trtq2X^V8dee zU@DfeLpAFOd=r&548C_|d@aoFPG)(B#e|=a3{#D((w!;_D#(SBB)}H28RMg()wr&)+1I<0XZ6OmLNAtpTUfT^Jb?rjj z!y!?w0o^_!3Y8{y_gF&u(q1Ieqlp7h5G}wRAbr&z2uJ$u{=}ZHUUZXYK#h3q6@pQA z)q&RSt!fYUwO=!{+V1!4?dm~dCb~llkL+#R)%@6+6^T}e-#|(8@ttPC*Gv{i&j2;*TMhX)pT7)5%K zLhTIbHjQ+je)j7}I@S;rV{8hS=I!mWDKSL~takW2AT}aU2q1)}9%yTA-q_K#y?IsQ zp6>4ERn1!=`v;h%2YW}RwvxJ!o+s|x9hZ3sGTyPf`+%&d9@H(kV{;%CRr!endohtf zXT8vbjil;UuSne6)zx8XHMY`PPTb$#1By2OkIK#lIIgR@<8ME-Sxd^YC<-{S122P1 zL>r9kI7HACma?RxqlN|4R;4bIY}t`Pi5FQ8A{fw=k~~O&)rUU{!Ct7_IFuF-_0T}+ zRJ5kJly*#FAb^QQ$8^HS6hYmoJCm0F&OPV8_jYYa`zEV>|2^lPd+xdCe!Sy%ZrCY# zMpu(`_ukQ6Q_N=L)4ODUErM&hhj;9vcQAo{r)A3t*XSnqceDF@ljGUs8YIfPs$CJ$ zWs|GyRd=zFNQ;F*0FTfzibQMW5ViE~c2PWA*G>AhMZa#=uNUjrE&4U!6u!wB?3Ls7U9w>v z+A+h1TOELn{@!{Zmcehtn!#(>!QQrO=gixjO9dNgXdD6Ief$QDU59awT;W4ZSrS3a`J zxVG;`Ptl7VhN_DucC#5lj?NU!W{%yW5^SoJ#!>YShLN>_0!Y$|)4O&u zXC#VEkM6z8Jz}%wJ55w|WoM4#N2@gEbDI$N^bjx(84m7Y4o25RC)=sJw?4rNBU;5p z#Q?H7<;!<~?C2+XMDvf`)4IFY?S6gdUF@Q#X2#<=UH^f^3XSR*o(=@{ST>+{eee3wtn`wdsahOKg$c#ef2j;eM_n_;l7_Zlzog_3; zcQYnkBXoPLUU{9v@+S&2@-@JxTW-K|pT>1?FCR~jf*^Au!MTdv z3lHUUHNK{@L#6W25&nrNF2}29K`@fw{Mt4w^$2}EALQguQBVJf6T`n1b(Vs2T|)7C zbX?@VoZv#S4U7CCJ~#3~USTo_@_p4(KWfeA8--lvA~H{ZGwUe(>P&4_L+9}f) zg`Ew=NS)Y0aF^IY^flbQ!o9TJ@Rj}CR|H>1-fR?JS%-F>Plz0FNo4J-mcrUXe!kEs z9?51WMZZVS@%t9zd7STy)zaATJoLrH_i+AT;XrYJ=Ti4#X`wvdlj|L{ z>dt)iYhQoD_sOMg%x;asXO1Cvh7UGz82tn3Cy!US-;m%Qp-KN6y$Ah+;S`?ZK22|e z%X~tW-*cS%2WaPDBai)@Va`?66jDxfxT*qArD{Agm{<(VMO}_Q$9;sKR2!W`V&);S zv-QPet-QUW@6++pK9SqofcLD(J(5Me?3|d9E*tV1n0{B+;hV(owGEZi);;1&BIjys z*fCKpH8Nw3@JP>r-u=&7TDy4d!ut8=aDZUaH|M%?S@rNoAET?Zqt>d7b;t?^utBwv zw^4ejpN5Q=Fbc0jpX{vor5`t^T_sMS@jUlQJ&NQ7;-KQl^S6# z`=oO?${6i=BlzBW(dS4um+57wL&I)TZo@41lgO?%is&lhv%|4)Xp9ZVw-N-owbfE> zaH(@>vHNg#>%shi!v5kyX};X($z|6kruq64;u0qRhKtnAypZUg%aqgd@kNpUf)M?O zC`UgD{w3+Bn)jn@)puh|Uxxma+8EPi&wB=I^Nr9mb>HSmZL`rf_CpIjZEcG^UQ~mI zJ=8yPnOoJ5#NO(h)~^0XsQ)PbHaK6DZegleV5ve`x{xWMuOfy>(W zveVo@p#3K%cU1aKWW0O~|2a8Zs!dA6q*?Q&M)@EO&E<}!>aBKVo7!12(GHbrjq-eH zp?k4&skpyzAb-%UtyW)E^fF(Pnab6NpGtkzL(m3&|L%+3#M`F20{tYPG39^NF}msy z<1~}4&X==;i{06w!&x0mOPyk)Tu0hIPgJ{dnKOx|Pe{>5&wP2IwAj7WxxaXza4>(^ zY3s(0=yZF9`YtzitUb3kptrZj`tmi9GT74VU*kSAD7bA6>ZctmQTW;>^!h}pGATAr z+dP-qkf@CSF6xjnuP;;PRxoxIJS=rEUxtH=+2Mu!kgU(~Q3LV|04R69mIp(;rsUz9; z4UHJ6#h-PhbnAnZy|aOA=8Aqd#Vr)(i?ZkXgsLLe47HBkBiyqtht>J~AUh{prPSWo zNM!9^hG)j+g((}SR=$p|8TMVtIwZ@a?f;DDE$~SBq>c$;X{qBa$Kh$8d#wKH3bYZc zzn07sqTBQ#Xj}a}k%emOi7$H*`X{XH(blqir2h{o4Hscbw6f+CGZpC{89#mx;kIV% zy_x;J{??a!ANl!u+>Jh$*wD(}O<&1=O!g77#ftR^)&6glvP)_EF**DCJ&)M$LZYko zf|}U-+pM(ra`hlzZ27i+tw;38K47-;v{TU9`ykYE_wfC6K6Fn6&w1;Iv#mb#_BymV zbZ5h1`!Kw$)ra1``t&^e{tt5YiJV4DU9!07^Gc}gdB@z(;jun@XBi%JF3X_FXJ_I* zYoE`VZ%W6b@SP3#&+@4#eiypt#;VM>mzy`U$5Kn`6J0G+Au{L_k@@?lnM18VbK*AA z-RLK@_l$7AMf8(r3Km!YykPpg2YoX6MAp5BxbM^c8ME&3_1wD*?K10{)N@}ya;Q`0 zM!``HjDE6D75v^wFza$SyJ&qIqRnZEbSeZSXN<_Gg#YwP{g^RslgzX|OOAJgH1 zS?Vl6FvU?{V-8vDt$ac*=Y{uC?JY!&d);b@0>GZ@eGD?oZ2$?}#2BS%!C( zb`8!K2%9j06vaJk5`t#^ba;UKCT-$*qh)eT5TWz6nJCb2tf zWMz!`6^l1Y`}0eM#bPcqkhXeUZ}Sv#KNs8bxVRR-xbYCDAySR0tX})VZf2*v)V%~ z|0jc#N4IcDMwL9>%Cv3!kUryCfrtG<|KMU_WTAM3&7f=q;~Cf)!s>gd0WWj7wRJdd zEBzz-KD3HHj5F8I(#s1t`jqqFN6&S|Z+%_vB@ZIdylcgK8pKKUg$=5@^om0LEk{Li zXK_(^XWl)!MB#E%eh2fP(9cT$2*?|*8Sap8e?c$!d-(=m(T1^gazS5{AjjX?hAo_> zzrs%+Cl~n21iA7OYjJIo-}Avwf6Uh!+B++lOK4X`;rd~Rw7u8bz2|>n6n>k24e~Hc z`)+BYi$A!_40IP5%sRdTIGgkB_Tq`ONw#{Lw`?tbFGAQMf98}q{+j6>qfp~!#!r7~e9Z8FMWM#Q6lXK^pVZ&f{?Eo^Qu{Ca zY#;wJ{_A{<%P{@7Mao-yvo8YE9~o;;mKext8M@gk`wB2~U5x3URp!PR4}rrJel2)y z3ja2^H-$%|P-9}I&#oxEWtsZT-r1Qb)R>sj-{oUshTrdFU4{>WPb2>U!=LnVEW;lq z&eR9JZurl9+{y4~z$?&C8veT|)HsvjZ}^yy;qQVQDS1DRLXGkD=G zSG2!!@f@rD)p4zl11W#*8tiopezSk>dhmlOO#iJOO5q#9O9}3HqmL1pyv@*$CiI;2 zujudPGyYZ|NN~q5`ivO^W0~!4x=(7nu|D`C@IFQltkJXuk zo_hp5mBOC~OMjVtI$lHhW&Cr>$}_%KClY$@G4NP|^UOCIJ2L)nLf@LubBzC0X^*%6 z_rSv`yc&fXYtr(0*7Ma`LeF0T9!lX7crb-u7lj&gvhuyqZ%xs~zXnqHIp8fR{0rbp z3NycJjLPI~f!<8fU*_XbM*lVF{V95l*pjq&!su`Cu_nV4QK<1H!{Q&+RC)YEV@pP7 zKF~Ol;fVN91^yAk^FCf=_#@zp&_7}L;V9IYkYUzajR6_{YwDj&)qg4qHSS~dGf}8< z9>Xi(2K

wvpHe==P#cXtfcjPzJBiBEQu$ zI_pa&N+%7Zo-M)xTc-l@^JK=^T8JWBPyCLio@T3Ltfe*Kv2}Z829Ov94AWXEnQ+#) z336GkB4R-6MZi2|Fy7X;1GjSH4<0*%!z|1w^szniPR09LKM6bnuc_-vd;q*nyHu>H z4Y8(8M43BDECXgJgLAfd4|s!7r)s^C`Vd@dB&iRxhineiSLSIFp9AkI^AfK}!mQgB zDf2oBEVtbRW&TcL53mclq*6cW77m{GspKoLj{pxSgJjzgO543aCJhs4_c8FkCf>BB zj0v51Nq4=671ex!v7{lVc0U6@>Mn@1niT5AjcLfeJyEHB3cRRY^J8F>j{a1~wP}yJ zw1*q}7aidrf9GxLOkT&|>911zI5QGf*I*W=ydzuZ@acoI{aZP*lTt9pN!fZLF+D%q zzmX$5B;>y0;2gZ9I5-DylkMNgk?r5eQ5?J-y4;r|P$mEMIKy+4_7B5{7bxu?h7T`N+J8OH@OnB<_HHO(W=!>&>kqn_Ks(8cgKOK}u z9yZ5_86KCJ>{l#PHEo8TYsEI>7{!R*YyS-JWN#{b2BB~=bF?lRwK-UF5W^j9+H)-n zLLZnH0z#)~XwR|2;XWKmm=}uG123AbSsCq&%~~R+1C2u)D4})wNweRzmeGew#76Rj z*8Ho)t1KHQVdJl0B9qWx@&rCU8v^q~8cWoJnUX#jN;))1W2t&5z;p!DHjQP%T>)kQ zm_BK|@8oWU__5@s0Atg%W085f9lv#7=bCsj5LV`05^&Nn6A{-~RD&Zz zz2vZ7F0Cp(0#J>$p~)Fktk;N27*oe0WsqpcCV-w+tFe}9^$2PmS&((4gugr}kFJb% z;;wiqbT>y>Xe6B=U3Ug`x&vL+vO6txJ5|TCfkE+RJ8aJ*&3!aW)~4S!8%L#!)lB<* z+iVUPLR}$I?*s-gmD*=vYT@(>8JSLuRYWwlMFXoBVOJf$RZ(@c!iFDD<|E_Ca&x;> zhpG1&!mm9qPJTqDeml~T+B+@>7D??poFR1F4(2wlnD&_jg$I{cJ$b-WL_TLd>cJL*Vm?|587MVozlNvJ8$nw!x;jYX1(RzQDU@?xfW}y@iwaHaU6MLDK&ndY%WZ?s){&<7J5iQ zUBHCX!%=Gl>21J%W|=LquN6z3BDRQjUV1lpOD}^j)-s)nl{uHhg}_MsB%MUcP+>0u z!$jU`x-wHpu-xfX32W(WhI@9?f;nZB8IOaf{0cVHucY`8a6p+#5)T6R`?bK_Ozvqw zCt>NkNW2WZsEiu1=IHdcx~EafY+)Y)^AqqL;*mU3Dx_pZF(aT*8o>)&~(;oxE=J3<_ZR#RvhrUk1D$H!t(k6nO`e2fy7h56UxjY z@dog^dc84m0KF)(uL=2|10e9(&R?_a-)>f?1s$J)qV~G#xO13{4v5roXJSoft+mu~ z=W_r(ai4O#?YK#5``*E_9W=xkfz0?e~ zbV)vwMex(BG`;eyXoMXF8Cv@4ZP&rth4>{L1ZOmq_*kF|K4if)A7XI-m{6S zTK0OvKI5TP?|7@Dri4~ks+C+$z0#^A-m0WajDeD}fPgOMGKs6(Mh3w|?h@G+zCaOe z3p-B6oce}oVMVGJA4Rf^X_#pSkeY8yRwm0CJjU3IAepVT63HP5X(iJ6H^8s763GoP z1eVTkr%i6}nY;k?GpPTjacXAr9jXh0>r$Au0QnXO^#R$gK!-f~ypsN2lMmL|ZEEAi z8S|2CnxgBeGa48n!zI9V!ScIk9~Ie2Vg<0!POhDY_%momB~xWm1z|fryLKMq%J$f4 zG#pJRf}AGajHfB8QQ@WctSU|@EqA4Fo!M}`EBeM3nN9IlU5-$MZZhjo6soX{T*Lnh z*~FTjRhmw^995Orbb`U9*ZxA9Vao0QEx$uFOJy$NT=5H3$)&AP`HQ?ST98DiU5M>n zc0y)ADo4ph>SYNfqO$$7fJ!Bp1)`FrThlcu_kRSGmwKi7tU1^e+*p%xm%D(2%DhYB z3E(k=pUQ*u@nse*g}XjJDR+69iqFwXlX91T1OKu$P>hyYT`+(yKdL*p?7!u7F6-{L zPQ`p(!I~>cT;?_#NbLR2?8Y?haF=)R_Gm1Ow`)3(n!miGw}&xW47Iw}&F2GbdkP$; zJ(#xn%S*gHSgx&uU3cx5zuem+35v`mz^E|(x~HKq;VVcrQYwFX2X8N(>Fowx?+4a~ zag}}6X>|O%PmGj@iCp`v%j93??Za-hSnmEc@YNthzsl@agW;angVkyGAM^EXu>)=h zJvxJKN2d0tmC=4EpIH1a^8wiG!=!8fK=Cx-5~P@-$2IEI(7zo0tBX+eZt@;$fK?qt z17&1tRb|`V3VO4ouUkgz(4xm7;D9oStH%Stua&_Ad;AH|d5!4xfmcDW1`Uz(!V@*A)IKdRT|Ccu6t zZS}>B5P8-uRW*H;qwd)>x(&F6y3V6>yR;-zlBC0^^maEQusNyNq`48(HQQ1##v7Yc zp*$61yb%eL!WeJFe6T^a>ARLThf zA*&BP^g)z1WDR3rmTi4y;urX421`F=M%pMw`~0T0KnxoRQExcO$SWi112LnIQ5iO+ zU|>eov}q>f(Gu=(E8M2TR0H95Rp2_IV3_|C!X2hbAK~`t1+)+INa?US zUpLtlV%UAJ0Cf37#@y8|k!!qZ1{Bai-BOCK1+G>HZ_+?U-FFkP5f=GFm63DKa|0Vt z3g%f2tgs6xaSsYQyny>WL|*We8$sXgP|&3vjiB$nz+pO|78pj*mofF#3f7L`eKIGN z(FlG9esm*9JtQn6C}rL#)ybkpFo4w}w4RKQpcMNr%~uCcQlb%*Mu7$}%LEJt0~}yq z1;~F?0k0Z623Xn(XsOJ*Buar^iMBW3xb5o2xb2lrq|QY&knNSO1+G=U+MCJtN;d;Y zk0~EJYHs=lhE;fjHLSjbZf971&m!3@%!;g^q)SSM)en0mjbW8p!)h#Z8GsTr?@1Ro z^;B&c*(pMSeayFJ8&(%REoF`)sNa*iUQfhTtE;UCyEQ&EJ!M|CE`Y(^z;)hsORr+_uRR=#=O2Sb z8)RyA1to4k(Qzc*^nk~jPdbjuvN)8-K`lpR=%h?LQfh&Lqq0FjsWNa>HW!$x3>=kh z1vV)IM`h0eniIuQ*=GPFku+PvJMfzm;Ly)oQ=LPOu{_Yk+x_ zRyvg${yszUBK<=%wTsfJRR2zZE@Dgn{l@`1B#LNjD*gAL4op=eYXJ9~mW<5nDV}b! z2B43rIRj|fTtzZ~RuP+GDc(wvyk{6gXw}B1c->&DJ~o9(w2sA^$mphUEHmg{8kj** z31-kV!3>(J14WtIeQZ6rPVr1rabJ5Zd@ccWalZEXB<=?GF%)Wm(J)>Fo>pc)x1Lba zUpML0W}!Y{Fwk3>t4Z7dR4B87#2wCmv7eL#sDP;{Jr+%&-q zinRO`N4mvypyqjTG_WJkmVT)P295@v2MksQjs`9RE>i}M1|9%*DFa6XPXM+J91RKs z6d(mff%C`Pem|gJx={oegn|Z1yNUpV8Uqd0=LeWUYzjJmlZ2A~?|2v(bT$>sXeBck zgdq-EpjQ8;n#^Dj;djtVyW>DDErZ9t=z8N5_^ey4FepM)c%Az5>wu!B^!V3rMgN`I^??i_1_4&5= zvgtFu@C_nU9-e*KGSYcU%Vs0@g%<@R>$DfiMF<5J7JFg4+b0FdB{7mqe3Em$@KQ_A z8vfa|n7j#l)xY;@Q@Gt(>#jnOKL`f5P~jqLDME6rV09;;d_Q{6@7 zX@Dj|?Y@TI z46G9?@KtATW%TT03Kwa^%tf?oplPkngH(AI__gTUhEMUiAWZS>G*NkxL_46RGOv-C z2Mkx{9nU+j7to#nrv1>_$JJ5Vn(#nv+O!`!?{eT$(Op9WZC%d07tqBjH8_tso3icJ zs_TCkH2N1+{;5XWX`n_5kZhlzk4n?-X8S}{ZJmapwqexdPb4cVql4a35KVi`T!bhZ z+VO3p4x$)VTWlhNK_9Dr+9%;rx(KA<532$;s>e?>E`~qs6re?ihJQZNKfk^*4FCKq zfl10R{E=&cU^_VIGto^C$A>@i0g8`6H$6-Pjb!BSf#={=A9+d*xAr5I_XRt-_A&weH`9&hC>u@5fKO|CnYh`rk7CXZAW^4}R zY9FHNb-*H3Uk?wp@8{uNs*n0XnJ35)3P;TN zye5-4NrPamB848^RFHK&XGq5@qa&i+dvTO|TVBrh?4mNR=j@bTmG9qLKs>e=J^cF%>@{%z z{sNPt#f*P{fk{as?d~tApV}t=uw91?Rr2pI7?Gp2e}BOU-LB;BFBnmzw10mAZ&On5 zx6rN0_u16MWHQ=<)l)L2sGU+IIuw)O0}~{5x3IgvUS(OSTOcxP?2XqrQIE2D9HZ670K7Iu5=$segwH=f^bgr@ zE5A^oTFc4ywYB=%TI~o-8T*WGf)4c+%P)9fGBb_7Z4sJf2k{z%_ZA90(A19idR$HU zW>g&WlInbK^bQ~=p<`eoyap_OGJaT$j}V&qGG6hZhz$vO(O8Fjg_#s;8?RF(%)6ns z@j5$%SrBR$XJIRtteK(qaW=OIGbMC&oX#d;#)PhpD}22$6GB%L>D_QE!DL(7i@ll%+gF7zzJim6tB%g+HTOjm4oy2xQ?OG~(x~+GE`9Oz3&kZf022rVf<0 z3zZU;lKKg(EBOAXS%%Z~9oO(zpcHy3b&-2+g;YrHY@fEk9{yl{l7AzDi@fkhWS9Fhj9U`)c-h8<6>=XFO_0v6Wb_O*>F*I{y=>A^RPfzBu zCUJMnW5E5&(01z;{SeC_0}0CNSH<%T;T?+!56 zfVm0^GX0*&AoIM?b(PUe$(du)Y|0g81wyFA;lxn6tn%(J;5PbeQ2_hOrK!|0wE*SW1Hs2RB1{Hwn2r zRaMsBm33EH`caQE-D9GnQdczU%p!v&GYFg)cSXsq+Vj! z6$Dzrt`GIW*FF z!uZ=jZ7^K94vp;LYP}5VMKd4N1wjX&f%>@ad>Y!Dg;$9QY?l0JXIT>v!}uSCf5t4L z=@==B{wBcNgeGS3WcF}lnmb%?aNXa z%jPH4cSN>;*{BXbGSHsroPEM?LH~v-{2}G6ZlR9EuY=@1mXyy z%zxIm_Vi2Qj$rTfR@rN=?22C~d*7AaV#~sTZ(&LEQAH!)NgfC_rbC=$;VFK{+u{9F zPSO}JmrGo7EwEZf-57(hn8RT9LS_;9jP6P>@W>}}l8ja%=?r>{hVy$^^dg_gN$NT$ zxrF8NB`@I;nr#+wj~2%MM5qC!k>#jgv&|*{1-?`Uiqgr%hRR?h6Wal8&{yUY@(%+0 z#9n}|U~0CRI1Ctqa2@qWv}eeCHPlel|Kg}mTur4F0d-7w;w=I7ePs5C`l|u;UxPUk zP)89HpR}&HXi_k8bj!#wlFWxfyoP1^CPw|4MxD`JGRxAaQRl>;fqxo(RxXsqsDzue z5V+J#wK(Sildc7~#?tE*GH7Khb;nqG`w;ZPgHjxdUwFhGH8bDQ9gHpJ z$J;8#vT|LgphEQaw)qZk&OAiJ#Yon6y12-7dea%5E^hq`ojQYATx!knWAhyzoRuQ8 zU|5s>0sLJV$WO-9C+pjMGRMg+f#x!pkDb9x?gpmB8O&q^H~Ad8gEP#OA<0fYG4EAI zr$q_b^XzaAV*y;w_yFRH6qBz6u4qau|D(}!%NDm8+$O?<`Ow+Z=ACYG*b|RmGz^0Ttkx#+@ zrrcmfz7*^7IjOLmAp4Bpn%`DNr&8b@lrChu1H-31TYTPfaD}n-|(sinOOP?x}fde3{|W-HW*neKF|sNUQfR&0Z{FE1I`= zy5wG=n)@**E#amiuW3z~z)Y|x#gUfEA#(`!y z({9yp?s05SV>r_uqjJ{SjP#4T3oxGt$a<^lF2HUckmgot(l!rTEVVxX-qX}>9z08> zWC_L7zA@7}^5#TWYkHW224gU*hXV|@GrgWJJU4!&GP=ZGxy&jy=DyY_t?^qFmI8gG zv|{wsm~N&sA5NEC@{ZWv|E&J36P@I@Qn|b%j`Grzidw_Ia!=-qXrM&dp zU!Y7g&6dkDtgqZ`cJ!v%euOJunSL19s|>y}{V%|)%AlL+p8}sq>^D1KnNBpF{vYQn zGw|6N7z^Dm!_nazBh8AKO_k9p7^(j*ulyV>AI)axBQpljrk56(X0x-i8OuQ}VKrkm zJENb0sm`cWxY{h^UPbO@;5o&ZjMvS2_{?Zz)>4FWR+jQ>m6Lyql(W_;XC{s{^DHT6 zty9iSc7HR^M;3)gY%?imv0009^0m`lImU@_B~iv1gxze|SxyyDZox1qhwoTj=l8x$ z5owE*Q!va?1|{VjG}ob={0}9aaz}YtrK6m*rlP{Qi7Z%cEVH-RXFX!B>kqX7PCGMM zZOpvfX=mmkGW(o%WW~XUZ!nk(vv+e;7nFZ~kQ*IlPe+PIA95(=!yWl(PGjJc81s$lj%GJ|5 zerD%@qu#2PemAqD;Odwq^XcQtpxzL0gUpoq1gX^lZX~$jw6SwwJ2Or8S!TwAz&^{5 z^J`Z`HHXVS$JZo3f)SFA2e!Q<`1+~_(z&7Ve-OL2D zp8|d(#?E&JF#9B!cN}9hgR!ia%`oh+=HFoK@*Kz5!qVjf zY*;#BM$lia-p=r6e@UmGIK!XALS{}aXZUlVHAj~ZvQ9X|pVJqNZspJ-ehz*-2bDlw zyXMOeYn)l1n~M-xtu>-Chze@~n_(iEvkTZMf}e|q+8+WAz%uPit>|!`%#Nl>W%QC$ ztf`JkY_+G`IM&QbX%4f`oL}2$*UZ`e97H+i35l|qv+Fs>!5nL$=j~}$dwTe$O{-cg z;f?axpUtg=jmUjQZ+$bZP`wO$&R4)GGnJXVH33m8y19&FZrJE$a7G^&(Cn`FG|bg2 zlOliSc7fqIwQ286bZ#4pTIu1&$VtksGz&e`H7M&wSr^U6-2zM*m{MDt8KCIbqSP-a zJD0Mv#pHj~q`z~?xfsse(JVSEqZ3bXod)?8y^a-xuF>`(e6?32q;F7!_$)^URhq;& zY^Cp2X=AxeSXD1H-BhQWt+N-IlmiX_f9f1{b^5E$b#||Et)=7o!YL+wraJ$loGY_P zkDyX}DdE~{)7MTi51Tr;Y1YV7mC=hCPI8g-U1wJsWAz{`1M8eYNwTZ!aCuEH6K5k|6xN3>x2{5 z0yMCowoqM9^!0gM&>K|e@Z$Dey}vp>7BE9Cm>yotrL8vu)El7w#4Q({y|)}{A+zJc zN5hLdm3VJE)S`PlZ_&Q+G-$o!P)jalKQTKz4O)K-P}Ev-C_D{XCmm|(Fwa|hc6b`J z-gT&DT|IAEXX)U*0Mj2#UrGEu=M$IT!TSnsmBim`>5paw%cm7zNl))}_c2#6+AHen zDYVSrJqL=oyz(f&c0p5bYOTw>=g_TYhn35LMd8`rbHs9s*ahM?;zs>@Y$Hts_1#0<*OX!j7Y@694RT(K?mU z^j)5tLNNioye+`5dXHMBF}KW@epMO0;0Svm>znltazn^{P8H=B)8L};3hzF8lBJi= z0j3nqWesv)wou`@)Moi=3pY`yGrY*!useyt`O3_+7siZp1loB&Y%et5Q5)zn9< zqQ_Nf$gbfr+en(+RmF%K^kK}a7P#qkRk_dFOO54QU63`K!G83KtS7F+#mLciv7E(F-U$qvv+Ub8)6`9-VjZPo8<3ax!TduE+j)YGW zRk&$*4E?Wj`d{@14PHe5_AO87|H@IoNNg%nPSq>Jnq4>0jV9MCCj%40nn^bnd3GYb z@^Vm%y%G#;0GMv!$=3b#2jXu zu3G{@w7}ITHpb#SS{Kt;0wXew)mcC)rm;*z5u!8DhLNwhhIeMYqr?^xms!7EHQNz8 z!SDMbR(x4_mN!>z$I)&AFxshP#VM``>(x(5 zCB2>df1s(1-_-Wt=2j{}jOvLcmLRgU-Xtng1*ir?XGyG!c$hEnk(Ex$oG6JbPwV+~4M zQ$SCV+2KXrWCa0X`vTp=Grh^R9AXBDDVWUUVux4*VkL`b`kQ$H8kLpUGulQ&rC_+U z&c`mw`g^0A)t!&!(+cFJP|%1~&S`$RVvLHaqt#VbzZR9z*(X@Nw~(bYekHtU<`nO& z0{Y$rKL>%mxQu<#f)Gc6N4TG{wML6It@$VLZkSV`Hg5b^(dMeEFo!{H+}N*r9n2r_ zvrG_;w;`)e7=fg6)epdb!t5dK+ibvGO*pu^Q<%e|5(c6=SC0lpF!OqyU8}3F0J8w$ zb_o!_1@Rk%+eLh<&3}N;5N;PI?Q7bxeQg}(-LI%kHLjTm=!VJ6+3+--@ZFn?ErHlI zPkEgsjLPWT!*Y?7bk^qZ-zg-)HW%>ihv)#)4zPbz5qpk$qJ5s}z9 z2g^v*fxX<~okhEsFQYZmYCHG`F7xb*aYhtzX-Y4~sZp2LF^Wz2C-X=GW}bOGX1-Y8pK{)(%HYoDAT{Ys41QPIT|_zN#h~*l-i*$ z(Yr}jCV6J)E1S_eA%9QkU$ABBrgn+nUDPf^)vgZxI|4Vu^(a~Ld7=NF6l!(P4No^Vk(41K(WZgE z?CX`YjSSg*hgiPI&pT|h7DUR>V8)R92lEn>o2il>@+O&=FcWjPU9WX9FZCz$_f)T0 zlb5M3VdS+mFUP9q{%l@us~SmOv(PKlSVxVV8s-(;6{eKA(Y(@J8RPYAVSb-XorlP0 z&wmLY2o)*0^yN~LgOyzQ3Y~{$d$N_M+nZxJWav|%I1VE8DaMff+N%VH(D&R_Q(UCS@U#E$Dxz|T0YXgp zlR}5y`chm}D#VSifiN#wL@yAzzQsxN-(BYn^Q_JgWwWh{N&QLaI zllT}5&&XkaT|pI5nQ!hU1(mCYz?j`Jq-50P4Cx*Pm9Y>Ip`1I7vsU-Kip1JtKo2hDOm-yJGMcq5<0PQw<;*kpi-c|?6&GuJ z03`v5kz|I8#JqsScrXmvadCBfk+|GEAQI~W5_74ew_aE-t|qfaBrXp~Yyh(^PNG62 zt}+icg2XEU2{g6#)_}wVWOSEw&Q$@4C&3_qfQu6%vDrM-6T{G_&+S0)(6uiFBoN@* z_e5fIK!OQ=ErH*WxTb|j+-V+WAT>q>Bz~gKw~mC~=u=rB5_dWhl~FMGen4WDNc`G7 zA`-Vb5|vEQmAZvnvZx#k3=|2yY(h${908n9oFcmr+uMRQU$X@|?agnX@}(MHMU_dw zSY_6eSPm>x=2j9n0Qw$GVHJs6fSZ-Mi^Sc)J`dBrYK$}Ot5BXbZQgV8v3;3G)+c6J zW%SCUc#^g2^v!+nKt3C~tS(Go)|Dp#d1&@?W_bbL^^oUXk5gWs4K6R=ESG}{HxHNu zF6VbE;q?sAslR9|1Izp3dr7ORtooe_@;{0kjmO8t7qtfqcy+o1kZsw@oZSCY9-n$2ih8Ldd7V)ij}3z|*cN|g+>dc$U5lQMYB z4Z8t7Y?LvO#1Y_LWyI)XfF3=_K89j4#*zCS@T@X3NPGai$Gg~ASKmRUzndFtvYHtR zhmm{r?@=}VJGCRvQ}Zj}3uRvAYjrTSE=)r`8J_)v{elMHT_B%F0pePhtxP=Th(;T$ zoSNoVM3XL`kIbS1ieIM($8z*6bz})SqTjmv)n*NCj!-YBm1)la(%x4ata{xs;Q3;D zxl=VLqDB)1wv5A+XuQXj5uvw%H=|^pQ6GACEdys?_c8Dxtl>-<<(1Jj$36RxRixaPFwWnfC_U(~FbZK^?WKVGXu8=2&KynX#S%Ak<-$awv1>xDV*nT!49r+*bO zQllS6T@Iyg@co7+7JGdO%zDD<`ZbzZ7UWPa4BA+~KIrqxsO;q-GewfKeeR-I(a84s z6q(074^DILHGHDMG$M7toQn=x%*7VM% z&O@Ye;y(PCD!Wj8h(@`krn$6CqufH%Jj#UmJvDO1o0j~9zPE&{Ofhw~Q77jMQ*6_A zweo#?$bV~c85t~ik-v$`!zPy)YtvcJncNbkAq5-yq?gg1qPbU_0`zE}wrr$}%9kcp zm@1UjR8*FkR7ItyNoT!fQVGQ7X;FFHq}!IH^U_Vkr_vb4e)}Cw7h#6EhrXkKo6LIn zyRIa4eKLE^_jm(~Ba-iZk70P_I9g8aAFXIeX1D#Z4xKzCmOR~WVZIXC2S0~^Ht)HE zKaYZ}QnL7S>>*>P<-SU_9{-5^8@8K|=-fPHyZMODZ8Godk47MdF}9meY>&GBll`-K z*NRTsW*^~Y?|Bz$MkjH(NqYSw@}OSs4D%*h=8L7&Yo5V96mOzAlbpd*hHrLI%A;BK zjK5MmC}SEg=7&J!ZyDS_@mEGlG0vXb^;JP>QcLWXyjF5XK_%TV9LSR1dybX}tfRNQ|um902BhQ;{cEJ?;Mqb&~ zQpqqEVRu80lzsYoH(Zg%2X`X|Z+Oe_qUl@;+9L@LXMb(>5U@2RK|g1)xh+Qj@(ilo zX2oFzR&_J=5ZHC-$bZpb*24(w=D5JFql3azF@a^DT_g9OKR)#DRaRoTZ8y921O;DW z*=?P~uA2uc=b6n&EIU~LY{2|B)9H;bvXu!UTNx9XUO0hlD9gQVAxRDYASP5#_L9@N!)4w=UqIdlX6Oy0 zYf+I2_d30@Cg=5qioCh~DA)w(A){QP?glJp!y%33b!z2m9M1!fD{~=5RP*{QcXDCNFd&*&jg9%jb8wI zWXy7mZEyU>ah#`TB*gKJnC6XoL4i2FaUjq?k>kAF>Kr3RIns@qX!F`HXu^-2=*CHa zo-&gY-B<}+2UB_Np(`i45d*ssNz0P+dYg+fG-3@w_s*mby9lP7r(6U(@(<5k?5iE` zxLEPf`OH{h$F}64%D&$YG*@6J_#VT}bQ5w^wI#NZojN^NRrzcLyRsQJO`hhtnHQ*_ zGP?E{0=HK^^6FwQ7uarKbPuqdf#z{NXsy;(xw#kUVZpo?DfkQUsxnWK_#XI1($A$< z-fNryLSvJjQOqM4ZK?p4`JLu89_e?wp(-%aynE2H`q)sVKAss#Xw19H^|J}ZZaV1t z+4KY%y!ko7N9 z7U&^Q_Xng8D?@Hh4`nS#j#b1OU?r|7HBc$*ayi%bQE_GThUNU)5S`zGh|`D2;6n5b z7(vZmv51rZXmUng$%RyhNOs@euK7WJ?LBz;cOVQ|)(vlg%B|PCDW_8sh8@LNMxi%j z%DQg43aHR%>M?$e=BD+)4L0E1@WB7*l>z6hHLWyYh3JB)9dIkFa*~l(S~XFT49_0R z4wr?sfhTa>LC{9?@bEA*RvV;DidWvWwc-fRAXRK}Gf8!~DR%H}N7xi%YZnt+?oG`> zBDQtv_cghC!HtNmc}>+dqvTwjRm#9}J2cmm!G(~yX&Z2hGJ{Aw1Zc~etJl32jwdIf z=%ogQGdyqe1;9|qNz89I^Sit8V{9%--4%A0nCaujk7b#n^D5e|bp32b zsGDz~pS*#tpUroJxy|)+Gi!sJaXYb$);2vtPkFO39s&#!so`pk z#%{(LZXQk()ROz2|0)8hrEsBDOQAI*tCm6y!Oonud~qYtT2?J;8@&Ep4~7>$n86KW z_KbII2G7mtkQZF=`g1ANp7ItxhAsQ2y!I5b@X-NG@7@5BOL)$_@RZ2;&zUpbhU|fF z;&bMZ&u&{8y*4$FZOvY1f$GbmgrCt2a3(NLnTaG013X;kWzQw?CGgQeGH3g(tVB+&DgvC6zb;sBu6tK`2&;uGL)Wj^O| zPi*#K#(!@85ICvK`JQ(h zdcO5XRGP)*y!DzxatH4xjMth(GHV_c+5t^vX|~2z+U2g^uL<0iGO*jGqd@*@Zubx1 z8~mj`IUT^~(UT6~H!*;`EuMGJpMLSfnT$I2;moG7bN8mPbNA{W&g|fywkLi#ldkNM zJ9NIpn%s%I_a801-HE&RAD#IJC+-=k9pfkNQTy>0cj7)cM{V4R```j)+==_(B4yl( zJD<)H*X4MH3Aqnw3h_Nsw;#|nDJ4o#97UR*{{hWh7_}$vpf84+J8<_I4G!Ed5u*tQ z?vOQ2<$Yh$I)7P?!9C4h>g4S;b+I6`SHn#DhpDGZgY7WO{m*U=*5!`;&rJiB%+63$ zGKXv}6rMk%Sfr=w2)AgH5F?5k#t;=C^0pPXj3E{X@r`bh^G|(?=84)j8UCx_Rrtf@ z7>O&KC3@!a1~J4m)wq6nh8VFuP9o|SG%@yBR(%w-xJb7Y7PK;Y-=3>822B1f_KSm| znYPX-aX$id-XZ@TT|Tklok8Q)p^aTgFoiI((3MmRKDxP)*3n}}yCVN(##}EEUTjf% zf3*MZ^1smJFAUHBcS5OmylzQ!7<#+A`|PGF*TU68R1SxO%G`2kD^u#f*<7Xdzsa7IG?3h-8UIQtnxLY7pmEZzvz0lA1nh5J)tu=3 z7`z&Z# zF>3B-CS`yeLF?p;8_^)geVvGO}WTq?fvpL?v*IGPjes1K8S% z%v}&|fu47_5EE+n7)8f`=T-MP60ZWsW7fEP5)})2o3oL5jr(XIYrOsMz+aWY8n+{) z?Yb~q&>Ir6#_ib2_B^XM4L0RzYmE=;Oic3FUB$1xu_)zfS!2(7x&1AQ|EjtO za{GtC`@R&8SSiTP8k-In%@`UzAd5&H7g^w$P6*6${uNEW;62o)L;@J$c{?INvMjH& z6UmPDV9siYj}TpMvRkBurYj*VwG8&`d{z zo#CX#-p67Po$jm$)J&S~ed0LYITS?CY+t3j#-PUhhyQ`V(}Uo+lOVct8yd}j*lBbp zQ`ydgEU@u77U{cQG8De~H402$K>M0~DaI6b>S9Svk%z0DUnxWQ*!dIigUG!JxtbF^ z?+(oDjx=SklslLmv|B|?yWm(N)aOmAiAiNSuI%71lx;$UAJFbtOpKC@Uxz1Jnn%BW=ERmpBL(vlWi3JHaOwjF@*}+Vn{f5tOqKcaPFugbBBa;XCRzkfw?;n&I@Fowu9LD zxG#XnAuE8gMElx#+!sLPMF~L6^W79H07iiDcAj+Q5cM{n}!LEGy$se87?P?3A znRprQk0WE@Y2y$ZOB@^47x02I-wX2ZV(?PH&0B-DR_36jS>BT(j*agc1q{Q+p9*?g z31YEq{N2jvmU##}@0Q=fL8kUhg~YvGI|1$U3V$GR0MPSGdAGoK%|>pM0***?*ONdbyIxYGU|F;a7u`ktvqAV~xyv)wdUEggncqygcT~d5+=1}! zuS*u;?fM9KSF(E0abLw8S5<@IJm@ANT^Oj+x92va4@dRsyL_r!K>721=2uX?%TcXj z+Nf$Os)qxr!@!&kRr_ou9al{PE`jQnZ(~%u?)Ir_%jxY9uUSK&x*L&**Q)uzY*F14 zP~8k>BUJ4C$6{1#+~QNcWfSUaA*$KQsP8t>Kz&sRsOlbQYz>IM1>#kw zzFTzykoWA47Vz@DQ{QevAosw;K`ezA<3_9x`{+C_gn|{$Vf= zI_7ViWtnfS!CisTARq6T7|)r@)KQL68C+oZ^T1K2j#UBT?;zevn(AE{Ac)+%zh!8v z-O%pD4DKw#i8nJIMQSd=DpFSqMw`32GB!BX%5d<_LsWJe-p+IPU+PQw_Edfki`m>F z3{Q`c$nfrL12jgc1+^-px3`vX#$S3K4E%^l3gG6>nZQ(KI+8%BcUDL~<6*L(54pR6 zeagVWozDSJD?{A4^E2Q>WhQ#|UH5yEm?`K4q6KqcmEB5l(E@^^*4+N2e0-kI?Dpd^ zW}k!EV575#*>q29$Lw}HAFQ7Tv$=OUX7?!W_MHs%;qhE*|~04wilD%+u1R@ zW3ME8+gzB|VB$d$mxIWcR)f{K+x97xUTstz%Yi zE817nGP|QHk=adRRui?2m91x!z}dD_$K!|$KK3;sdzc<0zo?A-t7f?-M`BG*sLA7r zO{$CJNsPtzax`d?A5oK+5}QFQ^ZcQhT7kY5n_Ou> z^5PB7V+nqa9e5ua{MY};%ZNNC4*!A9!Fep=&%<~TRcmU@X;m5BI!RKI&I=zyw5^yG$$YPm>+#@lFvGNd+luKZ+!pS|1$@QWd$-Zo){`!xhOU*q^uVo$j)M2o>Z7NH z@8-Jq(}rf|1$T9iP_2l2sjO<%Y@5po{e)r|9FfaQgY;5a^G)B>kRFGr*{GfWT3Pc= zJNH`I5nkR@4)P~PDXn3accDrgerY`LCZW(<@lq z>nF=}9GL8X-mF_ZVRa*Bnhg%w% zPgyGI&{+l#^2&7gd(tL@?R#Z8^%qR$njgW!D=Vl!eF2f9Dd`cq^*vnIG*dcC+WVuy zRlN}{)b^OM<7zMbKa3q&H>E!@@a2st_0XP~+iLpqsb1K$rk_wc0VVWm*x09x&qY%O z2O1LKJiS7$;GHNk_ZAaY5ez2QMpr^oFVVN{Lw!jWY+e`*lIJZ_-7?kv5!^}~*K3n3 z>4)aFW4>CS7!h5;S$&_c;-_u z%%FR7_?@O`8oodCGVfDKfDs>D3|wj!c%Q5J{baTPo6T(R3!jR2kh$vM4l{qQ_k|rk zxnsbySq%R?z0e7zi+`R+_CbmWyQWV!Nzu2*?-h$MMp!HAuZ{|&?k@j3?1Odn4n|#sy=3Jd+#>PRn~deQ@~?p*MhOZU}DnMe(DUE2MEsCX9>X+^H zyGrRC6OLqWg>&%nAn`1fUeld~1$~x4XN7&}beZ)u9$33Pi+F z!NlEv2L7PT%On`<-JdCQ0-vMB-9J06?A?NS7yPS74spBCY2|PWDmC-EQ2jj$f1;>m zS27>laFzBoy&i{^G2ovQ&Hqxp1Zdv~A?Uhax}uw^i0WLbq%;1*lYt3Br8lJbB*j>5 zs&3;*Z$i;pV6`o7Px03j7f~#$OUGXhzX&|%Q8B$AVFkH)IVwUxhvDn+`?j<6^C`n3 zWQwWpt9&?ROgD%BEA8&xYi+6ZYR+cXspa05==t(v~5D>Uf5V5HdOK}u!8_tRY%&n&`}SD|DYMI zfoSQU_9zz3f`4><5M^P4!9_M0U@8~cAP`pUy(mYeb;1k0vChXDLuM|TN*_nfZfdd^ zsTSPbo&WwEeWZsagtw)A9aeF0-u1!X?{1v4)5`rD*R8rrG@)O3U=M0<-E z7)SK8h(SK0j?E0VJ@GaO7}PhT6a)x(1=pAsoC0Yf=|3qSNg}qLrOHV9p7FpKbX{(dtd-9qX0+!BwHS5!BSyU29m)tATnfyVg%rF_;mfvFTb~!Tf|PCbp)GKGmfjSc zRDmUB_buq;p7lT_-;sjWi>8qjQfB(H<7$L%HDMA5JLHFIJzU0!LM2`}wMVh(ncO`x zG(d(yX7z~lj0o4ab}y_YDO0H8u#eL-VaX-l;C!{DRtUap1e9zmEJM9uwGpG+LDPov zSlpFVOAiPocP~yGnlDO`h{-kfEpkmr_lbmwDVfQ(byl_)DNY&+oov67A<2V>2|ROf z($@5?E$%dm;u_6si&|;X6}45F8(%x*k2H^zC6BH(rkL9RE{X`9Kg6sut4S1`Rvx(s zjtZ>+{IRL?n#p$b)uqG}w+U=Gc@_n{St&SV?V7e`2vHT+7ri>h49yoCb^VUQzW9An z*YjCZas3!`8j{$LTvVK{k_Him4_NOVz!5YL)pqysJP>*NeRjY+H9F*Ad48DpxTtt`NS#7;vRt1mX zozW(lCgP!Wgm}>=F7o2cm?gH2s}{DCo7Wzi?-*~_)(Id`bo*S>G%FwDshcIaMnY=BWnv#uqAv>_4?I`o!b_U#G=o(o@MY|G12Q{p)C;^j)$0} zSZqmlefbVABMWGYa~?%J-LO)8cS*H=?Nm;Tilg54^e)@^2Nqs z{3dd`n`9Jv5)cuZ=YmLxBnMf-WCu|#CvYXs{#@7(#e-Tnh*u0I7O#@+;GHNn_8JMn zEF}om;ypfF!FQt1`NBvKs#qy90AFyDzdw=imIr7g^u%YC{b6ML!^nv_LA4npH?EAl zV0;`pYjP`404ofP$cA}lA>WH@2K?Ai;i7Y`SUe%BS*Hmon&*l#_f zZV<=`((yoEui6yF)KSePm@pDWC7m}sgT9*@?KI+f#Xc)c9JZyMT;ofG#vHf9Zp&94uXqWIoHjBl>zx`%+tWR#T`&?~ZwRb?oNa%ZA*MT=Aw_0rmRuc!PHl6hn?OBI zFePo>ih>bjIA)o8Vm$F_=D1Ieu9a)#mtEt2so5rG#lGct4?MF++%tRHQP_wbM2lVy zWf`f5r}fqh-zO$~7rQmMzM6qbtB0)EoYgOecP;#wS#Bq^vS8Y>Gm4#%oOKO2eIBz5 zu7Mgjvq`KlXu{+o30oJibDoqnBv||?Gt^2lHYxGhphSeG6VGEZu?t$qsXxP<>yQXS z3P=q3=LIX$VTpATYn*TO>7t19>fwnxk&BH;6t$}~^7M7f1&Ihv1F?{q5COwM1WXEo zXL4Y8k!qe%Llq^Klz6z}lhOXtA!5ZCk0gZ8SODguCkdWP+zi5*7Wi^Ux9-YFWLF&p zD|MekVR)u9SOoe_69vcHrxh}J0JK_nd8hkde&B6E{{a!gtsg7hB3J8&QD)!`~(YTKm%MT@x zM*hUoIOZ|d={Eyjdu*3kb;r`_>*> zNULmi&71_dT0;Dv>LnN@*H~f(s!eH3G;Bx%H%U$$^Xc1A=X5E)(WM5?lNK`*w;gpx zWOt*nF;{b}jJQ8#T24-z5VNWw?S7S=YTPNu1XE8-Kcv`2hTV0a6)cAB3e7INS_Q7(I*3`eEJZ5v*Kb_W@kihf z-DmDJzp%#&#cA^5c9vlA&ta_>erdzqArNKaU^{9s@o^Nh*W$IEz7Lkzz*AgNaTF)& zd}hGu5)U|CPhZ;dZd{Foy=Z(%(8DqiTM|f_J+)imTYB2W^>%hzZK~{(h)mRUUsu3j zs_*GaG8f3CkKwtfs@QWB`^dX@YsDXdz|VqLED7*w!y`O z*haxBmI+UCW}|inIkC*X?M&}w+mk>Rmyd^t99!Qy?iq`VT)WjNP6#>w>x?R3JJD3Q z+rw>S3gf}0X5jNWlZwxB1Oq=N)DGBjT1`}89Y0LClc?Z`3#+IXZR16I+8I zrZx=1QPf&tFg`XCz8@yVPQPLs3RVZ%vQ9P=%zuG2{n5fe(|`f$#aH=!`q7@`*be;7 zQ}ckQSko3kI5VNYC^_B1Ihsw-qB9Qd8LS+O_lFbr42i~Jy&jK=1%u?kgYD5w zyF{NZejs#_ygC*W0{8My;p{F#(@j_hd|b(hZ66X<-zgSWOKKU;vU~~HBeefValsD> zy%ZjL%f#HWr;jyE7eW+3IEHeEP*xK`i61Qc#X~`vWMbXail~3$N?q&p0aoK~EuUzl z1Fb$17B+)iM~U*WJB1(!$ZLnh)MYi}f&(W}@p!~`!N%8gKMtK~385g|`0EefrDNxl z@#T2JN~MM$sO$l_v!~dhi}f0bEG_Q6=`l8C0-4ZJm-waqh)wJUWl*LW0Bi&=f`6kH=dj+CKe;U1=I|<@SQf1i`apAqSCZ3PgvH) zA{EWgSXb2Xdv=l2KRu7fb^9(T|0Jh=;Pnlxh1ms`3n3hQ1iqcfM)dT>0;kyuO|b|r8#a9UJEw|NH2py_#U4q3? zM?XMxN;F`1GF=jEp*vSEw2L=Av*>ko&W;-V6|r0}!Hq%*%YxGv|86NzO_Qwst$n*F zq0yzpoFl#muI6KD3D$asxuu`ZpyV3~ROwp}+TjMRxJ+3p#62Yoql$YM3xvR~i`TRh5)pz;Tcfi4%?b{j;k+{Wotuq_5BJe{&?G~`6?QDGWTg@6>+^Sb? zw45pilYE2I{iR_r;|9lvr*&=ve(V|)eh{D0*%t?5O2GcYdLZG{)GqZ|3TPU%Cj@r& z-P{jw_L`);*WV*s_t(*-TuDofv#gboi=7z$QeGeCUvs}TpVQ`*j)`-W%#QOzjvs1pnd(gc?i}~;E~oo9-^=iK0NvvL-Q5puJ$UO^ zB!XPx7sYzob%k{xbs8KMT4U^6Tk`Aej6E^Bas3@BRG;+wN@i}>b^FWM1uFj-%EgJq z)xqh@f%xH}jT!?KHCPGw7o!Fxib$a&@Tt=d4Pplfx^`m2ZDQO{_9ID+K#{*R?WCL@ zR#M}0cfv_R;M|H3@v!b3De?Fqu&dbqA(5f%n6dfB;Lh+*Z<+7`03H2QxGjm;?bgW5ZJo}iz;3Gj`_PE z7P|Qhf$s9q9?}m|iXGSYRLt!a!#ayrB#GT*!UYIN)LC}l>XC?x5JBbmS|PCQM4S5` z%XQ8R6IK)c#m2;&S^SklwHqH&0yj;KPv~jB7pBMkKN4GE#8&CQ6hb&8KQs35)j6qF zMjBrk1jk;&=K3dcXP%AwUM%NMTwdrvJRacc6{CxqF<+?`P4M=Fmj30VqQHi@2B{PJ zI?`HQ|Mrl2!IIY=rrDFT`o5(l-YOD5E(|={mc}lHMuQ{$IGWP|I~V(Y!(v+j=6|dQ z0?BYNaOND?EItQn5fIx^XcgEZusVZ^++3&2U#+c_s-fCQsJwy7 zzf;7&Mi@+dgxgBM2UHbn@76%0ZAO=hW-_a zZl}8z6TIZe-6lc%c>L{I-R*iM)=3OB+<2m_t_+?=h3&qz=Ge5u0Fxi{lDRc7_Lp=d zvW0gSC2h%gDnobGzO^8!GIwjP8RcPAUee={XCtYLytHrYbSmDJbS!czVnzmdn-Za< z_ajvq=7OZEfW|DJ#sw<$l0J((m0>RQQ5Q1(C%u&WR<2LxV={MsYj1h`FrydIb4#!;Y)e%=lQ)J&->WQ%p~o|ugdp(tx{?v zye9pB+`S8sTi1Ceh{pjM{eC~#-E4NFo6Q!*=9?lVQWo{HsD~w)4~bG}50%J?Q{~BM zB z$MuTs@cIWT<|%pX7a0q2$lR0iI1YZGQl6X#+Rb?i9_N*-D09M{C+U(W-tYbm<-=A= z(zXUcx`9+Hy_Uz{PkR3TP(N%V)F174z5;sDe!x+!D&D`P>Dh&zH3)iHq+03a`zu1P z(qALJT7Og1YZA_|v{+xd53;BG+I<7~;7!Q~Z!BQ`RID%FXTPB0eaSxOtN-`|l{CQg zcX1jFxLK0OA`J?KUPuguCuWfCDf*n5qHjQ z>M4bHQ%@;(hPW9fn|gw#08a|(Rgm7qKJ{-hGsAAW}0YF z9Nl!Px}ZvG7>5g(br5c|q^fGv8nucds6zSM;Y<}boI&O+sJ>0DeH#+rm_aEm0lNS& z6z#M41LzA~Xls^PHmczaFxvqY<0D4sXS&d)60`)W`C|p>7PWx=Bm$_CeG6ZGsx5mR zwjoRqRd7Pyg0t>q?yU0aG+3*VW=bddsBR-Any=oP6xvg9?;@`j+7z9w5DBrow(LF! zoT|^^cZierJAw=c9Z5ctj-tc)=1U)_qN9d%yfpYt*g4g91f9;;-}yiVogt_56}&{8 zNhdL!X{mB)R+MwdM9De)I?X|6k>=nmJ4d<^%dgwKh6A>3KuH@#ZSE*De5eH#CGAD~ zO)A1+NGQLwkz{I!IaX6WD#$N9!3_!!-vB6Peq*2HdEZe;dh}`hdHSN{_Tm>LuUAm@ z&MCh=Y?rKtaTKLwHHMF@kdU@#?C2W2%&tMkp0#`AoyMKD=gkzG!|B8q8ER_E{i@AF zcQ7?IAa4V9`w~LPFWcf_d<(04XT4Hle%yiOBt9xRMR8Ak-hNqqzB%lCKXz4Z4R?kS zEUjc#b$GXLU0!_a!F51)eUi}l`aWe_CXM9~(2MV?&~SVhc?hhuyb@0C&g|w3d6Mb3 zYRYzk2+7&@EXj#Ii9#j)x9}xQU}oSqFh{O`DlpTYwf};O-_hD8o_R{UD zuvxQ&u@l9h2N`a|<8!y*v7Lq}(r5&_beNgIOm@cc6?8H1gt28IGJ}+v$(?<*P3D#O zx)^T2PS4=>C1T-mBN8hk!2jD5%I4|&ip~8`X%|yZ+keT(P`4cD)>InBPHaFS!pb-nGWJic!HJ+&6Oy zH*5c_8an9ylI=ZR^N5o><9O@?vbN(#dGXf-r(%YN>@TRHIp=%m#?2kX`_U8lbKu+fWX2C-e0VHV;p^8pc zkS;a(Bonx|Sqee~{6o!RMf+ueH9+40M1&0z8&8KQ9|X>m&GKh47(PgDgCi*;CjxzF z+5CO*u#L&BN&7F5jrlVGI?AS-wU>ZgbTl0lwdmZE26@ZbR=Kmsf`)!UD1+|rMJ;~A*L6(47k%iKj!D3o4LGLDYG}&ogJXi~w z=vqPFK4Uh5=i&Uz=!cL*W-LF(9P&~rCtJ#%%t?%>^_7E>yE$%+JTLx z-T5VGs>xm;Tq%wyf#m-Rg<+#8HY$+Fi2SgEDlzyaTenS7o|=8|OIN#L;Xp!7r~Hj+oPoim@|#9)q)>GNXbjfl-D@>n4&Hx+GngF%gm%x+EDT zBrkMHGD=8dvgd}MA?G`A!S595kBg3qgFRZK+p4K!<_ZE^qlLpW!U0G^oby`Sf2I;} zw>|psnr@#S(`mYxKy#pn{vLXzA*W_wrVE2-g#r1R(ZcCj*pK{-sw}0d)lI_)+Cz33 zNr&uZ_QcD0cwV)=Z(CfwjN!E%_x|Y63mECyn`Q`U6-Qob9l5a9xg7XO<1T*g2BuxM z+GO`R4t=2Z9hIRwES8dR-~vPTJ3kvG0Jwx@avIputt613ZkxrWKd_Fr-O*&ad--(de$rmVB!3R%`$bZ zd~9eivo(q7)60~gYorXQwkOw*OIu#oz4LCi!F>5nk8s`g1g1#yZ)(7oBj|uPQp!1(7xXi&0!ct9CY%n>2fvN1F){yiJRd~2KEVK|?#JJ7r`2y*x*hpdo5Kh33)JB}1%^tVW`Y5@L zHb*tLQOIp_8%>UOxs4V^HILX>ZA^GT*4kKPo9v;9_ynlMCz2CFEjf`zSj%1;Z;qSP zn&W{9EqlS<;Pe19j~~XLQ0S>MmQqn?P>jr_v?kDL_YPI3vg+X{ZjXZ%6rEP9KWI?~1uV zf}RTm3d+n@LDejz6RsjB8;h>E8>Hjzl-p<%C=SWW9O-4ukp|sDHt0^cO|l70LYibt zt_Il&n#N3-Grj@NyoURtoHNe#l^81Z1wAGsC-jA%VYZ5BtZr3z(=A@L=>{g2G(f=P z6R(QYXkV&>=wAihEv5^w^sM1J)FLA$n)Mvj_N{J;4oY=}U!8>j9D~?5Xe3wF>Ryo)7kb><4?&d(b<@ z4@J1=HyQPtjJhl~qH@z6PDFJrq%)J8wVFn1rZfYT(oAR;rV*MA&%!jqvkina&BpW( zrUCGS>Bx-9rZxRTM*WacFBrK1_rgqN8>`ApgQOcXTQgAZ)=UQ>N?G8W7!WaNK>7!^ zImo7RvxQmcqd*~~D8sY0S;@5)Op4No>lS5rHp!Vd%*bVC!7T$CQj~?+62c~j(&9(# z1$?v^-ie|J?~LuVs#@B(`eJL5gj$P%opzkkClyV%emnh3R#p0!(o3LHT59kS*cIMI zn!BRAwA96&fhD{-G;b{L?yT*!qStn|C`4;#XD0;ifVh+y;P7^qwSyR4_1INnDN9D# zrNR=`vs5Ai^qJrr*NE|!ULt}~AxlebBD9x$yTHM>D*ypm1xrhTT{PhzJZ*;5z?iO( zrKPBuWK>9IkNRjVZ7mJF%nJB}r=>!I>QFb#3Mp>erb6@41^aWVvJhLuqp(=Q!|>*+ zJsX-=r%z)6JF!q%Fy0mmi}HrNP*(zE|Exgpfn>W(Ms(Aj*J*e#?Ril#QBhQ;5bfFE z9HkDQ562&D-& z=Q?vT?CQ+9~dX63U^ju*MwqKZQ@IiSQn^;ILh&3^LD!L=P1BuxkB|Z|1*+tX7uw(%_ znBn@4)(#9BAqYf(74O*EF>uwktOb+l5|C2i`Iw>o%0g6YUrMT54ki#L+6aM%6y)<- zauP5K8EM@@MxMHO7d75Rjd#(k2E>;iC?w`7aAH0)50{gf_b`yb zUUHyCaNmNTXb@E~>V~1Z3ax#04H^CFT6UdPd%du3aY(ALUS)K3y%w>FfcS^=oZVx% zxJGL%0tl0?y*MzIxMvv$SE(T-*0;{)WE}`-N9?LbRJZ|(;XtS%Qa9FGYcSTgZHsS+ zv94{c4ZLAnMi{k-#;P*ALMx_GN&rNV8735VQKkX_WRlm619yqjdlMj&KuT}u{Bc+Iwq@XM6R_#vw+#TBbN zFRoNppwr4qeFX(tUuhyF%H!KPdS_y=FnT`~+W0D0vl*}yF(WmN{|7=nBedj z&fFn0gc&4gYcivmlgu6YJKV3?-l9F^@2K3-_m*v0rWOq0Qr?OerC9M+z0TJlT-Dn| zn8KyJ8Lve+1_7Ay7QATuiUdO`*j3Y;SnXOm16E6KdVQ-RoNqO;3V3`APT?3$;e4yX z)qyM02A*dd7+j654t2x5PAO1ItF=|DdDK>0t59`owX=$X>8$!OhfUHYzKMAd!y^CM zodCZRvvn`1Ek_yyJ&J;#I&mKo?vp7SNw_T0NH|mB4vh*Hqa{mEXgu!L>HhGl&gAS6 z>_>n3fBl>P=||uBfy&t{b~LjUPD@>Lu3sBj-@|FCngfwr4vZl`D`QA2O3(#)J}XYe zLK{rJq)#eS@ylgPbSl0(cP=yqh zgt`2UH;jEj1Y0GbxfY~hz3-@?OIT2r8>cFnUsbmUaV#Vj>uU~2=zivf!44+;7z}aG z!v*YlC@BU@AV^?WnA>Q=Ac|Pg9466%j!|QU{={z2eeBRc(#L)zLTHAlvR1s0m*FBtMD=7=vtMjXfEMYf@q&6X>MG~DF zbcI}!8iMRMr7|Zs5@azs98UOQKNwQh64c6Pi%%#w=!#OghLi$^Iz$~2ip8%}v20d% zIs=$}rX3&@lj+MC3sJW2GVL;TUt* zOJ_cqllnk&fca|@!QhaY5*t)Sh^a<*14A{; zihAC}@zB!SOb%uSp^glc)s55=r8%>T{sdCS4Dh#YUX3OY%5T>N+D8M>WSV`}-M8D*Gp$UqmMSlVJ53uu4x3 zzp`y3TY54#$(>7j)>>Dh;8bA>(6MQd{!G_MEQnwCDGe@mJ(XFwqvE{dPq206gmP3kR1jODEHUoepfQLgn20Q|Jh!KRR>inC9fMmE_^d7gM1laO_Z%3}f(D?84xf8# zA}?qnmoyP9M6wZ zK2}v3uaEbgKD{QTULOxben^LMZH?Jw7YeC@!c=u?`E|3LUpLD6^&85$I@Oq3cm?Hr zJAlTNfBNIe_@@Jax+*pVTAIm9|7UR~5V>*?fBr0wJ!-eh$nU`G`pe=QHoaY-maW zEdC;6A;e>FWA`9$%`Svn2aT)(BQpCfPIZ23c59aX{MqcBIB-MpV$EqxiZb)L*%l{3 z*?WQTxw#;Bw{@p=gFC{U6H4zW;z58407OQ7;UQ_vS8zvqhvN+rAQctb5hGG;M{);5 zP9jm~G{iTK^8tZ-#F0~GjhrEJb}S&ZD?s=0yh=@ECY*1z_O$SkUofCeX; z6DVDY(!+7!2gZvN&UfCx%mGr>JB2qpkOPH#lPdup5>KE=3mx-Upf#6pj$@ix9`Q5`1g{Rr?1a0F*n zcp^3dw$vUu!}yFh$UuI+8NbX{BQ`(dpVjGnqipO2>7Qy&__%urjb>%i{xhmFS(~KD zs>ZEJKLckKpVlK*KUz_-1v)MjoNP~ml8R3yr(EdW?!t~tv`hMx05Q;Wdw!06KQpMU zT~fy1wt0SjY1Mv&8|;2?#m)H>jWhJN5&04$xt{<$>Qiuyqz_Sys9Uk^CXKv?ljJv+ zOxl8aL!3#cJ|*K%d@)7wiSZINqmolO*#t)R(~W7T=S2!5?*@?6_4ssh+VB+Ar%D2r zrQ#-NQd$(PXGx8_zQ(ncuetE)#EbZJBFm-0Y+sUF67yJb>+bneY1fB@{LqE%il5VI zpEC%lg6of&_Q$R)SM&t$vYA$K{Tb8#%!L8K&+D|GH`D55dlqcJ?2#-OJh^BA$usVp zj4`r(c@8CLLUDZJ!B;&NqTM1w4Y)Q@f86pl%a%LC1U?v4?55b}eCw9bt@+OU7^!Td(U}h|+=_Mxv*x#o zk1V8b8J}LrFWf9k9;+0$qEc8WFWf9k8FTZuf^J~0{Z{d$w7HYfC$_GHH*pU&6ShywqHh zQBQNJu!3-5r7RP;+Se;WTRjzP0Glm>S>jQJ`0Lh!8?D|2)<|q|sSi~GRRKZ?Mq@D| z^~Sd=E%gI+JBvj_N9Yi%d0t)*axO=}xBX#qp39B*{oyxHPJZ*`6vCyQi`NcffuI|= z#P{z${6Am$%~AFbb2gFcD{ursm~&Qb~q!3b0jZ|QU^+9ONGe8mzEEk(rP)puo=#R0C+LRPSqRK-?e;%E2TWApgmKp8}fy_6E1 z?5^T2iz8OWU3GDmv6aj=2MtUa;F?bmh+{ce#qp3eus8@JOnS^#KL6ZT#w8g}cy97w)RvW$i5kAdV=b zoeUk%9|w1{gG_3SvbK|<<1s6UiXBfKhbB_Tv&Yd+W(h6rq=AQak{DJcNj@8%4tLab z`hx=&aa3?1GJxpFKw?{TVj#un)IiP}jfDv`mv($D-0p|er_CG(F&||OCkB$%F#7ae zG!P{;9t6U?5BmdfEhHKfqQ=)qD%PLr7eCcsm#0b8{8~vm5?wSy6c{G942nT$8O(Uy z^L2D3iOJ{4vHmnaM^0mkO}1f_`!O$zFXt%XIgyZhM1LdMp9j?y4zObVNyE?t`^N+S5aOiwzFDXvl^DDxU=e8lj^X%4wvKOdMVfD1i-7aV752!iL ztmZK7E`}K8r1d$GTf#M}3A>Y^CPUpAmEW^<($rD)RO6KKP6g{k*)F4)q&LnOH70UT z=N=QUbC3Tdbs9Oj#iywh;%gvkvvZFRi7N)sicn-_&}J>n1X%RfxhDqTRh1Tss|N6@ zsk0tsYOHfl6u@2s*lPfDm?{E@vD?+w$?~b2Yop^1P9##mr6AHVEMd)XuOJ z&PWHSE8v(K<u-38$#Wb3(0a{-xu| zyRAA5-V?dU64up#K)M<*d0RCo9Z#RIz^n9$>{_HP?Mblje~p^ z57iIxnE)+5Ny7rajhHD0s(?Y(j8U5GvP$nw?sJ3=s^mVBUsh2ezpUzq+F)}~Z5@do z-xt|5Mlq)i26SJWl}%}&53$suhfS-telpS=!up8+;iU z8gle7N}J^_o!)?~ENVKSDtH>X>lIYbBnm7~6hIw1M+K51Ly$$rh6M`ObsHO1ks6OrDLoRrDK(2QZYHtt;(82=WypR zUM-ny$=4KoR*q`|B&X2A=^WCE`S7LA|jM!U3m_u@7EQtk8Vx0OLV3-u% z8<(H(jBGsdQ&2D|{`a~mgiKV{c~RC4e{Pp8nPxEaBmq-A3G^r@K^cu0i_5B5hL?ob z?KrU1I)@MTRz13j!C%VQhrwp?gBe?Kbe9F80rhFrVgKBxog|48b{!xH6? zs0po`YikIxJ);>fT|2VAxJ^hS`VgY=s!2)hLp9JtdTA6;Gs7qaR@ev}HlI?{@MOi{ za!dkM%=DzB04KY(?n5(mK*bIgQCAPB;=$6v?yQ2UN>x1o#0=^vye(P?PIXC5M?rUI z8e-sR4HMBCCTB~Stb4po>Q3Yaf}}(cgSFYLYmDSZdc&L_t4Wj%^*+6kg)cauvKz%) zq^6FfkK6+1NcM=}94X$MswtapO|#Bygzs>K4y*7T;k|Tbd&7I_f7*v)bWpLwiNg#g z4(ISZta6997`+ACW_0Gr7PuW2=cTfHJE`u3uHOy34JzP&%*BNF0o<`IeU(|JT@ z{_oZ!QjLE<)t`kT1=?ms=qk32{+X`zw{vtX!}g)Sxkn_{--SmcoxdNIv=4<2-{28L zhhvBD{fRu{wZmNpxy=(wJ-L_lgzcTw0{zEys;n2-@IU{X7KG*7N7kDPR4jf?|D zMDOa4V;hV9B>R>ywb|I)rlvH@c9&(m$Fi$cfw9@Kpa~7vH{0kD>jjvSPN7rX4D6>H z5A4qzKrbhApm2b}=+V^Cn}JhD(^Nnj124b^j|Gq23==#SLtj_LjwOy^Vn$lj$Tr%V zwN1tMCxun=K<0oOd&BAC$=-D9w_JHLkZ*i|SbRT=`nUh%Kgs^rzq5gGb-&@{%{b|J zd}B#|=&)4x*O76I==L}Eo5P@Qb~~tz{Y^=2?r(Kdzopl@fZN)?C8=Bc+uhV}cUQXF z`#X}_+3!1GQqJlXFW~wP_>t`c%6}lxP5tk>>so;WK}iiB2z68cN32<4)e5+w17S%G zABc2QjoAv0LJveGHF_Y{O*K~4fg3vzm(=)y13+!E*t-Ay*dmgKI?-DFd0 zN$!f=P3{SuIZ#Ih!qP8#Li8zpT=|tsC%$$wvL9=_tX2%BICa6T@Tfuz2ako0F&tv8 z`HZ=#l<1NDOpolZ9zdm%84m$Z7@ghCuUq%&fnnn0ADqPO&L5m?oW$%-<7DuZWz(*7 z(&T%y+{wa85-#A$)vlAJlXAR7>0||AvuBF0KTlP+&BJn58iF)VwRkK+>r|WbuvpDV z;7vBE_|1@d;J!69Kal8@|6U`>_~3;^$-H{1eoB9Z{R2Ngc&US`{tBC6e()mw!W?QD z!Q1=Ohs^EV;%)#Ix=d0L796SU#IlT+htotK}hTgH=E!&ntZq^c{ z4!v`=7sF!m9Y`M<;9jC2_G0V3P2H$~juKDiI}km@VgwEKMc)+}vWqW*5a9#EZ`iOQ zP_|m0Ms~sn(g%>eZa94)&-2jo2df81U$x;g&O?i4nx>IFkUfaP$O2&U)vJfru@xKA z2#m)k8w(+7>Ok(G{TEfv1bXWxAeBA{1$>?bkv_O}5Cw#WTjGpOkiiqbegcK~>n9?x z5zRV!;;W4M>WR=va}Q?n#J3prtrOvs_Ma6fa8IB3kdYsrh|z3fCle=)qD5Ne#For) z5>{=U=*Ss{>AQ0>{m2oA%H3VOhtv01y>Pzz7Uv{Y;cov)oH(F=2nFuJj@!UJ#d|!j z>bY_eKu8+XueE&z@Jn3n zMKWuQH9QmEqnpn6Uc&q-w4&Z?ebHdfc&WxQ_kY2ixlA5mwU#iBg@BZ@n&tqBrgZt5 zm;m&=VS5jo^T#lourIV}ghqRn8)izdu{U&GzN#NkR%f4YbL zo54+Xnl=+RCyF(x)V<%n7q;~6duyLCt7Y!q?=tGU_f|jAtAmAm|Co_~d~f3uu!zPd zTAz?Q7y%X4yw~>$niyVvp9mVSB(F;Me{?^p$w&7G9{^VHf#d_XtS0sQTlb44;fd{} zt^3>eJKu&=ZQt*E0DDr6lR;7sL>{zOlgNYV2O(1F!73hxfuR-@9%w&c1#3SLAXwnR z(1Q>q^k77w2^V=#@{pj@Vo0Cifr?UiplT)=tp|xx!uA9H2aVPv!PdhMsz&R9u-SS* zipYaGa?L$hAlJf!C33~ue*|yBHExE~nS0S@VfR)988MQKkG+^fa!o&&HMz#gH4CZ+ zSA=_9sdG_;U9~p+@)>OcpVQ)FU)tjqQB!y1?ilKZfvReEbOf?h8LYzTyOCyic#@BNuG6*#K|b* z#@7?Cyc>3$UOSBy9)PICp3)wyli@A72Pdn@ngK(NNB;T|Wc9BfiJyV!@iTer$xV`V z>3?+|37laL0%rne?6|q%Oz4a=fNB;x6G6CFgKB5Ok6INPel+$d0$V|Q!!P&dr)n_4&w*#bf2FrsApyo&M-N3S}dc?UvKAsWR2$YWh1{r~rp zd?T6JUArOnDpXIyFn=Hr+r9Wj8^EP=@Vag3oo?U_+Y(xU*zWl59WQH#eT(Gy?!?Cs zNbFAT-m#6v6-`ak=h?wOUIJ!gSqK>2JyQ%LD0Ivm*;MS|-+UMa{F@IK9)bN79w|K{ z#hO>8N2-YF0j3`QHUYo=aP|?@+^h+h17HO)2B6A4oOuLb88bbL^af%i9h=U|4~ISp z14A%({r~vvU#ie2Q=h~p_wpz6k6<^O{J0;QU@ZU8C##=?;Z;8wc|?u^h=8sv7c0IE zUb#A8zdo-US_j-HQ!lc=!?%fE>cr9;wh5FET4PDtaZ}RZk6v0>z0PlcvFZ7f z6|VVcTVT)Im0iOi2Yh?z0?J9dY`~eml z)qqo3Ew7>N#?PqC*oD@mQC|%c?W*+t!fP=hC;@H~GgF8x=&HWDwF-B-h5c=_8g1}L zHu-V`Ef5Z|pby{y1k<-lL@T`(Ub`KhW`ZI;|KWKS>G}3~7U}uegvsc3t z1yMWSIIoNJd<)@Tk#-(SJZ>%0#N!!$UY2=0_c*xZ9xouIiW z9xFVq9$?2n8l^5w)r2vMlnE3k@p$@i6d^u$Mx)e^9s~Fo;p^8ux_g1sJOIj%BPuHm zdBxD8vQqV8SR)-&39Y!^wVlT4rd6JP{k*CnUB{r|QjL|OxBKxmLd_M|8v@;2Y4NC# z7SeT$O*SsV^j$=``z{9bA)A4VA%vxSfO4rl-h9GpKg}n?pMrepPnGc?x8@UpPq8>zQTbEx zCraBeN>5au07mtR`V)|<{zMZYt!+Yra|uzV6Crx@35yfy!fcr^hQ%>~;+LMNJYjVr z74We-5fKXC=Y1+hxnrM7ehOuh{8U=RXFT3ytQ!1OSTaHlF6LxIIvjLqA!WRR5Il&= zE!G&JLipkal=4f}CE4y(#TN~_d~%ENoiY&Ha!awm_82wvp6!s##FQ};!)BoiSJH4` zTw81iH@t>Pk6+QLBHu3Zt+lJYi@Mf76I%5wiJ@3@bwulnGBD$Ajje%~b{ zD4;4cUYmmQYg45Kt1TFW;4H^ulc*E1$?6owywxe3sr7#6T{*wY`IXMQT_U)sO4Y_S z5$o+21R#@Xe3f6DuE|{oHJ(w2gL7syxN*#{!BXXR)aLP}bZx${u>1YyyY544{Odg` z4jMfSNoa_yPz5wS;GR0*G5Nm#s{8%GRris##rNy)x))7~lV3r?S9HQxj074$ zWFE((v4>#(5EGw=&f@q0%ll>9`wh_x{!(T>j;+0G^YRU}(FA7aYx5@o2busNW#opB zBl7U#aSIZH?-c@6|7U&@_5bZzu-HChDXH5Q8#RMOoeOy!kq~n670CtvvefxFCXYqQ zg{eI}ReHWM4;!tZ77nhP8PmlLpN}{jgVR0GcZgMG!KI0sbt$ruW zR&DkB2SC+7pxNKxQ))cL1a;Aq8Cd@n9(JMW9a-zXqJDJG(eQFYJTnj;+ux1id9@#l z!!p88)p2$$8i2N^JoJg%<7sl;^uueYo8B%%uXuiJnGcWsQ+pTaCV@YW$j4zyt~C+w zVPXGq6k(0h^R9a$yzM7493HEDtbn_Y{#e?Y=gqR{V@F?Fx67T5 zH^emABC8Hvq%70c0A?gw1Bs!{u2Q%2?MClHLB3=2y5F|Zka+0!XS-=1gYt23)-p2L z(38PT{wqfp`8H-kgf7hAvPf>Ha8qYp`nQq6nt|=Q=!y2XEHM#9kgSS;rBnbnfS4Ex`{XG(^OU$Vp}87jlD@Y$i?RC&uKZD#%)M<({CctO}P!` z-_pze4J@Q9!B3j$F2Zd@{yX-{-K>~{FR8Yj0i24|8Nhc@&#Le!dw_M^I43>8x-qEi zcuK=q2`!<7{&TC_i?`QdH@mk7G0nAC{r`?tt{E6k&J+mFQ~y-#!iN{o6n=Q2eS!0c z7h=y?JM8G(VJQaP{L+Q$1x{yF@kB?jdZBSa&m&%FA#Bd-*p>gu)RWeEpwyGOCpq`< zWZ_A0DLh$1h|n4mlv0Q?XLT4^deSnhLuvw3GhvKb9VSqm)RWmKt+P5=@Yz1AgIjy+ za#m({mNILoguAxFzdvm0=y-TLyV@4gFvk_LzoeADQ9z}|Bg|KuX@|KwuqIpi(&T<*DB zYP@qX@SL!FF7O<;dps90Ua(|8Oai^jV!TPBR5aVAil;#1|P2HvqOCxVX=mgPj5-{T*s%Lgt_J$#|64%zNI>9>=jvAL&}iN z@gvUgca+byJ=U%}2Q5d~2{h3ajz+CHIYI108=J_~{6HMlm6BV`bi z6|UK{d6R2{T=Sr6a7Ea{wQ|-6T`1qV02j;x=R(vBxK_@FbcX6?sLQota;=+OsaG-A zvmKLbom@L-1Lq8`2R1|ks#W%`)?IiMju(&rbo7eHD>#cpXA*7=f|if4G|ER(cd0AC4Of=BD~q?^!W%Oy zaT*+d0iHdAgV3@^DtAF$s&beAID!PE_QY9Y;TW`ra3$~EYge!~sA)DVF(&P8Fve_z zAfw-C)X?w0*HA1A*P191yiD4j=KL%f#K3+JEM*$C~rv+DkymVZ>+Jt^e)Fl^D z%pXunNBgmL&y59IdenjHVi4I8#w!(Zln&6h`T zd)8&#o)sUlHiu?2$NFUcha-2&JaVVINmb66Mq9^|d0;66bv}H9aUQ;%=UsU)rlIl> z9+*EE#v?sk;G;g=!h_{&19()9_AU*!_+(UTaLafl1e&cdj@_~HP50xrKXicmi#(#J z$OErIjc}Z@W>1qCnwk)(_G74xPoDb&JV_`p5Q3mYsP^Mc5O~_T1tBt!W?s@b&j=Qt z8z>kE5xxlmX4QG7wnzw>OE}3K2u+@yeL>F7mJA}^$K$afQVHtmBB03~Oi@3n!7LAQ z$qwcbqDV2mJwt>T6lpl+tz6nBLILz}mw70=7Z@E1U*PT^4 zb5|X%69El-8gKp1z{=bi_v#p+b&TUF*@KWq;ad`oZH}}IP*#QE((ur$=#3J2@ibg) z3DkQr_ZyCZdRH~Oaj<@+&tkhcXx+HGuHmsn1710ugM-7vW|>tTDRyO5${AO@12nEm zZpRUDbmNKMb;+IVa~$KL`Sm`g*ZUG4rYAgE54B)(xdsM?hJ>8z5wj4vn?x#oj(bVO zL~dwgbVBqGsNfGd;tKqZ8~hL!eh8cV94?o~14Dv%PS5>XPN(Mq&BN(=kj5xOrmCYR z8IuekBPM|^Kja(HDJKpMy@XUqgmd4_AFh}PI~3O8uu%`E5yRMGj<`b>+T)Ha!U8Bz zycV&7bMJW-1dR(n+)-L~INy(5Rb@vNohu!MHyo*NM;7zEqH$&Rr4KB@<<{kH5DLa; zC;|vwz`pT83r<|Ze~uIApz#T<=0L}B4>qzGkS9cZ?5=h;oGhvapP)n|K`yB--Kt_Q z3Wp_D1NKmb%wbr;yM_;KbiCkimS#v6L$d}F@N z*JHAmM`BNzC8%0`0j}%?JVB4^HQQ0PZ=SHn@iw>PATjK5zJ={T&}wIen_11uZa*eYgQ_d{4lEpRJx`;Xcdqz@WOk4v_w5Q}% z?=ylZr_T_+HTHoz^ECX<)92;m#`BkeNf;yj-!+ha$?PuDX#&fcorcIhc_WX3=0XA(ztOw zL3YyOWmI;u#9IqWlXYt}d?2_M#v_wBNedeu#{pL~-!Y~%5`oBgbTz2}T+OW$I^!l* zXS~eA-$QckIv$Zpgsh~bpj{Jo_y6acIIbD1yAzcO=n&syLYGxIo%xY;B zTg?uw&_D&;k|7;okyM4e;Z^v%FloSCfNae9Af@0%8)Ff_z|CbMegT}3W)a`e4+iV2 zRUXjS?GIH~8>HOePQ`PwQ&C${340dd3Av>S7~8C&`ht6C;^;FtFn(*HW4`O6 zVGW>=*(0}5Rd{nFbcc%xf!fbvtuBB)tk)uzxCm`J8Qek0B5ltH7vOq=K-3qO5$hSj z(z#TDwe)U7LgY(=Hx#F#Oy&s5`86{?`L#Muj#PDAr{{jz_6qaUz?FK4YOS?#t(a=B z#djmbP4vC|@H#b>KonBAc^;^epXBZ|qS#`RN4JE@w6eRt+ggfsd{jU#yF0sSuDkuX zPEf<)O>Q$I21&4+W&UuHcVPs6>QBFpd&dgqy}fl_MqD4kIes{mb~HAIM{F!PX24kq z=ne;oP_c!P7LT@TjbO1EH+PNZ^g(vPF+>K(@Id4k$f0mSPIOH{ZX}W)MQP+m^P^Ha zc?y6#LktW`&zV|r-M*RTjH&r1@36g~%DkEOf~xUuhO?@OI~T}xM`TC0JVC&T|D?C6 z2{?B@@{5@EpTSMLu(a7KA2nV80S`d}kX{G~oJJHP@bj9$!I>-1lRz9-5P*Px4iEoa zV2)dugaYmu5DEe)6r@W9Dh7p>zRx{Q3b-owc~A(?;SrvT^e8-UP!K?fk6r!vCsecy z&%88oiWKs*m~Si3#^>;e&n0>ku>3DF2%u2l_=Qvq86J7TiBs>JGq|4jA}FNiIC(Rd z=~2L)G(tfDg@R3{I1T(OXAj~y0!T22b-O%nPwi&^Sp!iZ1kqA_pL=j+lxTrj6mxAB z`bOOpK9Zsvd2J)iDGjYT)1JizH|^Qdobz4u^-C-zTzCVl?G~;=*pllW8vJ_hPtoz_ z-L%i*%F@arinOvASwbkX6k9R?tcHylCB2y!SZy9Z-R7mhs93y0_?!yh7->Wj`35uY zp#kuD00x$vZ$U)Cc)p9FDJL5YCU$?z(JGA>9+dKxM@tqmQXYlBM*ETeY-@J5~U9ced zbSC2<3p@s2f!8w8GB~jue0gwr{};PxHrv?0HYa454+vWDX0{!tE9dCa4TkBz=p_wV zbvahNE!k7w@@{!OXxBJxhSmgI)d!3ayJ2mwf7Oi*%6#{)W>;snwSd>X#1go*4vzYD z8dN%KzTE>xh^fBa;oa^xg^HFnyINour;Lw6plkkA4c~J>Jz*RI44c(8oN;4V`5&^4 zUcYWk?P0VCkTG;Hw#8BdS=<%L$-CeGqwc+WS)5@EM)}n;dsbxt82q5|>{^M9LTRna zMxiRL21_E#H`!O7-RALex%BsW$MpA?I=k$bRcDuf+1j~6BZZCR99fuM3us)T?3a!D zhEUuJpyD{#9)+93b*kVf?IdlEV60x-t;3Y-lIK6J)f zuxBp0uNcLu5n{`+Wg&7&#g)6li{>N-TU46Y$bScmrN3P<}yz0+n_!)rCPEz8N519rxxM>a6qg7|gii z*25LQ*P=rPwGNTeTH&30T5SxGX$t_NHc2NabkbUB8+mK<%`gK$ZwJLmL5`5a^}5_K z$t*w+zrAdSqA*?%<5p$fP>F*yctw?=PkfjRATli5+jdO4zl>E@EPr+EeJ|UV`85tY z4M=Or#ZTWk1*@YH8C4^#8d9M279VC2VK;b_yXu?s=z+d~9%%ihk04ZF;n#SZ^{*`y z7tA9dLJPQu8B^Xm`DgUkk~dhhO~^Qw8{DOe82J4^)kE( zm>9w9IMB}!IX744t#IYJ#10r-Vn=3&wL%j})#qAs5VAEF-(j^bMw3lz&bI?%@UnWb zuE35E!eWK6n}n<5ww6N&*QFmna7b3+_U$l=ttS>Hv8IL!aMez0cWq%Cek)MY1uX#P)UpoUX z$Q^JcT-Gd;D&{d0GR2%wb;Zj8kq?z*3<=<;+I_#uj|+a)6Yd9%a6j%=!2T+|wMmyZ zCK1z|xmIajKqZY>+MEd2Gzx>?=cS(CAFK?4Mg>RE-$bLTmnc*RXY*?s9bB18Ip{*6 zw^)CjS3z_K*1RDW$_Ne?xIQ8pau8Ida8W9ysq}Ag7v&aiVvs57cE8-k5W%&ink;TC zB?&6JJ`fyqywz27gNrrjl@9p11mhnJ3JXk+0F0g*ngyywxW-Uu#422fYiGEFghMt8 zr`p(rX>iqPJ#W&#^23b*+3|g*ecw@V!rCS61rx9Cz+^3?bLO1V1ih%!MbMBlBD;7DSUgmO_g-Sn z8`Fef6Pip~=Py+fx050|JDIme&H93F%66z?fHZ|T^S~}UF7_p)XBr+nN zmNcroRIfJC=7Utk)P;I!0!lX}l{8CO_a-y}C{49U;aP#|>W~$(1O`m6P1JFHo~q*^ z_EIOju@^yj;3K*=r_S2bl^L4x%$9WuPHpjyyvxcz8=T!{-OQC8t zMfhh7inSXkw!z3$gnuT^A$VdY#|NpHitx`wW-J+(G)vf21PL&s8B%ywptdO@Vx%W# zk~2_55)V;ClJMIU5t>c#8pp(JU=BGB%*FLJj>Z@vK8L~WWejeO+Z}bZwutm>XigLn zLLV68XXD*eMB1bnx`AQ>6ir2>XUm*~sBjVjTXaChR785VFl))Uq*=nIB1nLWXpq9Q zdK_=oDq2LrNU!M0iYgwWh$`WGicp2g?_)+@eSb1Mjr@kEquXX`qqx1#sE-=*0w8hM zAH1h|mA7~;RHZY?eN)zWq;Qj--zd= zR*{>@Cg*g!jxz{3QTr*LC0g7pyp874;sOA~!3B<(Z^T13yKGJR`N1kNX*gZ!gHxGp z95Pe+DR9V7l{mRunrb3Mr%AY;#i{00n_z9+8jDQ(r-KlSxx`P;a^ahi(atz_@Nl8R zTBo`|eolq?M}rg4e{ezrnaVYQilhDsYYG4s6H}>h4Ws2Tc+bjMYux?<^DFuha0mXPy-{1EKDlPuHx}a5TL?37&M#p`A~u%b zj9g+YtHar`&X~t6c-0vTj}IYqD6uXBZNQJm$D!N!czIlOT^bcgAAli#Ml|pN4?=s>Xa_s0klARvBZb^Lq=3X8sse zVBQNzLB1+{hze{FH9hWl4WD&FR>y1Tp4dceoC5SE+s?yl7^@-5bUQ~+xr4+!qbr%{JW)19pWwq$2 zsLSZ0U@xp$6M$t&Hob((afKGWPFLBDFRRP$rh9v~!LBJiKkRg_xUIcTMw4B@o#b7g zN$hrZ`e9jVcNSp^s%vY&9Td2rJM6|(y>!{b?uZ-PfAPM08gu^cZ<#4X!w;LUzuX70 zzT8*nLrx2QCF?#3o!wwxn0-m?xlW?Hsqu(f;vzTB4Cup#7ud%@MPKY?IR6T_wY*?J zqx0BdkG0cR8~__hyDVwwmigF$QoeyW_sxK=Z@@2ID(V`+cAo_R1_-)THR~NV_6WT? zdFf^WQI!F8`r$PU#SklR8rCm+daxBKKA>La5XGTDyUnh%{;8s;%;oUI_?*4$`H+O39+I6GCV6!SE z%Q=!xRG2ijfE$@qe#ft}H~6dEjiJY|5;gd;?bPleoBcYkacAP-YrP0w!R>eWfv4>u zhfAeB#>YDFSD-$HPk}^z-Xj3wywd|ua<7PRv z{1Ujm~L-vpTM?^QPq*!hzgB1$5d-< zYYhDh-?&Zruquc(!RQz`jc3MHZXz`aL!T&3V6f8!r|@KK66cE`7UnZ&X2lg)9#1IS zkQ!UI;r+MHDBBhFPP9)o_ipV~oxNLks?ME}eYg!NvrhnCQnn}5`K&|n@IRcFUc6II z@|3g~KHX4j0{69yviUFxj}z>e^tFPr&8g)Vd+#SM@;%v7wq>;eas^aw)y8KfP-$&g z?wQ42jx8_NE0U`?=*_oofj_98xQqi1vR$}_h9k(gJW#$GLXMth@H+^4jQ6jfVA8&$ z>cnaF5OT3)S3YcNQz=Ig^Ouf>N5#BJFR{K3xlXAwUr=v^?dW)9?Tq}!5QW|Mw7T>L zdN@8EHzwp4)y8S{#1rb`i|Qrl?;Qb$3Y7AMoDBR9a6S0kfbXd{lr4(+YX*4G-F;YmsTT-?)wb2sLZ==K()$%C}>&s4+K^K(Z-;PUc z!J&%y;(;;0AEq<`M?pd#Rzyifd@~SO9zGrD$K$ugYMZq7e{0%?I&)E7zG71`{3kO- zFzzbw5|~>;hbisC-QsnA9O8+4R0>Tcwgxvns)b`2ERlE}C=ICGcnMY-!9HJXX^jnH z%j*_uf;C)3I5CtMQmLWF5MD8#A_H{Qz~CI9qtQ_n8;y;s)aVv!iZxtBxH4WHN4L2) zu3A#Ftl{7UGd62x44Jjr+IBn^LQ%FIjwj6|6m|w?YtPUaeNM$G60~ekZ3CMCQN;@i z`}DR@-Cap!J!(&5IRgD#ta6%=sNa2+K5Y5JpITq2AKipRKW+UKZ2(T7j9xi<@%~8+t&-E&E0me; zM%KshQD+L3emo0JwZ>X>4I${efMJ=cOa%d2m@4qKJ!Mn5>D;u+W3!Q}V3tD#W`Z*+ zgn0!tmou3em7B@WsKQKPMwPG=0G@0%Ij1sn;T6T37A$&bkWhN$-J0x!?tLL!zFzs4iVrFTH`)mjJf7 z*yDDLZwhu~6@EpPFcQKDsfH8%>`g?59Z@>Nwu60_tW)<~iZsA7x*j)0R?w3{--Ib4 zj0sfSop7s^JLASzlr0?a*6`H&RU6ZyDvaxC<#JjSs3+2oUTcb0S%y`{Fwv_LxRw^C zfrB?xawv`V373hgI;L+ipV>iu#I1`Xsx#sr#fQY~6~K^KrN#p=-xQm#EVeDe zv}SxWFwYc>urg7bfIDbRsOCf*<}9zV$qE{|t;x=$icZC*RD3EurE*jCDdn5?!?34A zfJ~Z^s2OuxbE=J*edXT~+<{#V=^ZykhA`2u!je+KZY@<@EG=T4sIwFIW+!*Sys|Jd zNokmo&>+^LRBEBU0Kc>tTU5MI8>X_DUBt$O!XlPfQcJj{8hhDz$u#<^yL`*aznom& z9u6z+p*TQQmvKC!av9;KmisN2hsB9Oc98oD;Ag0o|Z3fknfV{0n!$@*Q zQnDKzhoa3iM{E)I4?Q2Pc;s;4&e@$*in@`T0aWY*<0O< z1K8_(<-!&fxifkvQVK^^`Dn#V<|QkjNg!UaauX2G_*S9K@FuQksN;$;T#TYx_{vQA zjs=e4JEqwGL*1Le*LB?Yy|{PoTwaI(2@)WsG$zsl=yld-paz!h7~3d}H_Bj}%7N9a~p;a@|ww3eT>4ZXL71H+&24eq?lA z;n=z-*YR=PXV>u+hBwK1eBBdzetKQu8BctE9Usmm2g9VJ$}dYOgsY_B?S3Y2n=%oD!FdPeyz)&&0yBNMKgj-iU>h@no^^6x+p9_Oap%gmS(iId$^VvzHd$yY$$? z!m|rs5HY^FkR{+#3kz>AJOQzt^xMO2yt(MaB4R_{g4o`xZmzLTwm|z7ljfM9_ zgik~~kbHR1V{RWGeB=v$k$g--HuQO3UN-BpFd5ID_UbMl&kK1IWKES`kSk6rsB zA6(-*7f8}IFRUrNxaK7&f$y>vUR`r)P2rt2$JVkd^ZA?DG5O+6g_pE>^F{V0o>Tfw zl^$J7>9-tbM}@PG-)P(R(23T(xu)>$n$fj zR(NIIYwPT5%!Tjr0TAZlj;HS^Jafl$cNCuII|+RI;H5hXC-@Ws%FkV|-Bmbw*QvV- zZ{PLKU4^rp@lJ70fG<45G3-F$?vr;H-oE>(duZ;SXYMJyaL-Hk6i(dp>OFj3{rA@w-dX?N`oidjXEqj|-T2Z*jz4~N;}m7is?4zsg=aUM*kGp! z7v9@&n1W~Z_OT6*Z{R%RXEzjH+3@Oy!s{F|%(=$zZ73Yx_{2s|C4P#6C)-c97v60@ z+))_qc)X+V1m|rtD;=+N6kh8%#qIYyzTZ)JuVb`aI97h5TzHbRfEjA}wQ}K&@|)$t zcgtsaey{vqx$t=Blbwa9I-lt*Jj>~_)ZBTpv+(`Sr@LTxUC(wEUhF#2Rd}WAwXVYJ zoc;+GaHc3Hb3U=D@Fd4`f{vTMyQy$;(|g>GZXVrScyja8n+wluer|K&dCrxfzRf2$ z7vA3d)E3~k<(Vyo7q+~#rEp@)t6K`MaS8&oxb2N?g}1l8v#oG;+k4vzhuP^)Nq!5@ zvFV)p`7NAeCpfA1J%c~%zBhFH@n;B%|M;69=k^oNuo?9eZ+wE=Pd>v&)=!@LB&q@@ z#ur}w)Y(q~7EW0$y#DF;2)U`t|8|v`+_0{2$7hjH=vIX%@R@Oo=c(|06s*GG@I>J7 zgy;DwyjpYj&Uqp{Df|D0Kn-scZMP0n5MPhB zNe3Kba}5U4b67+#sMIUrEtuGo^-tCpp2B;G>h)}W;d#@--l{)U&vAWET*6zIJab9m zS#}u$Iee~#w{UX;`Ks~sEZYM?I5tbN@$cQ~ETlh;hIT?4+iMN4H56WFb058K7;R+R z-myk(ix(RiAFBvxTcfks`11Iy!jtT8p|7)^Z7#gRE)bf+$B!9xN*2@;+4EbnsLsxM zcNVKEPtL}z@cL}q3W7?)=7Pc-MDKGu|NHX`@610tpHTPrP_78oK>Hzrz3?sp)mC_#NXx9pXqzY63Qx5?-9{MZ3vEPMo@gt)()L1|e5)V&1JgqQF*a#2+rLZ~Q6h@u8#3CWiNqpN) z(#Jut?>C-pk^p>*U!8mwE7OU2kfEBR=wpg!q!^c}PW&XLA!V4|mZQ<11Q!Nau>(DJZB-ZgSb%+rQMsV-G&?W~&k} zTEHmS@>bhhZH4c*z0+1W+xBi7k;%`1sBd%VZw($DzU-Lr=*i0pPha-xWfsgiMH$1j zCsgK{%L>nV!Ph~(w-%pVTzGr&_ZJg`_|9Usr^qrrEYH>p%MPP{k1jjL7v7gWu?*yS zYFXiFFsSgXvR~v7^}<`r-UXqiE|ct5(9H^-q~H^-+dsFw@ci-@mlFt!?W)}rpjRE9 zsbrScwG>+B)fMI~EL?X7Bgwrj3-lf0kaD=nzsuG=T4Y6lm4+ALnvb&wwD2OU25=0S zoG0Z6A1#>QQ=TO){uBnXuwEOyd}M$;8r?b><1>zp_{709kv6?1j@Laq|$cgES=6*{cGYVv}vxV+*y z?bbqbs$eVEHU8?|x|HM9e%b1mMlOXgs~lO8P;Rq5?^Aa}Lm2>ghrQ$306MK$E*I;S z(&X=UmE)ybJwwA)efiEfw=-H39n5NHyw--1uh>Ilu_exJ*KKobpq+ZA`O;#pbf5#c zdHMO4Omkdkk*Yew1N8Dz6>UeCSyfterHOEO!RU)4!E#zi;g~h|^O{Epf z)(>l`I4c`PE?ffW)lM-SsCaQEI()BgEY8oxWi)pc2v+LLgQvRa?b{Fg=HPvi{^S^!*p|K)}HxhasV@q>f0*rH|yJBfQ_**uR2%&Su z<6xL0H#pu#QUDC1guok`*f5sODobaDQ1-;^9RV}P8?iW##tw`(Rw~^-sil#)*s86h zW=>CR2Dah#TffTj{ac$)Yr_}Q+SrfN+JfW9!C?rt81|3FSsHU|To`7I#YNyZfLdLF z|Bhr&*MUfeN5qH}QN|$`{XR zh8;Kg)j6~mXs_*QXWZ&#|^6j#BJ81gC#Hg+nb`c@^D!sm0b2@ENO z%f;SCl4$E>4#J<;@kaQM1}KSE$?#nG1?FP4a4P)Sp)q?dSD)dXJd|r`o_SuS0VTki__bUv{3m*sGyxi;-CqQwB;~{m z1Pn=tPOlhUG?m<0b=JX2+d z#qhPf-ZX|5{?(zfAj_HtHqT<|m1NRc!of*q$wYo;4^wv-;t&1JHDMw2$$979j zkGc2}JyzmcF6yWX0gJi7LOS%^@)#U%#F!cS`wHlAR94&}1Y%85}}(bS}^S#Y>gFP|^DZR~|J6n=3vN z=Xx#rm0UOEY!;I{mIquLRKgjS5Or)*UGxq!;_h|{Z4}4%N zUKsmUwsNBF#yi4)gi2#;3z)+7G*WS1{uURUxlT7OisHmVhf8%uE71&~MAbY&GWEyJ ztXN!$mmvr-kXkxAK|s;6Y#px6v8l++v2|gG)TvxtZ5>(M?h(O3Hi@Bk~##OOAB+IKTy^2N~IY3 zq{@jW2$Og-S8Vj#(nAI%Y4U;{{5_Qr?9k^YCdQ1*AGOK2258(2R6bUT=&{1p+=)oE z8S2TG8IC1vWq&vcfd3L*9i`DS<2`yr-_=E(=iXj$G|!cWy(bm7ym{YVNP~73S+;kb z?p<1mGdhq`!J?HR71`}1nAa@Miqo*Pc^&Io0|sX<=K>h~H!ew6%LMI4oVLM|5p;LG z(S(}N;To3C$9J()#6;cBU)W`$&>9hnY;?#-UxQSE(w^GGp#Y0N<} zO8dwHy7x!YDx7Du1C#)G z$&==!vEvI+R@ut~xLk3L-*3z}!R!)e_y9D#g)Pj^i!+|7;j7rUNkSwN~PXae+%{F{3&KwV~faX>MdT=rPAqGq%7GUWVhR`1#s>sU}ev4shAULL$jXS2$9R(;@ zNcbmT0T($@y$ms6V*du^ad&AzQY$VghClcUEM{YzLkyA1|G`&Kp~5#oWWS$*DI8uR zizI+?C^v<%qI3uQOtlp-(k;#TDTp6LB5|#A$qX zy*$c{RRka`_o#l$DNo!;(v}HB5H~i*dFD{A#2bL_5uq76Bc)@pH8zA!1B}d3s}a_Q zebtnU*M_~QQo%Ir@TGEdi!AbgAZnd!s<*%>6+FAn4F9rDBdVB*DVw`7{2%8pUV6+h z>R(_-H-s0A3D{IL;Hr%C;?>lATmFXG@#=EC4L26QsJD#hb$j7)URcDNv-B&*EzV|Q zH=M1@-=MJ{$GD&p!+YZeqCR^9t_}HHnkt*(5@fY%J;Ru((CTsi3cwwt8i;#tF*B?{ z*q3wFrFJ&OCrrHzYm#V_K|ktg8sejmATp~1yYAe99Ra(%p~vj%+~;(xXOifgSp+6$ zTt$})A3bR$0&=5Gsa!~DYVB&P2$D|tb9Z7LSl?Y<$!M@g@a;@;{!gNMZp(j+H4P~2FEzSORh^e!7YR7n6-=3l^ z$%&s#TbxpA&IprDKeSsWvLt?b#*8;9J_VrYmsrfNYJJ(?sQ5Aj6A|;v=kP-KEp;{g zi+q_$K~ZfC|L9=>xC$r6_Rne0N#h?s>}E9(m})eR@<7Ry)1`kG#ttiMu=2}s06 zK6l*xV}_&p36;cMO<;|=I8Dkwoh}|_7%tPoFM%6YipjL3;~jf; zV_b5`=u|$?OY!nje1^1$k7$viA)pk<{|5uiT!V0lP7`JTjZ~S?5)W%NI7|;Q%n|3T zxDdT=Kpg7Fj9)?Is0Rl!C1gbR$4mb(#uI`EOje*Ml%6lDjG?@%;(26o>A(qQ48IyP zhW(Uz_^tb-6%oJgPvq>!CG8wX+Au|D9k zc4`j?L`kLF;_S}Yduio) z%pq%l#$FZ(1!1s}8Wy7abfh(`5nN92SZM;`=wBa!0fBe(IrO#sRn5j)@?p^JB*3F? z#xO@y66NgY}-HXU{w&%cMEGSKX_`J9QE7L!5%12~1_`I;dAV}=A~HCknUM;UibBYG$2 zp`$-GRpNNs$cc;%H53IsJm473Ib%|x-SZ&AE@!#0S&&xwFhJKTEX8gw;LfbQ3n(YY zOZ`~lmF#s)Ysk!5koQE~1Sn#FWOpO)f5);{G2F}&LaE47b42|#bsNf9bFEfitICaQO79=w%Vn*>c$!{?0 zjc^ma!5Nm;TV}AZeTE0-d6p?@Ea^o)$%Q+|)^Ud`^KI-a=$GV_1h*_M^RBQ8R}7yo z4Y!6X0D7zGq%85yX4R<^pxB~>t@{WP@@=y?T=eJ!=~tFDh?ikMXqP}?QuzN13YI#ZC3wU;{*OUKs2l2|kXX{lZOUhWV%IW3|_EsvZo7d`N) zD0>T&!TPJL(3sH~#F9gDq>c@JK*yTS?U;3l36Pm|u^khLXU5V|ql-K%%+!%D;w}-L zh__gWeeDK=%XF)Bl#js2>)#Bo&J?4@q(s~xmFD<>awuc59aQ}LK`D`D4=|9ET0Xs{us0R%i zh@k3{)18i(uv7R1!i>Y5X64uA=;&{`17RW*pq}dR#|+&-p5KQ{ub~^T+c{xRhPMK< zEC&kAV|@#=A%9ht7dnw-0k;!?-wNx&xSB*Ll)uqkL)hA#r&T*0v;m~KIPI9NFbqTc z4?`4QUmymqx%h2UrVR(Q-0s-m`X0s|jW^>VDuzvPkSgWS%j;9G!WK+<>*l1+dTOTU z)s7&nmSGSYDi+z*BpneJKufd1A6fdd5zxA}@LUYvsF{$omTT;+)eWUuJ~rn17bZSa z^FD8$%K?4y0hsH}1Rml8nv1X4DzJfPOw-n{X%n_rcxsxA-{;2~7oonmN%xl+mVn!K z)gpdqNth!x2v3wQmrVe+keA1^(Hw3v{P|>tKNENw^&s+mLPB}cl%%$dGqT!R#h;7j zU8y&11UOlp$6ap`w$zVmEHh67{uj?Tapy3SRNtC0raYj96~jNi(08r<>b(CRvzIH* zg>^bwfIQB@PA8XK_*iv%H(;OW?QomH^WG`p>5jaB`!~UK!(Ts4O-lxbqMcmn*MSDM zAxM|f@j#bCucAMV|o9M4vbe~{v$uOa$tQfP86cNuZU z_BTDx{Y*U(VQfDNHYT8BQLa;eU7efkYCl{2wu{Twk?~|NQ7xzJd_hass_N1wP!4mY z+e^2H#?m^y)Wv`Lo%pxS9=oVEwX$_$ezp3}P8=~4^Ro8p>OOfvgPvC%^i>)(Kk8}Q zbzz|oy0&&7N!|S4G+p)c=fLv*KEk~B4Ao_k-RIL?cl7o8DtXQFJ7CY0+6gP;ZHL1G zNjD~WR=m11qS#%H!J=S-tBYuEazJns>mT)1s*Nd%Dfx(;w-@q!g7p8Y%eUunjMj(W zloNS|Ny84Zn!`VVbj;gFMbp3}$xYot787nw&#b5poJvrsoHNKGyg>(9W~75eZCy{G z;H66IH21cx>PVlKxg}%Oybio~yPnG?MCBLJN_1KQVp&^ZcJxC-m-l@EW5kxhrAW!S z_|0=R=3Hk*4Sn4rE=`sYRHv8w^;%#B4N-Ft2~FyFf8aCT9IDzZ=;#J==au7~EP_}N z@;sLZ1-x0_DPB0Pro%u@wS=y0z~v$FwSfw5R`)CBSxNkxi~KBHvZgP|#al?h?y!ua zRJ|0+X3&@>T@82R9$m`ueyU~_Kz+DG;$b?#`zfxrkf#U`y{l%QO$cw6;wNmL8h(lJkudq8-!RU(%B@a~1?$zQnSP&O_oI0+=$^G{} zC6Hdy3Dz})Z`V~oC79Q5%Aj`)pJNz4=Qv27;dq?mx0*#%Q8U(KV+S*gTb{HVl71CcBYxBc9wG``B_vJxNBs`PSF zn9P4#&a{3>3)vo)iTAHqdJro27BJFR(?FxDfl z)uY#%Jj)i=!b4;qi`XmXavjMGcDqWo$g}j7-e`0WtSqgNmI#S_RhcgHCZN5lB$mN! z77YYovpQU~lX{z8JZ%a~zPQnA#Uu2iOoay6AZS_9`^yN{oEaX3Jf)qotb18kb<2XW z3HSlwhy;XB2ha*1wO&1%D$jhsG$`}g@%NKPTl6mhW7u|V&# z$3#zA2<#DG@S-)#^`H%mSJ&Osa#*%TXGNI7ImHpJ<|TEwXcca<2@()|e$^JW%9))H z(_u7^#MwkknDwkaTJ)uQf?Gz6pITC#PSad*=R)6g05#ldeZkd>)AW1Y}JUr zn3;6XtE(cZB_T@q9GsDrhc$s&5W+zV#>cc!xk~qIK$YWhO^0?^7!Vao&{0LJCQLQ) zh&<)H?3y!=FP(6ZTTgkq&oWc0E3EcAM!E<0xTcja&Zvcw@c5Y7h{o_3sziXGm*!-x z9T-W7z6_y+_)f-X=G$|Xr^FF&kxC5?>lgvU z`x75t)^K_oKUc+XleDAWtW~SGx{X6j;{zL~42HY-b@;n9-&Q;;DSU%wBF!wf9EW&8dn}b6J6~x6{8`FxP>9yF&Fp&s5nM{J zd~BGx9>}vuV2DCzSsFt*eHi=XA8S9G;cVd zP+As*A!>V+g$cHTEr8Fo8*kr+eBWTLpgtfTXu~v|CP}g9r zkdO!tVQJM7xeijcaAuV>)MDQ44O9gLw!%5+-QN5v`sQq2jTI`RVuzYfb_c4fgtpjK z$|e)nSlncd;bdjapdoxx>M>bh$JMNlaJM$$MLUZ1XDU-q`MPj5PD{22(SWGrZncL5 z(J+##D>f4zU>}a-&?wg?>sNjgIfvrqyk;0fdez}|)g6JIbc%=rX^x|YV;{nYv~Vyd z+=2IrTe+p~7@MRx(bl&q)EIt;tpoMkGg9s){XspNx{OsOs`FxS+L4Kj_QWA)w6Erz z(VjRYhEN~;8I1mIR+lF;+TYJPqouHu@8oi)*$={|8~GX~8le9ztPvqupnXvynvayi+$IxdP+dc(Clrq)*C&RH3RFz& zV}Q)|C)8O>$N42OD1G2AuCkYe=8_ai+f=NV6P+){fvq*lG6(wI~=fMgZi$@3tC4qj- zDvlCQqzV!TA2Sm}{kPOa$XOJdmrR|S*&@Od0@7nSwn7Vn0M~|8Mg#V4$1b*fGyzb6 zZITOplNu0Upf|d)l+6Zwi5^&(K-Fxbl}<^Sl!u5v-w)UI-5p|NMkp( z_~VTAJZ|;Dv&D?1H5k`omM!7@XoJR$ZM+OZq85enW?gtgW8qPVB8^c)vTmk0XOIjf z6t^^Mg6bfBJ79pNZ(7b}`Ku}XR3*+s>z%D2*ZIY86+tfuGQw#nu+*ic5XY2Ng67o( zj~ls4^op|~+%2~pdwD{*KdcL{AI(+FBToE;@hq)(G(H-0--vM&=taE0DPq6+NJZR* z-u=5er8I|MHr7_9@9D)B@*Be+*E#p$g&&iloP-%FzRlYIJCDFvU@-JL7ybjLDP1Ur zzYTYs$kCKv45zG*)5G5(-xQ5;u68lH@>G5lN0blvgS80?g~W5tQ! zf^|nK6-E->BOkf#S4{ym7J|T}+Ug4P26r2V*Ex5qhXH`GbBT_U^}vM^*q~xZW+kxG z2gtzDT;EEOf^PLdn>Q$v*K7#r9OGkRx4r=ahY=302C?9Af{FLqyREVQ>lRKBAoxsAzf%n* z>PaR~!V?d$B+u=iWT;Rt_K&*;=ENDl?RT4Onht83%xS=OWlzyG)ub>@rmNSGnB}7H z`!wMvoJ7bMWe3JDE>OtDY@A7mMmLLC>$;6Wic^{sBNsoVG4rUgOQBWR`U5a#fr&V< zdI50A0f5E@|5lTtn&%jsEy z9m)1&4(HMpW6{d+%A`W%BVqUe%Rj`Ah-tM%QX(xtk@`9RR(ea14XqMCYghJnH=sq-(zcvU&Mi2u z#0t6N=;DbwMS{v8laR1Jco+(#v*1Qn@J~4{PxqWHrb~K@mcxhU|LA7(#;|8WR zBmA?6#kJNj?i{h?W0v-N=H1HD&#$z?uyw%-uF4{JwSC)|+U<1A0uSni0J3 za#?CTzSuLKrM~)WieQb2j7*f}cCGqWXXTf-m}dY39*fV76%sZk58F{AcfDxZ zjVm{Bb+HI)wek}s8aSG2QB(Ot-Vvd-qDEwR)j03%@dw#{ohUD}LTDN_uhYZde0U6S zG=?AKE9qI_oba6FnQB*_D`w$rLW8Fh8t7CEc~hIUYE~n;*=5BJDDY;$on37grL?$( z7v5o_q)2dKUoJ+XOe1+KnswGjZluf~n&@9>{|&wlh1%Qt>kaSB5|fnI>l)^R4FkaE z8lEJx#Vj{)j~f1(?X2Z)mZH^j-;4`O2b`o(YR7Mm*a)TP&i6!_HKkEf{p6e3suf)q zr=}JMS29iIcq_j7fK7Ty*rNpto!BCK2rnwIoSfie zNX*O1758+eV9lMWHfxD80~3;er|M(2x|65aV@y@aR4rPS7;(k$MHcbTm>VxQQYn{D z8&>DFe@^R1*=nyS<0H(KS%=(rP;Xm!i5-EcNHfEi3}Tr+2IDy#TON%wF$wVDjB;EB z)fu5~0}D440*>&%nY z$5`CMEuT#h~Xjj*DC9)lYn||d55Q*3m72s z9%b-9w?&+q{R9foXulKcb;$z%-)!ae-+yjsw#ke;i;+9U8|@ zA*OZxyuduS{CR_NS=G5zmdh(0_`uJ$_HoYC6czeQB7J zYfjY>i_#djPGFx+grsAKH*Jd3x4Aznd=YeNQf%204KjSG^t`S%9Su1lXEKDj)$+oU z#0Rw0Ao!FyC+sVv%z5 zB+>7mNcXd_;_bDUtUs48h9E8xFttSkOXV;Hk}vjARMCFE%x z$vn-xwoK1e13mel9S~Wb8|V{cjxp9^_^4|&0P8Oq)HfetKn>ydzzC~AI_&g&eF0dk zSto00@MCzSW)WO#+((%8_3|}z%3uc zXp+ncDJJumv z!ya<0{%H#}5C~)OXjUN%i@N$Yc_)dyIcsVKKJUc+0v?=SFT=@N@ zeE-wGR^{6@!SEfRkbuEZbddY=o6A>S==Ca2bDnHo^k=a*AE$`%s~D5}vU^$Q|v4CE2IHlBJ53OXA|Z{(DP zb}jwR^i8B%H#Y+Ijd71LBYc8xcaC7?96(HCo4XaDgj%m6UkGJ7%^@>rK_vVkLeM`mzjl9c=wP@y6NFD%r!m# zw?pOAB)C-?*a}Z-YxvoWgNVp{`3V`PdUX1jC8U*d*u5vG?fN`hHdtNys0oT`+mL*k zXjY|K6t^`kJRn6%gI+)n$l(EzDz`BXfa#rqeDJ`ao= zif6{W^Sq2tcN(1j8io|QjOU_TEJ$8&Gzx=^jabsjR2*hWXK-rE8o{#RJ}@wV>cH(Y z1~vfkfYILVMK-FovdM~lX*zMuB|$ZQvZFrYk!ep50+sa*`K*1)K`|P$0y@24NfvZ^5Nguv5R>L!F<}SBqvTo_%m^B zF>kF5Ds=dS)N)bUl@;6>mw+2+4q@MEZc(nd6aHKoe%ZW7FtQ(HxHGlef){>FcYc?h zCU6-YTcWrK&IlcM=5J#>QWU>HGRtUxatJJd56%n(|J0gj67J_SX%}u07k$7;VM@C; zLuE=`=60=#=nrSt#;=-(5x~K+<)^^wV_>ZTu!H*it{_c9l4`~bEmV$5oHa^VW*t0F zs9fAS1H+R;X~(TtC`t;+m`R@zrg$cZPU1A7x-qNpAZm_f{B&x4Gx>|csxe`NwHQ8L zrOdE|KqQvlYqH|ZLmAzM%h?<#@c$6DapH$B;+s>4fitz!v$9!oVtz{6r^q0;P_e7FG@Nm zWtwo5bz##oQp!Q}4Pc=I>cwzc&^8u{oPLrDhlT?4{Isi0EXihgAQ!Noy5jLrNKe6s zQ7D8Vqxf=M0@32$kljY!w5~YQPFF@@nJ;w+v>o9fw)JSq8#_mAs^_P*8D?shzCWO` z{bX*zoF;TyM|~5#25EX|I;qDtN~YQ_N)5xQ)$Rbq^l&G-sGgF~Z$q5EMb1$z;$^m? zdt zfXitobYh%(G?rXFG?sm+ri7h-A03Rb%ddtG)=RJpcX=H`F!j);W{xZ&H?y$pxbj2A zV8~3gzl1n)t2ceyS89%{F+a;8AMSTu7O!Ddh@@d_qApQzYBo-7T&@}DX&Y^7lw+39 z+vv4KDR1zm31bbz9PzAI`62})Hz}~y$bI8Ea$)1e$_1a!1A<{17k)N1Rt(o=af;;& zRBr7xaLl194JV_)g;+6Aoiuosb@WS|-7NEH=P|E@0W@FLXgiGs=A{&q2Dd?!8UuES zue4P%)e<&KOYqb+ETwek}zgAPJ-UE4t7 zy7qY{nRDc6C>Q2cz~ojU@3@EmUsgO?qgZ~-23v`r)D^|DAGFw^`{ae}S9tXgCaTCs zrqYiwvsmKA=ye@y$|b!YUpKx_2m2B1a(7zEv0ccp#Y9!DsJ$2HOm90T7CoGtUA6x7 z4i+b@z6)5KI858(UFJZSZ|uzY;}rh%$R|oNb$Fd|HD-vI(>u+$xP)!OJkJhyYWT$Tw15rcxvW^Hu37zWKV_}b$Kz?Z;#p`rtLs{yznT;~8rOUt7eTqgh%+S?H3iD$KPI)aCA z;r&*c`R7|>WWy-*{%Y{t^@#*3-)+Jze?+e9d$WkEh|nqKvk1&n;* zLk5iW@9o)DJ&O?FFyWTVPH3R#JkVNyV~R3UGe7jc;64l!k*LTRhc5R1Jc>GAa93-6k263XguWpamvq*03cd#L|qbq*g)r+t( zwkS@tdggY#j!vy3Vwj*E>lcrb#-X}Kf+OIo_|fFlmOnR5=L&j%xpsY(ew{=GowB5E zgA^Jt&Q6w?p&Ej;FK;+IeP#YW^|-jtN!ty!qIjz8?FrJwQh|Fcm}*-gMbV&^T?0DC=Zcut?_Qf;L$ zQ)jt+srsZPy)U^}g*;cL{0MMsTnfH*v8bWtah%x304)|MIe!|u{1ISS$^#R34Hw{Q zBc;eqdc-lO%(9;q=-aSKG~R`6k=`_w?-5_6%c(qBjT}8BvX|o+pRiJ;{{m;NER+O+ zQpKRdJoA|#>!_<<`ESZPvYR^2=Mf-H2f5Z0B9@V#|iORlCCKF_1bo`j=Z|oFY(w(^lIc zR&S=d5?XA_JM=!EH)J+-QG^q2g|=I_#?tz@^|822PdUYHFVi>4N}O3YqBB9+kBg$c zjI}*(*rQK6#0v42=!-HRjmtK~JGaCVecBT5?1)=AVdY_EUC*i)THRsgF5B2X67Ss1 zwWCvS-+44HYww76Zi?4);J1}sM(t!!cD>cNQ5);Yy>nF6+=;1QqN-6E=v2;D+j!4! z=}=rok;8G>Bcq$vAEjGVvusm)Tr(Q4-^iW)Vrx^^8h(k=-kg4ykz&=XktdUGtm)u- zCq1IE4RH;}Md+3tUzSImbdoysyOBIvUk3JWRhRS%Fx7sr~9Qb!^~eYUls!eZDBTiG~>%#g$35 zv=Qg?*cER9d>iy-w_E7c#yEe2`g#@bZ=#1(&>62|Fq`6?;DEVLu2Y|G#u{}jD)EsJ z3{z<)%V3~Xq}y9?4$Y@)f(MGFpkDt4tE$g?af(2VZK`i+Nn|KsDa({Dv`GUxq&mRLqwK!E{3X<S2?otDB?HMwOt62+aYF&im z^VTwtt)p=*g|QwsGb{pgJvD4%fQRU6 zhdQ}dFDZv1V6oWa1LZA%fTCMzm#*8D_8C}7Nw`sz#4k0`@oiSG-D_0T2*M(nTQ{ty zW9l{i;EvH-eWhbiZ7Z(8&TU=l9H|fb=+;}U6)qGw0=xxvK)!aH0(aJKF{m;8j*WCs z7ol6pTma%VCGi@Aw!v$9dA-`vC>`<#9SDb^JnK4Ctg9+%Wf1L~{Fz^IJN2P6u6UHW zW$x$!Gv{Lf7MjWSxB}*7(B50Vw~N~Zuodb|qjzRdD?7lTUi}dEueE@SdW~#_@kUw} zPJjvk6Md;3oB-`s7@};^f%^io*LFYIo3)Lkz8wd0v#KRYjJhSRD6%DHkq(fXuMsh$ z_3Pv1L^bf0H1z-qYgDV$wrDe6Pl&3Eg3uT!xWd82doXlP?{)s)HAfEOxp7hZ#?2}X zGH3|zM~sv6bDgDi`X&gmlSugL*9AVJ=u45{M~=m7AxEi|4_4yMU(=PXZlxD}tz4V5 z^tFY?;=6j=90?Y2c@U%Bo`ja}u+XPWWW#v?h5&Y(WG^hi@{@Ao;V4pBEY)2H`NLV& zrlsYtYE3GPH^Q=?r}H%hQ+SZcYMk^IVl}NF9)KXtP?B0#7I2v%#j%Y@G35Tb6w~{N z=@213QKrjYT4OK!$-?to&(Lae8npF*(-0?hz>-spNPRAO#qEPN$i8pRmxI;w{hzUQOakzDx{Pv-~ZMToT}$C~?OoSsX3j)I7QV zpUtwaODJ(Ql6_OW3WecD*c-E+O3sQ9^XG}bPP(#)Xbl3-rcn)M%}7JUDd@pL)qUB5 zt84-`QVV5HX_?E7lD?X-o?Bk^1D#ww~K=KT9r1O(pD@KpLk)nCF^XffyGHv?I5agTQRGR!F zwS!^uFt}FGff^H_4IKeXkRx|YssAr=&gQrzeI-H*2=1CS)3W-p98FnU5!rcj874qe zDcZSRbAykFoSQ~urmmB|mwi;UDO1ADDLdiAvVfVMfm8CnGxFS#c&W}p7YbOnm^)_P z>@(D0GbJdt)`p+r?|Jr_&g?Q)YG01c$K+>qGeuroEu#QpMa79cGgIWIVsj_jKI*=ITme&c6Nq{(N!YCz187}VfHXZw zJyTU;^pmd|4G1Z6F{p z-1(!BGTYE&`H8n9!&Hr*ou_SV>E#B#?c)ugH(NuQfowE3-VAv}<X>3-r7@q}{vz1DO#>E)v_z^4~Ht}fcwJ*?X9#f}p;H7hWYliR5id?PI zFK6|#;VGxXraNVk<#hHHXL95Y;Rl?D<91fqllVg-p>Yzn?aUnZL~TM8CT2Aq#1jqL zs{?rim1iiI&Jj9i=RKT0U9p`foc4OV@8@JY8pnIE3NaS1gm8;Tg-gP6&Gs3-ug$6w z?ifo+kb>5()hTF>!A=vD7D=$ZM3YhFX!CR&mNvlBfS*^7q3Aodv|ON~z7@tzWJ&Xk z?dcYr@ZrqN#@VRdj>gS75rrkdFvlaeQ+Bz=)7s-T6oQO#ZFOop3I!Ln%N*nJDr`@Fq5h6e z4sMa4GrptPUop1aR{h8$l4~~*n_I#U*g}e{8$`V*`vNl(8vqiuY~bO9@~Vg#&f&Y^ z0|r{egN15$%t|{~Z6%+j@(nO6fsKv~WWV;Iq{k{&!Jw<2n5UM#U4BFxyD2O9YG;Jk ziQN*Ry!eP*u@jzIb!jpqbq?Rd)d6oa!p&LX)5;tZn3l2!o3N*vd{-B~MZ~zFNJd!t z?8W?e1z&{U!h*=M{{pC90=8G76~lS2VzmeJTyW>~zb$dr!{m?PXl_Z-scN^@8V+ogT}EHFVljwL5&<61rz;mI-E#?>SqcDW7@tet&g2zl*XwD4shiC%vf zE~4rQal|DHFwo{%0tG+ykX_=9M`9PMxb1&&Afvrm15T(OrGwnS_uV3;F3~Kdl8UA^ zxP0-5{ibzAMqsIFwu*++TCrob0x3Z@lyo&J9ZDO$W%)vBctK4;34@%k&oD|Kn+j&A zfm;9+M4RBmkwi1WUGo#`jCOS6A~7RQ;Cvl_8!UDvG z>1oqDvgj@b8JH4NJ&RzR88c%kJ?V-`6+rMM4b_P8NJfckP%w-r593`*W$6Aac)9|} zzmF0ye2cw$%%_zB_>4={u1cW>5 z$A)c4G@#d6#l6@APmP0$J*=?Lr3Hs&?pi|Is7^Q+dO=G!{d z6+&KC-2D@`ifEGW8kdw@7 z_%i&h#c}ARgo-$q&)E5`RI+MplYN}CY&_#9G~;wF|e*6PDGYOIWEDF zsGaTUHrt9rW70W*+^j$b(?kpp+P=tyDYhk!Y>I24Kwjii61S+z+?jX_)euWRfSrzd zCDG%@_Hi(@SEQhq3Q~aO<$p^7Y2+8TCTI+R)BUJ0^}QNIKR(T1zz87$@HF+3y=98| zVZUCM!tHnpEGb3caCez4?=l;#cZoY(HC8=SK)eXJeneGLS(lC5h>UW!LPTmR^cgat zuP;{8CU`Lb=&QnXJeb|;;IQyOvh!?aM`45lUikV~CKDF$`hze|RNwqc+UoEJLG$F4 z_RX*OT>U{9CowpVHZeDS?kT=#i_dXT20qp`ieWC%&ajR4e%IS2g?Ps3vQBH z0g;Yc3ZFTxeaDR+k*kEI04RLE!Zb@NUB=;CRE@8iZ>=dTL{n!eoKUO`{3vcECC>Fn z=Zqp1`_-C}{P9;5=WiL|S8HJU#|Gyq)2D{@#SJ|6hF`IfhhI007P&(9Cm1VSif^KV zX55VPEzW{La;9h)SJ%?#3X9(YIS8TngNMbObKwOW{*3U3ZH&>g>x)xAFJ-HWl-2>E zlU}M&i6@#om5)@?e@;<*Gq9%d084{x2qzx4P|6xjjW@3i-=ey-0#x?P<*R%apC#dC6*k;>z zE?^HVOH5!7S|SxcxJO;{^RxB1nMY)l{U%vFUg)PuhCK{a;#YK+iQh+I50+4lkD^8{ zD)L#o6o0?iBe_(w7*ImvQQxnWxnBUX@gMt08sKrZaLl(wSq@)M(`05D;ulk<|3vV3 zF#HIBK&+b*_3npKErB?t%fkMO^+QA2$dF`%nS2)JpizkSYFr3OIRzgKgX z>+y$0vX(*m_)w1j-n<9p-`ev9MXEQX^F?k`yz?O`sV53}LZg#;}IW$@Dk^A55 zs7lA##|~+NBWdH4`;M4#Je2j;==45l@R@iC87U( z4KO6Ho@T8W&cFjF=s%uIK`}Yx`Hv`Y@%oQhS;g=u%}vmMWDxwBcDz7gjiJZTk0hQJ zg&sftZ3={QJPaJGakED12=_N1nLG)p*q_&&eE0K@Xd)~l{J9;==M(hvxFF7xKJ|ov z3xePUY6Vc*_ld{i$bg?_H+LE;=tcr`7l}Fp-H`t8qY$W{VCSVA6EZ$#KbH!5VVa7h zWh(ZX#;gv8mo#F%8(!0RQyQZ^H*M;XULfkBxtp(|Kj>g9diwQAimmozO;@7?`#3dK zQxF>CDX0im4CSAgBBB=MS~XqbB1{PUCP+wpGhWNSMCMjaRbS=k6nR<3fd_ie1#U^u z^e1I?uJFszq+eK4&tgJN%1;7nsoiEizISey=8bo0_fJYmwK|(a?-b3=A8&5gPpY~3 z!U$I6is9Fh7l@e4MFT|oLg*yUl3?A3P2t#kj%4+71{6?{oCG2P@dZ~@Dne?oYm&F% zlO&_A>@LTHy35p+!(fIbl;dI46`KDd*pc<=bH;*5TqYLuIr`*WWRlP$U1lR_zRUy6 zU~IQoLM4UEdDqe^5MBKkxfJ@Wx>AXI%YKww)~Uzo)XylZ93M9witVTL3RZz~*BHK_ zUg-qa@P+gB=(H`Ts1R-+Z%Yg3f>I~fw2}vuq_xQHkP3Z@#|wheH}W-L`e`AKUUY_)LjNGtV4T7?_?pR82-Ir*6hto;_4RPh>*zAF3mc_saUc4VwXqVQt5JPnC74Tn8u@&kZIUqs-2N( zkp?>M(S_VDyH<0h3Guxh*>z-cj+uOTxvunS3avRX+nn8kUHPXBIE^mHW@SF>Op=yl zFLJE@4a!P-Eot@%G&C;G1Z>0)xFmoXq{LWh))xNQsgh&5is3JDvepMq(PT2dPY)L! zVj>81G13nkNeD~GdzoO;Rk(%2|HHQIh2e*s6|tm?v};gB&^LyE?sLfZ+SGs# zE&MBrTNf^lQ^P)gO&IKG0yo?PptpPs|2kH%MZx!6-e4%b&% z-K2zeB-(;B(V9HUh3hY-JWj}x7o$G>7IiJ)U;SJ<^gV_Z=YU*SW3{jBhzlK+uO=jd zCp@}KHVTj3c;cOA)9d(ht64o$BuFdA*{YZL1%y9GQ;p&0oZU$!k9=qVx3}mc{{>oJ z2wGpwm%U*z^6<{)cz>3u?A#o++~|j<`HnUt$i{FtSK3Ku>zOL2mArbsUWj9|oZmBB zE$gUjTQ{Zy&F)&Cb>4*f1x8{5R@j+yWWq&W4FCGj7!lR^RkP_~_$NAcz@#0Ffn!&5 zbnw>j^+WpD0xiB-H)ad87Bj-BIc}tss8mEQFSS1O14kJ{UEO3DChjj+`D_klwS<3G zr+7a9DyKBc=7iah76`JjT{6qI?N01QQJ)2w;D0uEI~pM7(%;6BV-Tw<;B@ws+g;)B zq=Q`kLps>y+~>Bc@xop4?s{?zQF>EV|#+^m-$ zy0!mM+uGlny0ufOwWrT*jc7|2*KL1j>c@0nHLN*tGy4qApVl_@zfaxN_fk{8^r4zE zl&NH<^4poI{EJy-C1-?yFxjpYNmg@$_b*U%9NrXMN}br$UKe=qw<&P(rt$)>1thZR z%IA14%58`tuNn<@CFwtA$glF^|C!>FLkJ!z2*?&kiYF&`(sKjlDL^Nm`cR!*Yi)$@ z(W-R-#zzXDH&E{&I#xGeIqEbG0A=v-ZMV~l;pZoJiC+A_iDcB9gdD<^i-uk?{3p7! zE;S8~TGRhIYx;kiqUk?NP5-SQyJ?^J=J1OXU0)6%fMP;K;ZT z3S~?!V4;~1N2w1B$CC3g#Q+g!vx8eHT!OlBR@$Y&k1U%x+M_Z-WU zf*L59Y9tz!*n8fxx;VI9@=;OOZjpXVrk^hs>xFchI$ixZ|F$Tp96yc|lu%ddP>{!M z6;lAsl{Y6*6r zy=|rx8)D%bh+5=Z4V2I_)*^OGu|e`t0yVYWs|`d#(3YMrK1u=M1i>_#Xxh&Sn3>RkL6Y1D~9l6Qu%^u6jx1P5=9{lZwEe(^myiPCj zF<3SfDgtaedsQyrwt+8o>98CtOq8__Ikqnuzr2q_7~b1I(2?MQO1JBrNE+zXoKnOI%?a%FtI1# zQMmO88DhuKjfzQ)A0v4cr3jw`M0aX~DFrCk))j9AY#e4@ttYk-wa(Kb%EOSZZMU~| z2%LG3*1~+By0+Bz_6iMcm9Y=@@R zq`;C`Q=Gt&ELXJYFZ`JN=+pWH0d5#bmI-M{r{rdU5E1fOPr{Vh``e&J*?P7Cca5jP zgaDh6OHppi1qz=iC5;nWmMK)aOmX9FNTNxlRYp!0l}WO0JjuUZhxi*$GGcb(gy&~( zI(c$TS`tbTxk9Cv3h1rrEb;6^SGm|NR?ByN;Z_ZVXuVG{X!2dmoAH@|t7@;6)}c93 zZ(9=A@)+bci+fv!V6niD&zxFxLu4ASy(VIS54R^3waaZyl2%hGs)t+vTx^8yxU5Em zvP_SNkR%Z=XNBIP8D&4tB^-^AMwZ5C3if~&z@u`mB*&j-F#t=Wtp=i4O&&=mP|`E( zNE_6QMTE(5s3Rd;jdUjOE*Hh=`LyY(?rCp@F3e~)4S$&=`O%CL7O4bXy-OF~OOhmh zk{ts|D#uUi>lx<}=)Y1U(EnW$=)Y1U&^-<6`Q`HnbgW~epI`ZL1^Q=g6F(p8`85s* zMB!|BmKjx1T`|1^#)dnz}_pmw#ypgnV-nx_c}Y@Do$#-*6;lkUzJ<9u@7U4E2&P%_ zQVZUl>MK2yr$W%wYNJBO<;!8W(ItF>oO|bJO;s!Z9nMPYmS9G5?T3CO4 z0~CFc51)mx&-vw3enHo?eMC|(B)t{PE48D5s%s^w!-|&u{c_x<+;WzvWlNsCjx5ie zSt%9Umoj{C(|QDQj&L3~r)!vg&G8z96a2+QM{Wkd6}%^)sp9}$4RhEBpKdk)`}u?d zkXCqA?mAtoP)$z;Jr-;{ufvM^AoNNt&S2X4Ub7J|tEefevot3P`s_&SN{lK3FT z!i-l^qPgIz{ZaH8d6QbiQUm*^vtA&USRekYMZf4-3{lUb8GH6&D>m3Xj1HafLsM(; zZrOr^9r|j@-qdI2A}a{vD~tUH&-EH znH=wJWwI@pxpYj-6!?zm#R*UfyMJH&zK{CAZ$tAjeTLhR61p^e9`g+PlyEhX!w2K4 zEI-3jLg)L$M6DYh^PZuq5wv`W(qx%sKFgkvG-J?OFcw56B1L+d$PYv>)L|2xUm~&o)a!g8SHzn7r#iZ?q&qC;lm?TEbws7y{SQ^^+ z?vymAQ(}<^+N4h}GVDp?HT2+Y+)3!+r4qi5%3h1A5{~YXkV=G4>Z6Js zYzTL`-t^IO#be(%mC<#kn;gJX>Pfb%-W4evyb>nGa8Igtot_(+Rr%D^B-Dm< z5q3`JKAH>Vh(BZU`_R~Ax_pe2-pW3drqo|rdyE!vm&!XP8)GR-){RLS+8Ebwic61Z z$KZ{laS57>k9$Uy7Qf1H=lIgQKJ6nGQhUvHj_D8y0yD3$k2^`#TFf&{Pu@Da2>rFj z(U6W>hfPc6fY)gej2+=RD`c+Sa+Ead(k9s%h;uS2&te)@f+36;Kg~CJB@QK1$g!0p zPIP*(J7BWahi>U!#jRNi=WP`219!jpk{zhq>H@<)n)_{f(P~|2;gyc+B7jbP20Oeb z87I$rl=u~k?44`}% zak3=sUGp)_H?Xv46Ez*ow#;{|tT_rWN5@@&=Y2CU z!*zP@?V{q3YBn#NXWHFL|LdRjo%&dFy~X^cX;%uXJNT{^;}3gZ-2A*v+y978>L`}b zGme`(`P7tfGn|rJQ<8_M7BfGuW*0sNp#o+&wm?LK{nQ94C1ur6;B=BFuE?8##mneM z7R<}`erfU5O4%1FR6Ty?%o)ShAG&pyAw1}b2li-9EdF1mSbXC2c){eGLd#T;a*{&T zw4X$1Z_I^9bHHN3gx#D51c9PvPv`r=#5Z#*Y%>8Z;yo8k7R#>TyO+hV(+sm(xMW|i zR1{45fa}jDZU2lpHfQkt7Y)2VT-@{3=;x9>I-fvf$Q)f=e1uP00X(a-LZv>X;V+Vi z9!+M$2Z|7}V_vsbOCtYE1O`9yTcdy!ER-{CV<~T-b9{I1Lx9Utd!NUwI{|0q)#>PAc&Nq&0`HI^=33 z{nWRa%fuob!hcO&`?*8*C1KQp9}sYfn{$@ra}y`CW%1?dOdy-pT)c12Y0x-)w#qzg zW9)Ugik$hZmRRtyAN>R()WgT!Yl7T0IXJiZQ3?v=d97b-RZH^L5@=#XwUCwzOI(N@ zwQ2XHze=ZMGwkW%sKL0JAyBZZ<`Ae~yN+-9#qi$^k@QlU=5cvgI$1fa2rmiDy67;HkM-vFb@X^+6 zcSSn&>0A?IwY{5!Jd59Y%)BXM*}m(l8f^mI)eBIR7?2utyjVP+T;7qqUM`Rs!AT`0 zB_-!#G2A-)VK5vjXwb-}hd+7vv@_qoXW>aoIOo|_pmx)`^5MCB#!-@=$>N;A$^UzD zT$6dBih6;jX!4IJaM3DM3U>|9LS@cpnZO*DhC3TMi8ak{=6aYggK|(Jq0$%A*;yLu z>gqmwUf1_E*Ie7X|G?_uzM!z;p0BH`ISE^pr;~Mm z^@-reK8v?zdH_;j7U50~&n7@JlFOZLuKj1qd-%GSa4j4K@@oX{;Hgdu$Cc7zI0Q%< z2NsB|F%8ql4QrBKJJS5$($NllQp*4|?g-$-Nl{7j7zgV7M?PIF0&fi4jKhoNaeBt8 zz%pfyFTYg$Rt8Fgbnq48e&{j_a?QszvN>6^pDJqOTnr15B(WhSo1Wa928g88Sc|9$ zK4!N_6isDa_tElln31K}K#a++vN0sRJi$eH;Hn6nCcL^yES)XYN>7FjV1ZR1q!^Lx z(_#BEQ_bXQcOU7v1tfNsKa#y2W#1I>q_Z%vf`xD)<3c0K5ERfCeG5bK8_~yOc<)e`_|eRx4J(y>nDLCmRz;e#1U)y z$$kLSN{^TZhRbeh4eyLlq-r5@T+O=F_))JvIdf^%Ja@?tT!jv8GI5#??>YfDgv(i1 zir?@gCjVHChFn1z+#%a>Tzc4dT<8bxGS!FBJmWZeBq2bIM>~yNPxIx1D!Q){y2n+f zwDSvOTfmX0(BweWPe}?GG{f7t%RWVVI!%mcL}X{NG76E21RM30a>-1cjafo)WV`;Y zH`wzHjL)!o9-R=ka2M(sh-TQ`y zy9b8q#94Rr9rQwVL;L#&Qh~v~{e3;dLv?$u@9*BzKhRg#_h4`LfuX*^?%h2DJNEPq z)(!36H#oewe_*)ITcGj#DZayt@9sI+*PUvw8y@W0yT5x^-$37B&z{skfA78ndbekw zzqbwsP8X3jd11Kk!Qr}%n?JGm?(45# zeEkjAthwg8#j7_i=4Wr;c7A#Wdw1Wyw}1c8T0Ly&S(XV0D?o{%Lw zR`1-iuVRpRhZ=nVNcKfb@12npNC%OCkd1vR&!9E}f zWcVP1r*A*dzh}6Az#qu!y?-@u8t&^I-Zz*f4)*sBukP8?-!oKwZAagpzTv*r0|)l( zS-pQB5a}CS-MhQ5_kMc2bKl|e_6;3i-uLd?(I>=USVMi(G1%9$qkHhcKsTe`*}uz*_U`HH8MM3p0S3JXWD_;) z-OhE#!JZvE2J3oz7+h};1M!YiXkTyLfq|j^T?5Q=FOyxj6EKZ~WxVUvQsZOL5FFRR zO{2OA$Zh}<)UCRnF9Kz!n+a8!%F-`V7I4#`(V#-U#d_L9&%WEr^GV6yE}V_2J5;FaTwCz@Hk)%5)P>hjWm2PRxFI^ z9eikaf8Di1gS|eMx-_5N`(Q;Nf!9y$!`^)d`|7y%0;g>BL&LC=U8sgi-~W&<0#=so z2u`-$bf?d#)D3iRPpI!uwuh{YZ)dFAxu<8>5Xb^B{AI8;-O&hf7~t6uL<7}k>EIH8 z+f%I^@Mc-PV015dJak|WQz_K#9`3onZ=eoX?dmiB%;a_-P$}`F?gy{G?z-zB)1l$I zy*>k_)r*}>HT4gd$RkCx>fuY@X!}}l#@qI757ZQiJcW;LT z?ynnSW~p)C@NT%fqseeTjAwktfsyx?3ik92Krh_}D^b2Ucn`cG36#DEfswbed*ATT ze*UFLI12MCSQ%fVMUmS8Os%eO55!5ObZ~dygThRW$me?3zCAno2B0TUqiz^!y@HgCdgNLv>TOYSeeleI_pNyl?TEYi_u~Mng6G zg-AT?v8jd6X7W8lBIvVcXfH22S%_Q*`*!xjTLu1NVzM(yEEOZ-Qcwl!QL?a9z1ZoF|z63W`TKE3ztniL~+4I`E9s3S| zTXi|q2&f@IR;Zl&26_xMiCu*lnR2q-X&31TqH>#dq^ts;Ny=UJpW`sWNJz%}t(eoe@YrCo$>VsA7aQIXK zBN(O9J9qRB-@@ZV2M(?o+_fD(fRa7TB=in-_o7%0clQmVZamaIG~DNjusIqQa`W15 zXlLIpq&!R#l@QP!5Ta0N-R>cKkm9=^o_l}YLl6xuz#Slc)$g<-t?Xdk-hIO;ncaXI zQ93LHqwJc8dbjtgJ=oVl&CJ@mL9g|T`}=lvf9b~S>UyDZ10PMMI~Cu%f8P)SH0u@M zLTw&zu=l{Ak5t{YA$ui*RX7aow%)7O-hJw>;NtLxT&ndmiiZI2FqA9V!NtBKN5mc(Q0ZF#($W)15 zWZB5Up=wFP0uDUMA8|vl9>p|b#w}{efXe_?A)S~^W>CVHr+BbBleR9TwMsqUwlL}M z+;i^kzGv~#ojmRS_ndp~x#ymH_wMnY8G+G%M5`znixV9@!hiJp-$FzkyZNWRiQ)o9W>yRezl|TFBkF2K>bDB5lX<++q zc~xWq?Pu5Ye^KX}h6`a0t?JgDI|gnZ9os&zHTdoE@qw)ay$t(X>8^K9WKl2Z2AUAu zFk(FKVi*sPjPIAV{~nfBOo#(INJM1c{t63w+}X*wx;=Qy*x2Ym4@-QhWs9pa=-Ryp z-@Jvjg%G4dSA}%=-hthtOk+diyJbBs5_|P6%(08Y!7vscmvtpv16$nRt?uvDj%SN& ziD>Dnc15I$Ev~W~@8Y2$O%(CUDAN4_TjU zhdHe(V-+4{%$j)T-cg<`IJCE->oTWE@A%jbI@+DC*3&jUU8AQr=;>NLbvT8ub_Tm) zsIprYtvx#@SXFD!vzp&q3B)ouj94>xJFD26cJG?l?`-M{Iz4Equwg65=B)3_U;-XP zEPlqlEu4N^RD>@gow$<+nCqWR|KpMjB>Ye*zDr>fpJIp#$?3q~#57i;`eN4O)oaB6 zoz+}LPFrU`9TOs0Y^&a$jIm^^73K0hztt%s-lSJrH4hJF{~Mjd(<5THzLDMI7*2IG z;Zn@2cPmTd)Qy~Y^_|SH6Mb=_y{u%%SjDk4OD9}L3L5iTG%OFB$^@@Q<1DKEL^nFb zI!-989k?9bg1zc5w#u_$C$FPAbNGhq8yCw0Y8#S2+vU#=JxOR-8r|88BL)AqSyV!~#M`tc({l)@O2{u(qF#MumH?gV8n7$#&}Q ztqk+15v`Ja`GJ@EBX=M*8D5>xUTMXxnJ3J7YpaniJ@dp*MD`fLIXM!d2vO^ zox52c>-zz>rTLe=Gq`w&uvA8Pb4GZ(D?s%F?;9HZLz*B#IA&88G6Rsg!4x;-UUgj+ zLzRlNlf<|xZsMeCgl=!R{EJV}@*GDl$Q^ju6)$mE{=^Zx8U;ZIzZSUw%XJo4!0Y*W zdK?6~XH%S?v}@s^d>(o*684r`yJq=!r=(ndnd|-(7rGm8&G1If$SGW+p7#E7YtL-R zKi4*ugLBP7@p=^7xNgm0(K(Aneh)w4LtgP%5EP=Wa@0K=R{00u$B?*qt~sAu-?VtH z9P~KhE7&NvDUO;J5b-3x7IXqPS2>$a1h>pV>rLq`=pJ+mzTD=yi8#6fT26F}ID^Z` z3l5-L?@VF3SS`(l;RxD2_4GOJZ&RM<(YCOyt322@4gFAIrdVyt=Q^boPd)vubIoCA zle_3H2WltUEwZ04cB%F_I}SFT6LO#Y?JKv=hW&EI0gv#0mO5Uz)Ky;04P&JB#|sY@ zkCu)!9d0>Po@t$4n_qvxs{8hLzWco&oa_2UHVgZycM-W0{IHqewbCD@4DRK6QwC3R zEq(9#k8$0X!OQjZb?EB)J5u@UI0px5XLq%L{d!&Zs)kOk=&)`cp3W}gnGgry1_!os zbg3imDz|r4n|j2|Jz{6;i*MEQt`~iO6))`*xi649&WhaG5cR?-^}VpiYe2gl-x7Qy z_`J5EbJ4m-d`ajp#D=R%UFB+S@X(8A*B)Je5~y;cU4R1E8@ex zL^w3Yhsb|rKf0dpD))9DDfJv~Iv(EsVBu)-P|Hkty0yACAIc0O1H`$mIIhWW>!j|s ztnT?7`q|)v4}TQ-zd3?@e&VBoUyIH`ulJ>}iz4bEMt@xd{by=t%ojf68LZ7$LC^L3 zHuq|qjkb9b+6GTUn~XhPU$t`1NAkJXNxRN9i@mjeYghYu>OXpb?@Ch|pezf4KfrF^KS^J4|u3GX)1rNoc+ zIlRoZ*qjdS*VY9=yV9vMGA44jc$m! zu9%DwlYwuE9yeTq*6sW3K+^TT{au~VALD23y%F1&PEC|66Vt6>@9|JZ#*xzDCK(-! zjnx;`E8Nhal=E?^=dBA|ujJ#VnJG#Yj^o|I6 zWw_iq@}PCPboq3-+M3U8O7%kk2QsD1tyRiIU~DV+o%7%g-T2jXp?4;%7Q-d&OaC0e z9>Uj$E~{EtqttSv_1186MS(s|rV6AxJ4(6R@Uc}JhACEAsn}zU#pJmv<=(T*SUKHO zUM}>^v>te{I2#@>_y_Cs+UojzD9<84Ihij-T6X9Z^)E@Aszur~T?oJIcttIli$wnR zV_cu*+1EQA_V9dpR5*lgcG@w?^&G!8m8Da4V0b-|wWF+-b(8n}%kJ(SXk=nxVr>Tdi5h zWqqvA`W`oS&NVYbd;Lw1y`ujw(PQrj^hO){vP;n7M31RNk4WvmPs$$7+V8%!e&cx` zEWyQ8*YJGG;{EM?m*82c9<$Zt6m;s5HAi^RY-P0loGwV~+1!z;~&Z5B9N_ZK=F5^S*@BB4*ehoPXL{7D?E_qng)pcO;m+)Ah-8%>mI=9N8 zDY(f>eW!6QXTB--#nHV>@So+^srUm6jBWa2<>@v&oj!2C_k(8Vntm=s27Mti|D>Ba z)cP|Y;x?iI`U&kmX~Q+z*`wQ%;^gLXVfg$9l@qe?>P|q*Y z;iFRDxA|GyKDtbu(jM-M4i9kYWaHO<*^toMq5XokJy5G}+s78bw9D7?@$+D@VPl_D zZ)Ea{d?df_1eRX_{`C4Uh2bn{*G((ab2TLyc5(tpG?m2Zs-S-_Zbal1oD*Jiax zTK+SqD35Nk7G~v$1h>Z2aYH{!9e>da4}IR=eW=(!Q<`NlC=0<*8X_~WW48Jpfi)OR z+S-@2mHrV~j|`&Ey!Es6@(hkXsx$k!=FB%sD}9PVoR|-x_?m)frhZl-wXJZ(k>2DR zb{Dv`_@#U>??M@I^mog`0p>pz0MYlS;8Fu#(}0^A zuq{#bF7oK-^Z;LxB3GZpL|mKXej#}7KRK_P*GzJ^xj4bXNFD zf^hA-wt+5V$Q>sCwgx=nW0MFM`FA64it~Dtw?B>+=D^Pf(}ZBk$1aT@KhQX(;Ta!) zG<+O+i-(=b`a1Fx{V!>0Z|_{m$Go(BZfzW05Sv-~9C4oueHk2qWegd8&1b>z*pp%U zZ`(5LZH8YNM;dQ3yqW$R6mjZr;kDq245oiBjArm#z?BTX89b7~Zw1SE_40l6y>XG*)Ezn?43)`1^6BF(1SK3Z6p$Zo@y0BaQPY&PjjGQ~#3oSNL?LE;?3I5J_}L7W@pnFhe*^qXidV7z)tHdUYlD60gT_IeX{d>0?Az{x%=`F}wqL-_FPz1utgs z1oDBwl*zw0jx_#b<>~JWPiFWJ``C}sKMDPGN-r$L(T+**oYC=*3o?Lwd;SXi0gsqV7}3Kk?}7=Kbg|=jQv|#C!_bqk;az{|2|mkao(`_$KgzQ{6phOMrS_I*pcB6 zfTjI5{}w(37X5BF`k^?|n2_PwIMO(fVV=Jl_c8o=>OYmK|EqDNu^yv80Um)qVfee? zC2(w*{?z!5;q!5%u^Pkw0lt*s|L-`OoB|(KI>}j)#$F7U!4>GkhF=Um1AfNvtK#UM z1@M&N*T<2@P7D*net(wE_|=$*(R;v)l&>1TmDt5#Do^IGf2iS)gE}5py$6~*Z2qeF zScjE=e;h5h!_W>t>z7scgN5JY>GA*Jm)#-{rv|c(ts5 zME+NxUk6?O-$a5Rk6->GvDazCPebp>(9g!v2NsyOtiM(<-ZZXa`v1F+r5L{GV<(D> zEyO@tk=JJU1z?eH&zIs$eT>BDuZ|;)kr<|bG)7{W{?J&6VdhhfeHi||IMR5B;dg^E zXs*@pJ|FKe{72ySj6N~A2EVPJiq$y!*aCQy@w5KXxQ1fs@5dJ@Z_lgZEZ6SP{w~h@ zIER)0687zd!TPiK7`QKkpY*X0Zzu^ZcKb0we6#7DjF5{(|p);Rp z9K*{0bsRlB1^uMqFZp@g0SvzaJW2WM4PO;U z`krr?`A*;I4fFib_jbd~Z;uR8eqOQY^F%8eESNrwud~j-p6}+9(oVj^>4x6F)~#1bCZGANPy>7UU#I*f z@CK8I{ab{;*XsWs?{mvwU;k1ZU3tQ=$jAQr{!jga89#qeHF^R36N?9wkPlu{0=xBr z;TIwQ8TbpT- zDp>eMAMAbQDdWdpYo3ZsuS7vH{r^WX89K0gK@gTK%4&xps+-rTc_ zo0_Rlqsc z`xe8uVXq13zW(hz&!zol-zJ{l#kP#RDg5mcxYNpG?~UidU4|cGya1E(C;WG6M(6oJ zGl*9hX8ok!of-ay|CY?~qW|W~@ORK(^b`BgA5F|J>*j%8pYzyf8EoUF=@S0B1h)1y z2iW^W2DjGtZ}ZEr$4G|$8v1V%Z2jAOE#o@^oBuS)_z-)H=y+(p*~XXD$M|l>zrQZ} zm_3`BKh{OygW8_vUCei)zo+Bh+40)^$Nt+W)10$9C{Z=`cxF1CrV3lf>M|qU440HExu>l(MV|=h zau7Ffaan+yC5J(i3+24aW$UbqLOtR#B zHW{BA-_w|eXW4L?6X%oYe4bc+4xJ%Bvh{o}bB`}-U51;_?eu2ICSfHxOG!GCBwe|5 zo+P_T?xeX`E-8%^DDJw%Fv()kJhUXkN|Ng=VX85uD^0PY&6aFcpHr6Rm!-K|NupSt z75Q2@$v?{`lkws74$D4EF4GjW%X3*=29YLP$=~rn6ND^v#JyK8A9u)3>k-iW5TxC|7%ow-cXL>jYWNK|#XH0+IGxh6@L zN|HDu=_*MEjirQW!nP_QUMJ^Fyi-{xNOUiOinsG!03JpHb#PhU1YWlt(;y$Etl_J!q82& zV8Bn@wh^-9#2)sk$!0 zh6sLJ6J_>$?6?(^v#2u$Ynx$ft}L-+(Zgzl6ubI;MqDY~8zH;JhOv|Qrj_Vgt6>Ws zvX98VcTWp3Lb45j6xLjOS!uZ~BUB1&72QNg*z2|d8rbCw=CslIQG~el*)CH-%IL-> z?#k`aV*CEE;B))MSnu9BbO$TNOfj{JnDWlbN}CjDIoX9nL?@fRxIIh!9!J^&*DM6# zx4LNtYXEs*vVV(HST=Dm7fkk|5rS;JAhOXU*@Z@nCL7?02g53x(D>eVTR52Mk_@VJ zhY?lVEh~NP-osH*_QpuG(;Z|`#BBp28qr(H-VfH1+s7s*$Vwa7=|+WD!gZP%)3#f3 z<=HDj#oLZdu8M(O+Zl|ia=KfLbZZc7-n0KsNnzW}9I59Kf2BKJI2YJFKIW?5tbaw^ z=HZ>&_ufIj>>BgeL)*8@J_ttPQ7v0vB%DLcIP6>C$?SnpN0#(rPeTKLsTtf}r0i%S z#Q)cECEW5tTXgT~?$tJs_d$Q?ogjEQH;j*yM5BYRE_)`7c^)}?zBg8a;H3#c_<}Q@CbvZ= zIc}=s3qL;}1h0CBlH6a<*H_Q?kKF&uhgRSluH(yps76(D89vcnjy?5!AMOr<7j!Af z{q^;Ju%7QYv8yMd75F}GeC>`_7&(bN-q({u= 4 * is an R4400, which is the same rule IRIX and Linux use, and is why IRIS * reports PRId 0x0440. + * + * The R16000 has no imp of its own: it reports the R14000's 0x0f and is told + * apart by revision, major >= 3 (Linux's cpu-probe.c tests major > 2 under + * PRID_IMP_R14000). */ #define IMP_R4000 0x04 /* R4400 too — see PRID_REV_MAJOR */ #define IMP_R4400 0x04 diff --git a/docs/o2-ip32.md b/docs/o2-ip32.md index a1ee2daf5..0dba72c24 100644 --- a/docs/o2-ip32.md +++ b/docs/o2-ip32.md @@ -3,8 +3,8 @@ The O2 is SGI's unified-memory workstation: CRIME (the memory controller and rendering engine), MACE (the I/O chip: PCI, Ethernet, serial ports, PS/2, audio, the clocks) and GBE (the display engine, which scans the frame buffer -out of main memory). IRIS emulates it with an R10000 or an R5000, chosen at run -time like the other machines. +out of main memory). IRIS emulates it with an R5000, R10000 or R12000, chosen at +run time like the other machines. It runs IRIX 6.5 (tested with 6.5.22) to the desktop, with networking, audio, SCSI disks and CD-ROMs, X and OpenGL through the CRIME rendering engine, and @@ -28,7 +28,7 @@ serial_log = "serial.log" [machine] profile = "o2_ip32" -cpu = "r10000" # or "r5000" +cpu = "r10000" # or "r5000" or "r12000" [o2] control_port = 2293 @@ -65,14 +65,18 @@ in the file to silence the note. ## CPU, memory and clock -- **R10000 and R5000** both run the shadow cache model that IP28 uses, from +- **R5000, R10000 and R12000** all run the shadow cache model that IP28 uses, from reset: loads and stores go straight to memory. The PROM tests physical page 0 uncached while its stack sits in cache tags for that same page, so while those tags stand the bus answers for the page from a separate 4 KB page. - Linux supports only R5000-class O2s. - `[clock] fixed_mhz` is CP0 Count, 90 MHz unless set; the PROM measures and reports twice it as the CPU clock. R10000 O2s shipped at 175 and 195 MHz - (87.5 and 97.5), R5000s at 180 to 300. + (87.5 and 97.5), R5000s at 180 to 300. The R12000 (revision 3.5, 2 MB secondary + cache) shipped at 400 MHz; it defaults to 200 MHz Count (400 MHz). Its + secondary cache has 64-byte lines like the R10000's, because the PROM fails + with 128-byte ones. The 270 and 300 MHz revision 2.3 part (1 MB) is not + offered. - `banks` is the kit in each of the four DIMM pairs from A: 32, 64, 128 or 256 MB each, largest first, up to 1 GB. The default `[128, 128]` is 256 MB. IRIX boots with any of them, from a single 32 MB kit up. NetBSD's 32-bit diff --git a/docs/octane-ip30.md b/docs/octane-ip30.md index cc652a8e2..0798d29ee 100644 --- a/docs/octane-ip30.md +++ b/docs/octane-ip30.md @@ -59,9 +59,11 @@ overlay = true ``` Paths are relative to the directory iris runs in. Every key is listed in -[FEATURES.md](../FEATURES.md). `machine.cpu` must be `r10000`. `[clock] -fixed_mhz` sets CP0 Count; unset it is 97.5 MHz, half of the R10000's -195 MHz pipeline clock. +[FEATURES.md](../FEATURES.md). `machine.cpu` is `r10000` (revision 2.7), +`r12000` (revision 3.5, 2 MB secondary cache) or `r14000` (revision 2.4, 2 MB). +`[clock] fixed_mhz` sets CP0 Count, half the pipeline clock. Unset it is +97.5 MHz for the 195 MHz R10000, 180 MHz for a 360 MHz R12000 and 300 MHz for a +600 MHz R14000. ## PROM and flash diff --git a/src/bin/iris_bench.rs b/src/bin/iris_bench.rs index d26f4a3cd..d8779315a 100644 --- a/src/bin/iris_bench.rs +++ b/src/bin/iris_bench.rs @@ -270,6 +270,13 @@ const CELLS: &[Cell] = &[ Cell { name: "r5000-jitv2", features: "jitv2", cpu: "r5000", expect_cpu: "R5000" }, Cell { name: "r4400-lightning", features: "lightning", cpu: "r4400", expect_cpu: "R4400" }, Cell { name: "r4400-jitv2-lightning", features: "jitv2,lightning", cpu: "r4400", expect_cpu: "R4400" }, + // The R10000 family. It always runs the shadow caches, so the default + // build covers it. + Cell { name: "r10000-interp", features: "", cpu: "r10000", expect_cpu: "R10000" }, + Cell { name: "r12000-interp", features: "", cpu: "r12000", expect_cpu: "R12000" }, + Cell { name: "r14000-interp", features: "", cpu: "r14000", expect_cpu: "R14000" }, + Cell { name: "r16000-interp", features: "", cpu: "r16000", expect_cpu: "R16000" }, + Cell { name: "r16000-700-interp", features: "", cpu: "r16000_700", expect_cpu: "R16000" }, ]; fn build_cell(cell: &Cell, root: &Path, force: bool) -> Result { @@ -1077,12 +1084,20 @@ mod tests { #[test] fn every_cell_name_is_unique_and_maps_to_a_cpu() { for (i, a) in CELLS.iter().enumerate() { - assert!(a.expect_cpu == "R4400" || a.expect_cpu == "R5000"); + assert!(["R4400", "R5000", "R10000", "R12000", "R14000", "R16000"].contains(&a.expect_cpu)); // `expect_cpu` is what the `matrix` guard compares the guest's // `#machine cpu=` line against, so it must describe the `--cpu` // this cell actually passes. CPU is a runtime flag (see `Cell::cpu`), // not a cargo feature, so the two agreeing is the whole contract. - let want = match a.cpu { "r4400" => "R4400", "r5000" => "R5000", c => panic!("{}: unknown --cpu {}", a.name, c) }; + let want = match a.cpu { + "r4400" => "R4400", + "r5000" => "R5000", + "r10000" => "R10000", + "r12000" => "R12000", + "r14000" => "R14000", + "r16000" | "r16000_700" => "R16000", + c => panic!("{}: unknown --cpu {}", a.name, c), + }; assert_eq!(a.expect_cpu, want, "{}", a.name); for b in CELLS.iter().skip(i + 1) { assert_ne!(a.name, b.name); } } diff --git a/src/config.rs b/src/config.rs index 1ddc8fce1..b4e312a01 100644 --- a/src/config.rs +++ b/src/config.rs @@ -707,12 +707,17 @@ impl MachineProfile { } /// Whether this profile can have `cpu`, and why not if it cannot. The - /// Octane builds only the part below; the O2 constructor still refuses - /// its own, after `validate()`. + /// Octane takes an R10000, R12000 or R14000 and the IP28 an R10000; the + /// O2 constructor still refuses its own, after `validate()`. pub fn accepts_cpu(self, cpu: CpuModel) -> Result<(), String> { match self { - Self::OctaneIp30 if cpu != CpuModel::R10000 => { - Err("an Octane has an R10000 ([machine] cpu = \"r10000\")".into()) + Self::OctaneIp30 if !matches!(cpu, CpuModel::R10000 | CpuModel::R12000 | CpuModel::R14000) => Err( + "an Octane has an R10000, R12000 or R14000 ([machine] cpu = \"r10000\", \"r12000\" or \"r14000\")".into(), + ), + Self::Indigo2Ip28 + if matches!(cpu, CpuModel::R12000 | CpuModel::R14000 | CpuModel::R16000 | CpuModel::R16000Mhz700) => + { + Err("an Indigo2 IMPACT (IP28) has an R10000 ([machine] cpu = \"r10000\")".into()) } _ => Ok(()), } @@ -761,17 +766,18 @@ impl MachineProfile { matches!(self, Self::IndyIp24) } - /// The CPU models this machine can be built with. The O2 has an R5000 or - /// an R10000 (an R4400 is refused when the machine is built, not by - /// `validate()`); the Octane shipped only with the 195 MHz R10000; the - /// Indy and the Indigo2 take any of the three. + /// The CPU models this machine can be built with. The O2 has an R5000, + /// an R10000 or an R12000 (the others are refused when the machine is + /// built, not by `validate()`); the Octane has an R10000, R12000 or + /// R14000; the Indy and the Indigo2 take any part, except that the IP28 + /// is refused anything but the R10000 by `validate()`. /// /// Single source of truth for the front-ends' Processor pickers, so they /// cannot offer a CPU the machine will refuse. pub fn cpus(self) -> &'static [CpuModel] { match self { - Self::OctaneIp30 => &[CpuModel::R10000], - Self::O2Ip32 => &[CpuModel::R5000, CpuModel::R10000], + Self::OctaneIp30 => &[CpuModel::R10000, CpuModel::R12000, CpuModel::R14000], + Self::O2Ip32 => &[CpuModel::R5000, CpuModel::R10000, CpuModel::R12000], _ => &CpuModel::ALL, } } @@ -1037,16 +1043,41 @@ pub enum CpuModel { /// the Indigo2 IMPACT (IP28). The machine uses shadow cache arrays; /// see `docs/indigo2-ip28.md` and `src/cpu/mips_cache_shadow.rs`. R10000, + /// MIPS R12000: the R10000's programming model. The Octane and the O2 + /// have it as revision 3.5 with a 2 MB secondary cache. + R12000, + /// MIPS R14000, revision 2.4, with a 2 MB secondary cache. Selectable on + /// the Octane. + R14000, + /// MIPS R16000, revision 2.2, 800 MHz part, with a 4 MB secondary cache. + /// It lands ahead of the machines that use it; `iris-bench --cpu r16000` + /// is its only user, through the Indy and Indigo2 (non-IP28) profiles, + /// which build any part. + R16000, + /// MIPS R16000, revision 2.1, 700 MHz part, with a 4 MB secondary cache: + /// the `r16000` model with the earlier revision's PRId and Config word. + #[serde(rename = "r16000_700")] + #[clap(name = "r16000_700")] + R16000Mhz700, } impl CpuModel { - pub const ALL: [Self; 3] = [Self::R4400, Self::R5000, Self::R10000]; + pub const ALL: [Self; 7] = + [Self::R4400, Self::R5000, Self::R10000, Self::R12000, Self::R14000, Self::R16000, Self::R16000Mhz700]; + + /// Whether the part is in the R10000 family. + pub fn is_r10000_family(self) -> bool { + matches!(self, Self::R10000 | Self::R12000 | Self::R14000 | Self::R16000 | Self::R16000Mhz700) + } pub fn label(self) -> &'static str { match self { Self::R4400 => "MIPS R4400", Self::R5000 => "MIPS R5000", Self::R10000 => "MIPS R10000", + Self::R12000 => "MIPS R12000", + Self::R14000 => "MIPS R14000", + Self::R16000 | Self::R16000Mhz700 => "MIPS R16000", } } } @@ -1062,7 +1093,7 @@ impl CpuModel { /// need no tag check against it. /// - `auto`: `shadow` in a `jitv2` build, `full` otherwise. /// -/// The R10000 (IP28) only has the shadow model. +/// The R10000 family only has the shadow model. #[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq, Default, clap::ValueEnum)] #[serde(rename_all = "snake_case")] #[clap(rename_all = "lower")] @@ -1077,7 +1108,7 @@ impl CacheModel { /// Whether `cpu` runs the shadow cache under this setting. pub fn is_shadow(self, cpu: CpuModel) -> bool { match (cpu, self) { - (CpuModel::R10000, _) => true, + (cpu, _) if cpu.is_r10000_family() => true, (_, Self::Full) => false, (_, Self::Shadow) => true, (_, Self::Auto) => cfg!(feature = "jitv2"), @@ -2716,16 +2747,18 @@ pub struct Cli { #[arg(long = "no-scsi-deferred-int", default_value_t = false)] pub no_scsi_deferred_int: bool, - /// Emulated CPU: `r4400` (default), `r5000`, or `r10000`. Overrides `[machine] cpu`. + /// Emulated CPU: `r4400` (default), `r5000`, `r10000`, `r12000`, `r14000`, `r16000` or `r16000_700`. Overrides `[machine] cpu`. /// - /// All three models are compiled into every build — each is its own + /// Every model is compiled into every build — each is its own /// monomorphisation, so the hot path carries no per-model branch — and this /// picks between them at construction. The `r5k` Cargo feature is removed. + /// The Octane takes `r10000`, `r12000` and `r14000`; the O2 `r5000`, + /// `r10000` and `r12000`; the IP28 `r10000`. #[arg(long = "cpu", value_name = "MODEL")] pub cpu: Option, /// Cache model: `auto` (default: `shadow` in a jitv2 build, else `full`), - /// `full` or `shadow`. Overrides `[machine] cache`. The R10000 is always + /// `full` or `shadow`. Overrides `[machine] cache`. The R10000 family is always /// `shadow`. #[arg(long = "cache", value_name = "MODEL")] pub cache: Option, @@ -3447,8 +3480,8 @@ mod export_tests { #[test] fn a_profile_says_which_processors_it_accepts() { let accepted = |p: MachineProfile| CpuModel::ALL.into_iter().filter(|&c| p.accepts_cpu(c).is_ok()).collect::>(); - assert_eq!(accepted(MachineProfile::OctaneIp30), [CpuModel::R10000]); - assert_eq!(accepted(MachineProfile::IndyIp24), [CpuModel::R4400, CpuModel::R5000, CpuModel::R10000]); + assert_eq!(accepted(MachineProfile::OctaneIp30), [CpuModel::R10000, CpuModel::R12000, CpuModel::R14000]); + assert_eq!(accepted(MachineProfile::IndyIp24), CpuModel::ALL); // The counts are what the checks in validate() and the Octane // constructor allowed before they moved here: one on every machine @@ -3532,6 +3565,60 @@ mod export_tests { assert_eq!(back.serial_socket_a.as_deref(), Some("/run/x.sock")); } + /// `cpu` names each part, and the R10000 family runs the shadow caches. + #[test] + fn every_cpu_part_is_selectable_by_name() { + for (name, cpu, family) in [ + ("r4400", CpuModel::R4400, false), + ("r5000", CpuModel::R5000, false), + ("r10000", CpuModel::R10000, true), + ("r12000", CpuModel::R12000, true), + ("r14000", CpuModel::R14000, true), + ("r16000", CpuModel::R16000, true), + ("r16000_700", CpuModel::R16000Mhz700, true), + ] { + let cfg: MachineConfig = toml::from_str(&format!("[machine]\ncpu = \"{name}\"\n")).unwrap(); + assert_eq!(cfg.machine.cpu, cpu, "{name}"); + assert_eq!(cpu.is_r10000_family(), family, "{name}"); + assert_eq!(CacheModel::Full.is_shadow(cpu), family, "{name}"); + assert!(CpuModel::ALL.contains(&cpu), "{name}"); + } + } + + /// The IP28 shipped with the R10000 only, so it refuses the later family + /// parts; the Octane and the O2 each name the parts they take. + #[test] + fn machines_name_the_cpus_they_accept() { + let mut cfg = MachineConfig::default(); + cfg.machine.profile = MachineProfile::Indigo2Ip28; + for cpu in [CpuModel::R12000, CpuModel::R14000, CpuModel::R16000, CpuModel::R16000Mhz700] { + cfg.machine.cpu = cpu; + let err = cfg.validate().expect_err("the IP28 has an R10000"); + assert_eq!(err, "an Indigo2 IMPACT (IP28) has an R10000 ([machine] cpu = \"r10000\")"); + } + cfg.machine.cpu = CpuModel::R10000; + cfg.validate().expect("the IP28 takes the R10000"); + + // Both machines check the part before they look for a PROM. + let refusal = |profile, cpu| { + let mut cfg = MachineConfig::default(); + cfg.machine.profile = profile; + cfg.machine.cpu = cpu; + match profile { + MachineProfile::OctaneIp30 => crate::ip30_machine::OctaneMachine::new(&cfg).err(), + _ => crate::ip32_machine::O2Machine::new(&cfg).err(), + } + }; + let octane = "an Octane has an R10000, R12000 or R14000 ([machine] cpu = \"r10000\", \"r12000\" or \"r14000\")"; + let o2 = "an O2 has an R5000, R10000 or R12000 ([machine] cpu = \"r5000\", \"r10000\" or \"r12000\")"; + for cpu in [CpuModel::R4400, CpuModel::R5000, CpuModel::R16000, CpuModel::R16000Mhz700] { + assert_eq!(refusal(MachineProfile::OctaneIp30, cpu).as_deref(), Some(octane), "{cpu:?}"); + } + for cpu in [CpuModel::R4400, CpuModel::R14000, CpuModel::R16000, CpuModel::R16000Mhz700] { + assert_eq!(refusal(MachineProfile::O2Ip32, cpu).as_deref(), Some(o2), "{cpu:?}"); + } + } + /// Every old `[o2]` key lands on the shared key it stood for, and the /// section keeps only what is the O2's alone. #[test] diff --git a/src/cpu/jitv2/equiv_test.rs b/src/cpu/jitv2/equiv_test.rs index 24a631f3f..54f3ebd17 100644 --- a/src/cpu/jitv2/equiv_test.rs +++ b/src/cpu/jitv2/equiv_test.rs @@ -4405,15 +4405,15 @@ mod tests { (CoreSnapshot::capture(&interp.core), CoreSnapshot::capture(&jit.core)) } - /// The R10000 keeps bits 63:32 on a partial LWR while 64-bit operations are - /// enabled for the current mode and the R5000 sign-extends; with them off - /// (a 32-bit user process, UX=0) the R10000 sign-extends too. The JIT and - /// the interpreter must agree at every offset in every mode, and the - /// interpreter's result is the one `partial_lwr_keeps_the_upper_word_only_on_the_r10000` - /// pins. - #[test] - fn jit_lwr_matches_the_interpreter_on_the_r10000_and_r5000() { - use crate::cpu::mips_cache_shadow::{R10000ShadowCache, R5000ShadowCache}; + /// The R10000 family keeps bits 63:32 on a partial LWR while 64-bit + /// operations are enabled for the current mode and the R5000 sign-extends; + /// with them off (a 32-bit user process, UX=0) the family sign-extends too. + /// The JIT and the interpreter must agree at every offset in every mode, and + /// the interpreter's result is the one + /// `partial_lwr_keeps_the_upper_word_on_the_r10000_family` pins. + #[test] + fn jit_lwr_matches_the_interpreter_on_the_r10000_family_and_r5000() { + use crate::cpu::mips_cache_shadow::{R10000ShadowCache, R12000ShadowCache, R16000ShadowCache, R5000ShadowCache}; fn check>>(model_keeps_upper: bool) { use crate::cpu::mips_core::{STATUS_ERL, STATUS_EXL, STATUS_KX, STATUS_SX, STATUS_UX}; const RESERVED: u32 = 3 << 3; @@ -4455,6 +4455,8 @@ mod tests { } } check::(true); + check::(true); + check::(true); check::(false); } @@ -7988,7 +7990,7 @@ mod tests { /// masked-store helper), and the instructions after it run. The same access /// as a load, and a store on the R5000, still fault in both engines. #[test] - fn refused_uncached_store_matches_the_interpreter_on_the_r10000_and_r5000() { + fn refused_uncached_store_matches_the_interpreter_on_the_r10000_family_and_r5000() { use crate::cpu::mips_cache_v2::{R10000Cache, R5000Cache, CpuModel}; use crate::cpu::mips_core::STATUS_EXL; use crate::cpu::mips_isa::{OP_ADDIU, OP_LW, OP_SW, OP_SWL}; diff --git a/src/cpu/mips_cache_shadow.rs b/src/cpu/mips_cache_shadow.rs index e58d37a29..2832438c3 100644 --- a/src/cpu/mips_cache_shadow.rs +++ b/src/cpu/mips_cache_shadow.rs @@ -36,7 +36,7 @@ use std::sync::Arc; use crate::cpu::mips_cache_v2::{ cache_op_name, model, r4k_load_taglo, r4k_store_taglo, CpuModel, FetchInstrResult, MipsCache, - C_ILT, C_IST, C_R10K_CBARRIER, C_R10K_ILD, C_R10K_ISD, CACH_PD, CACH_PI, CACH_SD, CACH_SI, + C_ILT, C_IST, C_IWBINV, C_R10K_CBARRIER, C_R10K_ILD, C_R10K_ISD, CACH_PD, CACH_PI, CACH_SD, CACH_SI, }; use crate::traits::{BusDevice, BusRead64, BUS_OK}; use crate::devlog::{LogModule, CACHE_LOG_HIT, CACHE_LOG_OP, devlog_is_active, devlog_mask}; @@ -189,8 +189,12 @@ impl< const FIR: u32, const TLB_ENTRIES: usize, const MODEL: u8, > ShadowCache { - const R10K: bool = MODEL == model::R10000; + const R10K: bool = model::r10k_family(MODEL); const R5K: bool = MODEL == model::R5000; + /// Index Writeback Invalidate (D) also zeroes the primary data tag on the + /// R12000 and later; the R10000 leaves it as it was. NEC VR12000 user's + /// manual, Appendix B.20. The R14000 and R16000 are taken to match. + const IWBI_CLEARS_PD_TAG: bool = Self::R10K && MODEL != model::R10000; const IC_LINES: usize = if IC_LINE == 0 { 0 } else { IC_SIZE / IC_LINE }; const DC_LINES: usize = if DC_LINE == 0 { 0 } else { DC_SIZE / DC_LINE }; const L2_LINES: usize = if L2_LINE == 0 { 0 } else { L2_SIZE / L2_LINE }; @@ -367,11 +371,10 @@ impl< const PRID: u32 = PRID; const FIR: u32 = FIR; const TLB_ENTRIES: usize = TLB_ENTRIES; - // The R10000 implements 44 virtual address bits where the R4x00 implements - // 40; the shadow cache is only used for it, but key this off the same flag - // that selects its cache encodings rather than asserting it unconditionally. - const VA_BITS: u32 = if MODEL == model::R10000 { 44 } else { 40 }; - const PA_BITS: u32 = if MODEL == model::R10000 { 40 } else { 36 }; + // The R10000 family implements 44 virtual address bits where the R4x00 + // implements 40. + const VA_BITS: u32 = if Self::R10K { 44 } else { 40 }; + const PA_BITS: u32 = if Self::R10K { 40 } else { 36 }; // The CPU's name, which snapshots check: the same as the full model's, so // a mismatch reads as a cache-kind difference rather than a different CPU. // The R10000 keeps its historical "shadow", which existing IP28 @@ -379,14 +382,20 @@ impl< const NAME: &'static str = match MODEL { model::R4400 => "R4400", model::R5000 => "R5000", + model::R12000 => "R12000", + model::R16000 => "R16000", _ => "shadow", }; const CACHE_KIND: &'static str = "shadow"; const R10K_FAMILY: bool = Self::R10K_CACHE_OPS; - const PARTIAL_LWR_KEEPS_UPPER: bool = MODEL == model::R10000; - const R10K_CACHE_OPS: bool = MODEL == model::R10000; + // The upper word of a partial LWR is implementation-defined. Keeping it + // is what the IRIX CPU instruction test (olperi) expects and what + // qemu-sgi-origami models; for the R12000 and later it is a model choice. + const PARTIAL_LWR_KEEPS_UPPER: bool = Self::R10K_FAMILY; + const R10K_CACHE_OPS: bool = Self::R10K; const POSTS_UNCACHED_STORES: bool = Self::R10K_CACHE_OPS; const SECONDARY_TAG_MASK: u64 = if Self::R10K_CACHE_OPS { R10K_L2_TAG_MASK } else { u64::MAX }; + const CONFIG0_RESET: u32 = if MODEL == model::R16000 { R16000_800_CONFIG0 } else { 0 }; } impl< @@ -667,6 +676,15 @@ impl< 0 } + C_IWBINV if Self::IWBI_CLEARS_PD_TAG && sel == CACH_PD => { + let idx = self.tag_slot(sel, virt_addr); + let s = self.shadow(sel); + if idx < s.tags.len() { + s.tags[idx] = 0; + } + 0 + } + // Invalidate, writeback, fill, and the R4000 hit operations. All // no-ops: there is nothing held that could be stale or dirty. _ => 0, @@ -767,6 +785,68 @@ impl< pub type R10000ShadowCache = ShadowCache<32768, 64, 32768, 32, 1048576, 128, true, 0x0000_0925, 0x0000_0900, 64, { model::R10000 }>; +/// R12000 3.5 as the Octane has it: the R10000's programming model with a +/// 2 MB secondary cache. PRId 0x0e35, FIR 0x900, 128-byte secondary lines +/// (SB: a model choice). +/// +/// The Octane's R14000 is this type with `R14000_PRID` set at runtime, as the +/// Octane's own R10000 revision is; only the PRId differs. Both report +/// "R12000" as `NAME`. +pub type R12000ShadowCache = + ShadowCache<32768, 64, 32768, 32, 2097152, 128, true, 0x0000_0e35, 0x0000_0900, 64, { model::R12000 }>; + +/// PRId of the R14000 2.4 in the Octane, to set on an `R12000ShadowCache`. +/// Implementation 0x0f is shared with the R16000, which is told apart by a +/// major revision above 2. +pub const R14000_PRID: u32 = 0x0000_0f24; + +/// R12000 3.5 as the O2 has it: the Octane part with 64-byte secondary lines +/// (SB clear). With 128-byte lines the O2 PROM panics before its banner +/// (write TLB miss at 0x0); 64 boots. IRIX on the O2 never reads SB and sizes +/// the secondary cache as 512 KB << SS. +pub type R12000O2ShadowCache = + ShadowCache<32768, 64, 32768, 32, 2097152, 64, true, 0x0000_0e35, 0x0000_0900, 64, { model::R12000 }>; + +/// R16000 2.2, the 800 MHz part, with a 4 MB secondary cache. Only +/// `iris-bench` builds it for now. +/// +/// Values and where they come from: +/// - PRId 0x0f42: the low 16 bits are printed in hinv listings (revision 2.2 +/// under the R16000 name; the major nibble is the printed major plus 2). +/// Bits 31:16 are zero by inference from every full PRId read on this family. +/// - FIR 0x900: a model choice. No R16000 FIR has been printed; this is the +/// R12000 and R14000 value, kept for compatibility. +/// - Config reset word 0x009baf85 (`R16000_800_CONFIG0`): derived +/// from the PROM's per-part clock table (printed clocks and cache size fit +/// it), not read from a running part. SS (4 MB) is inferred from the printed +/// size, the clock fields from the printed 800 MHz CPU and 400 MHz cache +/// clocks. 128-byte lines (SB) are a model choice, and the K0, DN, PM, SK, +/// CT, PE, DDR and bit 24 fields are unknown, carried as the table has them. +/// - Cache tap 0xa is printed in hinv listings and is not modelled. +pub type R16000ShadowCache = + ShadowCache<32768, 64, 32768, 32, 4194304, 128, true, 0x0000_0f42, 0x0000_0900, 64, { model::R16000 }>; + +/// The Config bits an R16000 part's reset word defines: the low 25. IC and DC +/// above them stay the family's. +pub const CONFIG0_RESET_MASK: u32 = 0x01ff_ffff; + +/// Config reset word of `R16000ShadowCache`, the 800 MHz 4 MB part. Same +/// grades as listed there. +pub const R16000_800_CONFIG0: u32 = 0x009b_af85; + +/// PRId of the R16000 2.1, the 700 MHz part, to set on an `R16000ShadowCache`. +/// The low 16 bits are printed in hinv listings as revision 2.1; bits 31:16 are +/// zero by inference. Nothing else in the model differs: both parts have a +/// 4 MB secondary cache. +pub const R16000_MHZ700_PRID: u32 = 0x0000_0f41; + +/// Config reset word of the 700 MHz part. It differs from the 800 MHz word in +/// the EC field only (6 where the 800 MHz part has 7). Derived from the PROM's +/// per-part clock table with the same grades as the 800 MHz word: the 700 MHz +/// CPU clock and 350 MHz cache clock are printed, the other fields unknown. +/// Cache tap 0xc is printed and not modelled. +pub const R16000_MHZ700_CONFIG0: u32 = 0x009b_ad85; + /// SGI O2 R10000 (IP32): the IP28 model with 64-byte secondary-cache lines. /// /// The O2 PROM (4.18 and earlier) handles only that line size. With Config's @@ -988,6 +1068,25 @@ mod tests { assert_eq!(c.read::<4>(0, 0x2000).data, 0x1234_5678); } + /// On the R12000 and later Index Writeback Invalidate (D) zeroes the + /// line's primary data tag, which Index Load Tag then reads back; the + /// R10000 keeps the tag. + #[test] + fn index_writeback_invalidate_d_clears_the_tag_from_the_r12000_on() { + fn run>>() -> u64 { + let mem: Arc = Arc::new(Memory::new(1)); + let c = C::from(mem); + let tag = 0x0000_0001_2345_6780u64; + c.cache_op(crate::cpu::mips_cache_v2::C_IST | CACH_PD, 0x40, tag); + assert_eq!(c.cache_op(crate::cpu::mips_cache_v2::C_ILT | CACH_PD, 0x40, 0), tag); + c.cache_op(crate::cpu::mips_cache_v2::C_IWBINV | CACH_PD, 0x40, 0); + c.cache_op(crate::cpu::mips_cache_v2::C_ILT | CACH_PD, 0x40, 0) + } + assert_eq!(run::(), 0); + assert_eq!(run::(), 0); + assert_eq!(run::(), 0x0000_0001_2345_6780); + } + /// Geometry is what CP0 Config is built from, so it must be the real /// part's even though the model holds nothing. #[test] @@ -1044,6 +1143,24 @@ mod tests { (::PRID, ::FIR, ::MIPS4, ::TLB_ENTRIES, ::VA_BITS, ::R10K_CACHE_OPS)); } + /// Each R10000-family part reports its own identity and geometry and shares + /// the family's address widths and cache operations; the R5000 does not. + #[test] + fn the_r10000_family_parts_report_their_own_identity() { + fn id() -> (u32, usize, usize, &'static str, u32, u32, bool) { + (C::PRID, C::L2_SIZE, C::L2_LINE, C::NAME, C::VA_BITS, C::PA_BITS, C::R10K_CACHE_OPS) + } + assert_eq!(id::(), (0x925, 1 << 20, 128, "shadow", 44, 40, true)); + assert_eq!(id::(), (0xe35, 2 << 20, 128, "R12000", 44, 40, true)); + assert_eq!(id::(), (0xe35, 2 << 20, 64, "R12000", 44, 40, true)); + assert_eq!(id::(), (0xf42, 4 << 20, 128, "R16000", 44, 40, true)); + assert_eq!(id::().6, false); + // The 700 MHz part is the same type with a PRId set at build time, as + // the Octane's R14000 is the R12000 type. + assert_eq!(R16000_MHZ700_PRID, 0xf41); + assert_eq!(R16000_MHZ700_PRID >> 4 & 0xf, 4, "major nibble is printed major 2 plus 2"); + } + /// The tag store hook sees primary data tag stores and nothing else. #[test] fn the_tag_store_hook_sees_only_primary_data_tag_stores() { diff --git a/src/cpu/mips_cache_v2.rs b/src/cpu/mips_cache_v2.rs index c1f7db120..56c5382ef 100644 --- a/src/cpu/mips_cache_v2.rs +++ b/src/cpu/mips_cache_v2.rs @@ -586,6 +586,9 @@ pub trait CpuModel: MipsCache { /// Bits `Index_Store_Tag` keeps for the secondary cache: all of them, except /// on the R10000 family, whose reserved tag bits read as zero (R10000 UM p. 270). const SECONDARY_TAG_MASK: u64 = u64::MAX; + /// The low 25 bits of Config at reset, where the part's own word is known + /// and differs from the R10000 family's generic one. Zero keeps the generic. + const CONFIG0_RESET: u32 = 0; } pub trait MipsCache: Send + Sync { @@ -1406,6 +1409,14 @@ pub mod model { pub const R4400: u8 = 0; pub const R5000: u8 = 1; pub const R10000: u8 = 2; + pub const R12000: u8 = 3; + pub const R16000: u8 = 4; + + /// The R10000 family: the R10000 and the parts that follow it, which share + /// one programming model. + pub const fn r10k_family(model: u8) -> bool { + matches!(model, R10000 | R12000 | R16000) + } } /// SGI Indy R4400: direct-mapped 16K L1s, 1 MB unified L2 owning the decode slots. diff --git a/src/cpu/mips_exec.rs b/src/cpu/mips_exec.rs index ecfe79c38..0e7e64ff4 100644 --- a/src/cpu/mips_exec.rs +++ b/src/cpu/mips_exec.rs @@ -2322,16 +2322,15 @@ impl MipsExecutor { c |= log2(32768) << 26; // 32 KB L1D c |= 1 << 15; // big endian if C::L2_LINE == 128 { c |= 1 << 13; } - // Secondary cache size. The encoding is not in anything to hand, - // so it was swept against the PROM rather than guessed. - // - // 1 is the value to keep: IRIX reports "Secondary unified - // instruction/data cache size: 1 Mbyte", which agrees with this - // model's own `L2_SIZE`, and the PROM's power-on diagnostics pass - // and IRIX boots with it. 4 also passes POST but has IRIX report - // 8 MB, contradicting the model. - c |= 1 << 16; + // SS is log2(secondary cache size / 512 KB): 0 is 512 KB, 1 is 1 MB, + // 2 is 2 MB, up to 5 for 16 MB (NEC U10278EJ4V0UM p.230). It follows + // the model's own `L2_SIZE`. EC, SC, DN, PM, DSD and SK stay zero. + c |= (C::L2_SIZE / (512 << 10)).trailing_zeros() << 16; c |= 2; // K0 = uncached at reset + // A part with a known reset word presents it over the low 25 bits. + if C::CONFIG0_RESET != 0 { + c = (c & !crate::cpu::mips_cache_shadow::CONFIG0_RESET_MASK) | C::CONFIG0_RESET; + } config = c; } diff --git a/src/cpu/mips_exec_test.rs b/src/cpu/mips_exec_test.rs index 9cf022b64..b7734c0c7 100644 --- a/src/cpu/mips_exec_test.rs +++ b/src/cpu/mips_exec_test.rs @@ -6808,9 +6808,9 @@ mod tests { .unwrap().join().unwrap(); } - /// A partial LWR keeps bits 63:32 of the destination on the R10000 while - /// 64-bit operations are enabled for the current mode, and sign-extends on - /// the R4400 and R5000 and on the R10000 when they are off (a 32-bit user + /// A partial LWR keeps bits 63:32 of the destination on the R10000 family + /// while 64-bit operations are enabled for the current mode, and sign-extends + /// on the R4400 and R5000 and on the family when they are off (a 32-bit user /// process, UX=0); a complete word sign-extends on all. /// Compiled LWR reads `core.lwr_upper_below` rather than decoding Status, /// so it must follow `is_64bit_mode()` through a Status write and through @@ -6842,7 +6842,7 @@ mod tests { } #[test] - fn partial_lwr_keeps_the_upper_word_only_on_the_r10000() { + fn partial_lwr_keeps_the_upper_word_on_the_r10000_family() { use crate::cpu::mips_core::{STATUS_KX, STATUS_SX, STATUS_UX}; const USER: u32 = 2 << 3; const SUPER: u32 = 1 << 3; @@ -6880,13 +6880,13 @@ mod tests { check::(false); check::(false); check::(true); + check::(true); + check::(true); } /// FrameMask and the wide EntryLo belong to the whole R10000 family, not to - /// the parts that share the R10000's partial LWR. No R12000 type exists on - /// this branch, so this cannot run a family part whose LWR differs from - /// the R10000's; it checks the executor's masks against the family constant - /// the gate reads, and that the two behaviours are separate constants. + /// the partial LWR alone. This checks the executor's masks against the + /// family constant the gate reads; the R12000 and R16000 are covered below. #[test] fn framemask_and_entrylo_widths_follow_the_family_constant() { fn check>>() { @@ -6903,4 +6903,79 @@ mod tests { check::(); check::(); } + + /// Config reports each part's secondary cache: SS (bits 18:16) is + /// log2(secondary size / 512 KB) and SB (bit 13) is set for 128-byte lines + /// (NEC U10278EJ4V0UM p.230). The primary caches read 32 KB. + #[test] + fn config_reports_the_secondary_cache_of_each_family_part() { + use crate::cpu::mips_cache_shadow::*; + fn config>>() -> u64 { + let cfg = MipsCpuConfig::for_model::(Arc::new(crate::ip22_bus::memory_map(false))); + let mut exec = Box::new(MipsExecutor::::new( + Arc::new(MockMemory::new()), PassthroughTlb::default(), &cfg)); + disable_jitv2(&mut exec); + exec.core.read_cp0(16) + } + // The executor is large; build it on a thread with room. + std::thread::Builder::new().stack_size(16 * 1024 * 1024).spawn(|| { + for (name, c, ss, sb) in [ + ("R10000", config::(), 1, 1), + ("R12000", config::(), 2, 1), + ("R12000 O2", config::(), 2, 0), + ] { + assert_eq!(c >> 26, 0b011_011, "{name}: IC and DC read 32 KB"); + assert_eq!(c & 0x03FF_FFFF, (ss << 16) | (sb << 13) | 0x8000 | 2, "{name}: Config {c:#x}"); + } + // The R16000 presents its part's whole reset word over the low 25 + // bits: SS 3 and SB 1 as for the generic family, plus the clock + // fields. The 700 MHz part is the same type with its word set at + // build time. + let c = config::(); + assert_eq!(c >> 26, 0b011_011, "R16000: IC and DC read 32 KB"); + assert_eq!(c & CONFIG0_RESET_MASK as u64, 0x009b_af85, "R16000: Config {c:#x}"); + assert_eq!((c >> 16) & 7, 3, "R16000: 4 MB secondary cache"); + assert_eq!((c >> 13) & 1, 1, "R16000: 128-byte lines"); + assert_eq!(R16000_MHZ700_CONFIG0 ^ 0x009b_af85, 1 << 9, "the 700 MHz word differs in EC only"); + }).unwrap().join().unwrap(); + } + + /// The R12000 and R16000 keep the 16-bit FrameMask, the 64-bit-family + /// EntryLo mask and the partial-LWR upper word of the R10000. A TLB write on + /// an R12000 with FrameMask set clears EntryLo bit 18, as on the R10000. + #[test] + fn framemask_and_entrylo_widths_hold_on_the_r12000_and_r16000() { + use crate::cpu::mips_cache_shadow::*; + fn masks>>() -> (u32, u64) { + let cfg = MipsCpuConfig::for_model::(Arc::new(crate::ip22_bus::memory_map(false))); + let exec = Box::new(MipsExecutor::::new(Arc::new(MockMemory::new()), PassthroughTlb::default(), &cfg)); + (exec.core.framemask_mask, exec.core.entrylo_mask) + } + // Write the entry with FrameMask set, find it with TLBP, clear EntryLo0 + // and TLBR it: (FrameMask, EntryLo0 as the TLB holds it, physical page). + fn written>>() -> (u64, u64, u64) { + const VA: u64 = 0x4000_0000; + let mut exec = map_page_masked::(Arc::new(MockMemory::new()), VA, 0x0300_5000, 0xFFFF_0001); + let phys = exec.debug_translate(VA).phys; + for (rd, value) in [(10, VA), (2, 0)] { + exec.core.write_gpr(1, value); + assert_eq!(exec.exec((OP_COP0 << 26) | (RS_DMTC0 << 21) | (1 << 16) | (rd << 11)), EXEC_COMPLETE); + } + assert_eq!(exec.exec((OP_COP0 << 26) | (RS_TLB << 21) | FUNCT_TLBP), EXEC_COMPLETE); + assert_eq!(exec.exec((OP_COP0 << 26) | (RS_TLB << 21) | FUNCT_TLBR), EXEC_COMPLETE); + (exec.core.read_cp0(21), exec.core.read_cp0(2), phys) + } + on_big_stack(|| { + const ENTRYLO: u64 = 0xC000_0003_FFFF_FFFF; + assert!(R12000ShadowCache::R10K_FAMILY && R12000O2ShadowCache::R10K_FAMILY && R16000ShadowCache::R10K_FAMILY); + assert!(R12000ShadowCache::PARTIAL_LWR_KEEPS_UPPER && R12000O2ShadowCache::PARTIAL_LWR_KEEPS_UPPER); + assert!(R16000ShadowCache::PARTIAL_LWR_KEEPS_UPPER); + assert_eq!(masks::(), (0xFFFF, ENTRYLO), "R12000"); + assert_eq!(masks::(), (0xFFFF, ENTRYLO), "R12000 O2"); + assert_eq!(masks::(), (0xFFFF, ENTRYLO), "R16000"); + // Physical bit 24 is EntryLo bit 18, FrameMask bit 0. + let masked = (((0x0300_5000u64 >> 12) << 6) | (3 << 3) | (1 << 2) | (1 << 1)) & !(1 << 18); + assert_eq!(written::(), (1, masked, 0x0200_5000), "R12000"); + }); + } } diff --git a/src/ip30_machine.rs b/src/ip30_machine.rs index 37051b4d8..2c2c18bf1 100644 --- a/src/ip30_machine.rs +++ b/src/ip30_machine.rs @@ -16,7 +16,8 @@ use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::Arc; use crate::config::{CpuModel, MachineConfig, MachineProfile}; -use crate::cpu::mips_cache_shadow::R10000ShadowCache; +use crate::cpu::mips_cache_v2::MipsCache; +use crate::cpu::mips_cache_shadow::{R10000ShadowCache, R12000ShadowCache, R14000_PRID}; use crate::cpu::smp::World; use crate::cpu::mips_exec::{CpuDevice, MipsCpu, MipsCpuConfig, MipsExecutor}; use crate::cpu::mips_tlb::MipsTlb; @@ -49,9 +50,29 @@ pub const OCTANE_PROM_FILE: &str = "IP30prom.bin"; /// The 195 MHz R10000 of SGI's standard 128 MB Octane configurations (the /// OCTANE Technical Report): Count at half that. pub const OCTANE_COUNT_HZ: u64 = 97_500_000; +/// Count for the 360 MHz R12000 (revision 3.5, 2 MB) that SGI sold in the +/// Octane: half the pipeline clock. +pub const OCTANE_R12000_COUNT_HZ: u64 = 180_000_000; +/// Count for the 600 MHz R14000 (revision 2.4, 2 MB) that SGI sold in the +/// Octane: half the pipeline clock. +pub const OCTANE_R14000_COUNT_HZ: u64 = 300_000_000; + /// R10000 revision 2.7, as an Octane's hinv reports it; the raw PRId is /// inferred from the R10000 manual's encoding. pub const OCTANE_PRID: u32 = 0x0927; + +/// CP0 Count for a part when `[clock] fixed_mhz` is unset: half the clock of +/// one speed that part shipped at in an Octane, so a plain `cpu = "r12000"` +/// or `"r14000"` reports a clock its tuple had. The PROM measures Count +/// against the HEART timer, so `hinv` prints twice this. +fn octane_default_count_hz(cpu: CpuModel) -> u64 { + match cpu { + CpuModel::R12000 => OCTANE_R12000_COUNT_HZ, + CpuModel::R14000 => OCTANE_R14000_COUNT_HZ, + _ => OCTANE_COUNT_HZ, + } +} + /// The Crossbow on the Octane reports revision code 4. const XBOW_REVISION: u32 = 4; /// The Octane2's Crossbow: revision code 5, which IRIX's inventory names @@ -353,7 +374,7 @@ impl OctaneMachine { if has_vpro { kind.label() } else { board.map_or("no graphics", |b| b.label()) } ); - let count_hz = cfg.clock.fixed_mhz.map_or(OCTANE_COUNT_HZ, |mhz| (mhz * 1_000_000.0) as u64); + let count_hz = cfg.clock.fixed_mhz.map_or(octane_default_count_hz(cfg.machine.cpu), |mhz| (mhz * 1_000_000.0) as u64); let flash_lines: Vec<_> = (0..cpu_count) .map(|_| { let bus: Arc = phys.phys.clone(); @@ -365,7 +386,7 @@ impl OctaneMachine { ticks.extend((1..cpu_count).map(|_| Arc::new(AtomicU64::new(0)))); let memory = Arc::new(crate::ip30::memory_map(&cfg.banks[..4])); let cpus: Vec> = (0..cpu_count) - .map(|id| Self::build_cpu(id, cpu_count, memory.clone(), phys.phys.clone(), flash_lines[id as usize].clone(), &timer_manager, count_hz.max(1), ticks[id as usize].clone())) + .map(|id| Self::build_cpu_for(cfg.machine.cpu, id, cpu_count, memory.clone(), phys.phys.clone(), flash_lines[id as usize].clone(), &timer_manager, count_hz.max(1), ticks[id as usize].clone())) .collect(); let cpu = cpus[0].clone(); world.set_slots(cpus.iter().map(|c| c.park_slot()).collect()); @@ -575,8 +596,12 @@ impl OctaneMachine { } } + /// The configured part, as an executor over its cache model. The R10000 + /// takes the Octane's own revision (`OCTANE_PRID`) and the R14000 is the + /// R12000's 2 MB model with its PRId; the R12000 keeps its model's. #[allow(clippy::too_many_arguments)] - fn build_cpu( + fn build_cpu_for( + model: CpuModel, id: u8, cpu_count: u8, memory: Arc, @@ -586,16 +611,46 @@ impl OctaneMachine { count_hz: u64, fasttick: Arc, ) -> Arc { - let cfg = MipsCpuConfig::for_model::(memory); + macro_rules! build { + ($c:ty, $prid:expr) => { + Self::build_cpu::<$c>(id, cpu_count, memory, sysad, cached, tm, count_hz, fasttick, $prid) + }; + } + match model { + CpuModel::R12000 => build!(R12000ShadowCache, None), + CpuModel::R14000 => build!(R12000ShadowCache, Some(R14000_PRID)), + _ => build!(R10000ShadowCache, Some(OCTANE_PRID)), + } + } + + #[allow(clippy::too_many_arguments)] + fn build_cpu( + id: u8, + cpu_count: u8, + memory: Arc, + sysad: Arc, + cached: Arc, + tm: &Arc, + count_hz: u64, + fasttick: Arc, + prid: Option, + ) -> Arc + where + C: crate::cpu::mips_cache_v2::CpuModel + MipsCache + From> + 'static, + MipsCpu: CpuDevice, + { + let cfg = MipsCpuConfig::for_model::(memory); let tlb = MipsTlb::new(cfg.tlb_entries); - let mut exec: MipsExecutor = MipsExecutor::new(sysad, tlb, &cfg); + let mut exec: MipsExecutor = MipsExecutor::new(sysad, tlb, &cfg); // Cached accesses see the PROM's dirty flash lines; see `CachedFlashLines`. - exec.cache = R10000ShadowCache::from(cached as Arc); + exec.cache = C::from(cached as Arc); exec.core.set_timer_manager(tm.clone()); exec.core.set_count_hz(count_hz); exec.core.fasttick_count = fasttick; - exec.core.reset_prid = OCTANE_PRID; - exec.core.cp0_prid = OCTANE_PRID; + if let Some(prid) = prid { + exec.core.reset_prid = prid; + exec.core.cp0_prid = prid; + } // IRIX zeroes Count and arms Compare a few instructions later; see // `MipsCore::count_write_samples`. exec.core.count_write_samples = true; @@ -605,7 +660,7 @@ impl OctaneMachine { exec.phys_mask = (1 << 40) - 1; exec.core.cpu_id = id; exec.core.smp = cpu_count > 1; - exec.batch_hook = Some(Box::new(move |exec: &mut MipsExecutor| { + exec.batch_hook = Some(Box::new(move |exec: &mut MipsExecutor| { LAST_PC[id as usize].store(exec.core.pc, Ordering::Relaxed); })); Arc::new(MipsCpu::new(exec)) @@ -1469,3 +1524,18 @@ mod snapshot_tests { }).unwrap().join().unwrap(); } } + +#[cfg(test)] +mod cpu_clock_tests { + use super::*; + + /// A plain `cpu = ...` reports a clock that part shipped at in an Octane: + /// 195 MHz for the R10000, 360 MHz for the R12000 (revision 3.5) and + /// 600 MHz for the R14000 (revision 2.4). Count runs at half the clock. + #[test] + fn each_octane_part_defaults_to_a_shipped_clock() { + for (cpu, mhz) in [(CpuModel::R10000, 195), (CpuModel::R12000, 360), (CpuModel::R14000, 600)] { + assert_eq!(octane_default_count_hz(cpu) * 2, mhz * 1_000_000, "{cpu:?}"); + } + } +} diff --git a/src/ip32_machine.rs b/src/ip32_machine.rs index a7a42a6d4..cc42d6ffb 100644 --- a/src/ip32_machine.rs +++ b/src/ip32_machine.rs @@ -27,7 +27,7 @@ use parking_lot::Mutex; use crate::config::{CpuModel, MachineConfig, MachineProfile}; use crate::cpu::mips_core::CyclesPtr; use crate::cpu::mips_exec::{CpuDevice, MipsCpu, MipsCpuConfig, MipsExecutor}; -use crate::cpu::mips_cache_shadow::{R10000O2ShadowCache, R5000ShadowCache}; +use crate::cpu::mips_cache_shadow::{R10000O2ShadowCache, R12000O2ShadowCache, R5000ShadowCache}; use crate::cpu::mips_cache_v2::MipsCache; use crate::cpu::mips_tlb::MipsTlb; use crate::dev::crime_mem::O2Memory; @@ -157,8 +157,23 @@ pub const O2_PROM_FILE: &str = "ip32prom.rev4.18.bin"; /// CP0 Count when `[clock] fixed_mhz` is unset: half of a 180 MHz CPU. pub const O2_COUNT_MHZ: f64 = 90.0; +/// CP0 Count for the O2's R12000 when `[clock] fixed_mhz` is unset: half of +/// 400 MHz, the speed O2 listings show for revision 3.5. +pub const O2_R12000_COUNT_MHZ: f64 = 200.0; + +/// CP0 Count for a part when `[clock] fixed_mhz` is unset. +fn o2_default_count_mhz(cpu: CpuModel) -> f64 { + match cpu { + CpuModel::R12000 => O2_R12000_COUNT_MHZ, + _ => O2_COUNT_MHZ, + } +} + impl O2Machine { pub fn new(cfg: &MachineConfig) -> Result { + if !matches!(cfg.machine.cpu, CpuModel::R5000 | CpuModel::R10000 | CpuModel::R12000) { + return Err("an O2 has an R5000, R10000 or R12000 ([machine] cpu = \"r5000\", \"r10000\" or \"r12000\")".into()); + } let o2 = &cfg.o2; // `prom`, or the last release's file when `prom` is the shared default // and that file is not there: there is no embedded O2 PROM. @@ -235,14 +250,15 @@ impl O2Machine { let arcs_fpu = o2.arcs_fpu; // `[clock] fixed_mhz` is CP0 Count; the CPU clock the PROM measures // and reports is twice it. - let count_mhz = cfg.clock.fixed_mhz.unwrap_or(O2_COUNT_MHZ); + let count_mhz = cfg.clock.fixed_mhz.unwrap_or(o2_default_count_mhz(cfg.machine.cpu)); let cpu_mhz = (count_mhz * 2.0).round().max(1.0) as u32; let boot = Arc::new(BootState::new()); let memory = Arc::new(crate::ip32::memory_map()); let cpu: Arc = match cfg.machine.cpu { CpuModel::R10000 => Self::build_cpu::(memory.clone(), sysad, &timer_manager, arcs_fpu, cpu_mhz, boot.clone(), bus.ram.clone()), CpuModel::R5000 => Self::build_cpu::(memory.clone(), sysad, &timer_manager, arcs_fpu, cpu_mhz, boot.clone(), bus.ram.clone()), - CpuModel::R4400 => return Err("an O2 has an R5000 or an R10000 ([machine] cpu)".into()), + CpuModel::R12000 => Self::build_cpu::(memory.clone(), sysad, &timer_manager, arcs_fpu, cpu_mhz, boot.clone(), bus.ram.clone()), + CpuModel::R4400 | CpuModel::R14000 | CpuModel::R16000 | CpuModel::R16000Mhz700 => unreachable!("refused at the top of `new`"), }; // ppmem: the CPU's cache and the JIT reach RAM through the memory's @@ -1192,3 +1208,19 @@ fn control_command(bus: &Ip32Bus, cmd: &str, arg: &str, rgba: &mut [u8]) -> Stri } } + +#[cfg(test)] +mod cpu_clock_tests { + use super::*; + + /// A plain `cpu = "r12000"` reports the 400 MHz the revision 3.5 part + /// shipped at; the others keep the 180 MHz default. Count runs at half + /// the clock. + #[test] + fn the_o2_r12000_defaults_to_400_mhz() { + assert_eq!(o2_default_count_mhz(CpuModel::R12000) * 2.0, 400.0); + for cpu in [CpuModel::R5000, CpuModel::R10000] { + assert_eq!(o2_default_count_mhz(cpu) * 2.0, 180.0, "{cpu:?}"); + } + } +} diff --git a/src/machine.rs b/src/machine.rs index 2bc5ac217..3808a7430 100644 --- a/src/machine.rs +++ b/src/machine.rs @@ -27,7 +27,10 @@ use crate::cpu::mips_tlb::MipsTlb; use crate::cpu::mips_exec::{MipsExecutor, MipsCpu, MipsCpuConfig, MipsCpuDebugAdapter}; use crate::gdb_stub::CpuDebug; use crate::cpu::mips_cache_v2::{MipsCache, R4400Cache, R5000Cache}; -use crate::cpu::mips_cache_shadow::{R10000ShadowCache, R4400ShadowCache, R5000ShadowCache}; +use crate::cpu::mips_cache_shadow::{ + CONFIG0_RESET_MASK, R10000ShadowCache, R12000ShadowCache, R14000_PRID, R16000ShadowCache, R16000_MHZ700_CONFIG0, + R16000_MHZ700_PRID, R4400ShadowCache, R5000ShadowCache, +}; use crate::dev::hpc3::Hpc3; use crate::dev::ioc::{Ioc, GioSlot, GIO_SLOT_MAP, profile_idx}; use crate::monitor::Monitor; @@ -354,6 +357,11 @@ impl Machine { crate::config::CpuModel::R4400 => ::L2_SIZE > 0, crate::config::CpuModel::R5000 => ::L2_SIZE > 0, crate::config::CpuModel::R10000 => ::L2_SIZE > 0, + crate::config::CpuModel::R12000 => ::L2_SIZE > 0, + crate::config::CpuModel::R14000 => ::L2_SIZE > 0, + crate::config::CpuModel::R16000 | crate::config::CpuModel::R16000Mhz700 => { + ::L2_SIZE > 0 + } }; if !model_has_l2 { eeprom_mc.lock().set_cachsz(0); @@ -741,12 +749,22 @@ impl Machine { // arm below monomorphises its own CPU — no per-model branch on the hot path. let sysad: Arc = phys.clone(); let memory = Arc::new(crate::ip22_bus::memory_map(cfg.machine.profile.ip28())); - macro_rules! build_cpu { ($cache:ty) => {{ + macro_rules! build_cpu { ($cache:ty $(, $prid:expr $(, $config0:expr)?)?) => {{ let cfg = MipsCpuConfig::for_model::<$cache>(memory.clone()); let tlb = MipsTlb::new(cfg.tlb_entries); let mut executor: MipsExecutor = MipsExecutor::new(sysad.clone(), tlb, &cfg); // The MC decodes 32 physical address bits; see `MipsExecutor::phys_mask`. executor.phys_mask = u32::MAX as u64; + // A part that shares another's model differs in PRId and, where its + // Config reset word differs, in that word's low 25 bits. + $( + executor.core.reset_prid = $prid; + executor.core.cp0_prid = $prid; + $( + executor.core.cp0_config = + (executor.core.cp0_config & !CONFIG0_RESET_MASK) | $config0; + )? + )? // Load default symbol maps if they exist { @@ -800,6 +818,12 @@ impl Machine { // tag and data arrays that exist only to answer CACHE ops and the // PROM's diagnostics. See mips_cache_shadow.rs. crate::config::CpuModel::R10000 => build_cpu!(R10000ShadowCache), + crate::config::CpuModel::R12000 => build_cpu!(R12000ShadowCache), + crate::config::CpuModel::R14000 => build_cpu!(R12000ShadowCache, R14000_PRID), + crate::config::CpuModel::R16000 => build_cpu!(R16000ShadowCache), + crate::config::CpuModel::R16000Mhz700 => { + build_cpu!(R16000ShadowCache, R16000_MHZ700_PRID, R16000_MHZ700_CONFIG0) + } }; // The developer status bar shows the CPU's Count rate. From 289c97acf643f86b69b5b5e98b368929e1947b58 Mon Sep 17 00:00:00 2001 From: James Braid Date: Fri, 30 Oct 2026 08:03:24 +0000 Subject: [PATCH 12/31] pci: let functions choose header write policies PCI functions need different writable and write-one-to-clear header fields. Describe those fields in one construction policy while keeping the current header defaults for existing functions. Forward narrow Bridge configuration writes with their byte enables so write-one-to-clear bits are handled correctly when a function supplies them. Existing Octane functions do not use that status policy, so their behavior stays unchanged. --- CHANGELOG.md | 7 ++ src/dev/bridge.rs | 2 +- src/pci.rs | 269 +++++++++++++++++++++++++++++++++++++++++++--- 3 files changed, 265 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8eb32cf1f..e8c55951b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -262,6 +262,13 @@ hashes are given where a change is easiest to understand by reading the commit. - Memory past 4 GB is as fast as the rest: such a machine gets a 16 GB direct window, and the JIT compiles code anywhere in it. - A store-conditional on memory outside the window is now atomic with two CPUs. +### PCI configuration + +- A PCI function can describe its own writable header fields and + write-one-to-clear status bits. Byte and halfword configuration writes + through a Bridge carry their byte enables, so such a function keeps the + status events a narrow write did not select. No Octane function has + write-one-to-clear status bits, so the Octane behaves as before. ### VNC, and screen and pointer commands diff --git a/src/dev/bridge.rs b/src/dev/bridge.rs index 41fcf4fa9..96b267e06 100644 --- a/src/dev/bridge.rs +++ b/src/dev/bridge.rs @@ -1057,7 +1057,7 @@ impl Bridge { let i = (reg >> 2) as usize; let shift = 8 * (reg & 3); let mask = (u64::MAX >> (64 - 8 * size)) << shift; - cfg.write(i, ((cfg.read(i) as u64 & !mask) | ((val << shift) & mask)) as u32); + cfg.write_masked(i, (val << shift) as u32, mask as u32); true } diff --git a/src/pci.rs b/src/pci.rs index f81627099..679e23fe8 100644 --- a/src/pci.rs +++ b/src/pci.rs @@ -46,6 +46,50 @@ pub enum Space { Mem, } +/// Initial value and writable bits for a function-specific configuration dword. +#[derive(Clone, Copy, Debug)] +pub struct PciRegisterPolicy { + /// Dword index. Header, BAR, subsystem identity, and interrupt dwords are managed separately. + pub reg_dword: usize, + /// Value restored by [`PciConfig::reset`]. + pub initial: u32, + /// Bits that configuration writes may change. + pub writable_mask: u32, +} + +/// Function-specific write behavior for a type 0 PCI configuration header. +/// +/// Register policies are copied into the header during construction; the +/// borrowed slice is not retained. +#[derive(Clone, Copy, Debug)] +pub struct PciConfigPolicy<'a> { + /// Fixed Status bits that read as one. + pub status: u16, + /// Device-latched Status bits that writes acknowledge with one. + pub status_w1c_mask: u16, + /// Writable Command bits. + pub command_mask: u16, + /// Writable bits in the latency timer byte. + pub latency_mask: u8, + /// Writable bits in the cache-line-size byte. + pub cache_line_mask: u8, + /// Initial values and write masks for function-specific dwords. + pub registers: &'a [PciRegisterPolicy], +} + +impl<'a> Default for PciConfigPolicy<'a> { + fn default() -> Self { + Self { + status: 0x0200, + status_w1c_mask: 0, + command_mask: 0xffff, + latency_mask: 0xff, + cache_line_mask: 0xff, + registers: &[], + } + } +} + /// One function's type 0 configuration header. /// /// Enough of it to be discovered and configured: identity, class, command @@ -61,6 +105,12 @@ pub struct PciConfig { /// Dwords `16..caps_end` hold a capability list, which firmware cannot /// overwrite. 0 when the function has none. caps_end: usize, + status: u16, + status_w1c_mask: u16, + command_mask: u16, + latency_mask: u8, + cache_line_mask: u8, + writable_masks: [u32; 64], } impl PciConfig { @@ -68,6 +118,27 @@ impl PciConfig { /// three bytes of register 0x08. `pin` is 0 for none, or [`PCI_INTA`] /// and on; the line register is left for firmware to assign. pub fn new(vendor: u16, device: u16, class: u32, revision: u8, bars: [Bar; 6], pin: u8) -> Self { + Self::new_with_policy( + vendor, + device, + class, + revision, + bars, + pin, + PciConfigPolicy::default(), + ) + } + + /// Construct a type 0 header with function-specific register behavior. + pub fn new_with_policy( + vendor: u16, + device: u16, + class: u32, + revision: u8, + bars: [Bar; 6], + pin: u8, + policy: PciConfigPolicy<'_>, + ) -> Self { let mut r = [0u32; 64]; r[0] = ((device as u32) << 16) | vendor as u32; r[2] = (class << 8) | revision as u32; @@ -77,7 +148,26 @@ impl PciConfig { r[4 + i] = 1; } } - Self { regs: Mutex::new(r), bars, power_on: r, caps_end: 0 } + let mut writable_masks = [u32::MAX; 64]; + for register in policy.registers { + assert!( + matches!(register.reg_dword, 10 | 12..=14 | 16..=63), + "pci: register policy requires a function-specific dword in 0..64"); + r[register.reg_dword] = register.initial; + writable_masks[register.reg_dword] = register.writable_mask; + } + Self { + regs: Mutex::new(r), + bars, + power_on: r, + status: policy.status, + status_w1c_mask: policy.status_w1c_mask, + command_mask: policy.command_mask, + latency_mask: policy.latency_mask, + cache_line_mask: policy.cache_line_mask, + writable_masks, + caps_end: 0, + } } /// Give the function a capability list: `caps` are the raw dwords of the @@ -113,24 +203,40 @@ impl PciConfig { pub fn read(&self, reg: usize) -> u32 { let r = self.regs.lock().unwrap(); match reg { - // Status: a medium-speed device with no errors to report. // A function with a capability list also sets status bit 4. - 1 => (r[1] & 0xffff) | 0x0200_0000 | if r[13] != 0 { 0x0010_0000 } else { 0 }, + 1 => (r[1] & (0xffff | ((self.status_w1c_mask as u32) << 16))) + | ((self.status as u32) << 16) + | if r[13] != 0 { 0x0010_0000 } else { 0 }, _ => r[reg], } } pub fn write(&self, reg: usize, val: u32) { + self.write_masked(reg, val, u32::MAX); + } + + /// Write selected bits of dword `reg`. Both `val` and `mask` use the + /// register's bit positions. Unselected Status events are not acknowledged. + pub fn write_masked(&self, reg: usize, val: u32, mask: u32) { + if mask == 0 { + return; + } let mut r = self.regs.lock().unwrap(); match reg { - // Identity and class are read-only. + // Identity, class and header type are read-only. 0 | 2 => {} - // Cache line size and latency timer are stored; the header type - // and BIST bytes above them are not. - 3 => r[3] = (r[3] & 0xffff_0000) | (val & 0xffff), - // Command; status bits are write-one-to-clear and none are set. - 1 => r[1] = val & 0xffff, + 3 => { + let mask = mask & (((self.latency_mask as u32) << 8) | self.cache_line_mask as u32); + r[3] = (r[3] & !mask) | (val & mask); + } + // Command writes replace selected bits; Status events clear on ones. + 1 => { + let clear = val & mask & ((self.status_w1c_mask as u32) << 16); + r[1] = (r[1] & !(mask & 0xffff) & !clear) + | (val & mask & self.command_mask as u32); + } 4..=9 => { + let val = (r[reg] & !mask) | (val & mask); let bar = self.bars[reg - 4]; r[reg] = match bar { Bar::Unused => 0, @@ -144,11 +250,27 @@ impl PciConfig { // The capability list is read-only. 16.. if reg < self.caps_end => {} // Interrupt pin is read-only; the line is firmware's scratch. - 15 => r[15] = (r[15] & 0xff00) | (val & !0xff00), - _ => r[reg] = val, + 15 => { + let mask = mask & !0xff00; + r[15] = (r[15] & !mask) | (val & mask); + } + _ => { + let mask = mask & self.writable_masks[reg]; + r[reg] = (r[reg] & !mask) | (val & mask); + } } } + /// Latch selected device Status events without replacing earlier events. + pub fn set_status_bits(&self, bits: u16) { + self.regs.lock().unwrap()[1] |= ((bits & self.status_w1c_mask) as u32) << 16; + } + + /// Set the board-assigned Subsystem ID, which survives function reset. + pub fn set_subsystem_identity(&self, value: u32) { + self.regs.lock().unwrap()[11] = value; + } + /// The base address firmware programmed into BAR `i`, type bits masked. pub fn bar_base(&self, i: usize) -> u32 { let v = self.regs.lock().unwrap()[4 + i]; @@ -436,6 +558,14 @@ mod tests { [Bar::Io(0x100), Bar::Mem(0x1000), Bar::Unused, Bar::Unused, Bar::Unused, Bar::Unused], PCI_INTA) } + fn scsi_like_with_policy(policy: PciConfigPolicy<'_>) -> PciConfig { + PciConfig::new_with_policy( + 0x9004, 0x8078, 0x0100_00, 1, + [Bar::Io(0x100), Bar::Mem(0x1000), Bar::Unused, Bar::Unused, Bar::Unused, Bar::Unused], PCI_INTA, + policy, + ) + } + /// Firmware learns how much space a device wants by writing all ones to /// a BAR and reading back which address bits stuck. #[test] @@ -502,7 +632,7 @@ mod tests { #[test] fn reset_keeps_the_subsystem_identity() { let c = scsi_like(); - c.write(11, 0xc304_10a9); + c.set_subsystem_identity(0xc304_10a9); c.write(4, 0x0000_1000); c.reset(); assert_eq!(c.read(11), 0xc304_10a9); @@ -529,6 +659,121 @@ mod tests { assert_eq!(c.interrupt_pin(), PCI_INTA); } + #[test] + fn header_policies_mask_writes_and_restore_reset_values() { + for (status, command, latency, cache) in [(0x0200, 0xffff, 0, 0), (0x02b0, 0x0156, 0xf0, 0x0f)] { + let c = scsi_like_with_policy(PciConfigPolicy { + status, + status_w1c_mask: 0, + command_mask: command, + latency_mask: latency, + cache_line_mask: cache, + registers: &[PciRegisterPolicy { + reg_dword: 18, + initial: 0xaabb_0020, + writable_mask: 0xff00, + }], + }); + c.write(1, u32::MAX); + c.write(3, u32::MAX); + c.write(18, u32::MAX); + assert_eq!(c.read(1), (u32::from(status) << 16) | u32::from(command)); + assert_eq!(c.read(3), (u32::from(latency) << 8) | u32::from(cache)); + assert_eq!(c.read(18), 0xaabb_ff20); + c.reset(); + assert_eq!(c.read(1), u32::from(status) << 16); + assert_eq!(c.read(3), 0); + assert_eq!(c.read(18), 0xaabb_0020); + } + } + + #[test] + fn masked_writes_preserve_unselected_fields_and_status_events() { + let c = scsi_like_with_policy(PciConfigPolicy { + status: 0x0200, + status_w1c_mask: 0x8040, + command_mask: 0x0157, + latency_mask: 0xff, + cache_line_mask: 0xff, + registers: &[PciRegisterPolicy { + reg_dword: 18, + initial: 0xaabb_0000, + writable_mask: 0xffff, + }], + }); + c.set_status_bits(0x8040); + c.write(1, 0x0157); + c.write(3, 0x1234); + c.write(5, 0x1234_5000); + c.write(4, 0x1201); + for (reg, value, mask, expected) in [ + (1, 0xffff_0002, 0xff, 0x8240_0102), + (1, 0xffff_0000, 0x8000_0000, 0x0240_0102), + (1, 0x0040_0000, 0x00ff_0000, 0x0200_0102), + (1, u32::MAX, 0, 0x0200_0102), + (3, 0xffff_00ab, 0xff, 0x12ab), + (5, 0xabcd_ffff, 0xffff_0000, 0xabcd_5000), + (4, 0x3400, 0xff00, 0x3401), + (15, u32::MAX, 0xff00, 0x0100), + (18, u32::MAX, 0xff, 0xaabb_00ff), + ] { + c.write_masked(reg, value, mask); + assert_eq!(c.read(reg), expected, "register {reg}"); + } + c.set_status_bits(u16::MAX); + c.reset(); + assert_eq!(c.read(1), 0x0200_0000); + let default = scsi_like(); + default.set_status_bits(u16::MAX); + assert_eq!(default.read(1), 0x0200_0000); + } + + #[test] + fn snapshots_restore_registers_with_destination_write_policies() { + use crate::traits::Saveable; + let src = scsi_like_with_policy(PciConfigPolicy { + status: 0x02b0, + status_w1c_mask: 0xc000, + command_mask: 0x0157, + latency_mask: 0xff, + cache_line_mask: 0xff, + registers: &[PciRegisterPolicy { + reg_dword: 18, + initial: 0, + writable_mask: 0x100, + }], + }); + src.set_status_bits(0x8000); + src.write(3, 0x1234); + src.write(18, 0x100); + let saved = src.save_state(); + for (events, command, cache) in [(0, 7, 0x0f), (0xc000, 0x0157, 0xff)] { + let dst = scsi_like_with_policy(PciConfigPolicy { + status: 0x0200, + status_w1c_mask: events, + command_mask: command, + latency_mask: 0, + cache_line_mask: cache, + registers: &[PciRegisterPolicy { + reg_dword: 18, + initial: 0x20, + writable_mask: 0x200, + }], + }); + dst.load_state(&saved).unwrap(); + assert_eq!(dst.save_state(), saved); + assert_eq!(dst.read(1), 0x0200_0000 | ((0x8000 & u32::from(events)) << 16)); + dst.write_masked(1, u32::MAX, u32::MAX); + dst.write(3, u32::MAX); + dst.write(18, u32::MAX); + assert_eq!(dst.read(1), 0x0200_0000 | u32::from(command)); + assert_eq!(dst.read(3), (0x1234 & !u32::from(cache)) | u32::from(cache)); + assert_eq!(dst.read(18), 0x300); + dst.reset(); + assert_eq!(dst.read(18), 0x20); + } + } + struct Nothing; impl BusDevice for Nothing {} From f7be5e10ca481d1b04a3acc75f2ad57f287bfc37 Mon Sep 17 00:00:00 2001 From: James Braid Date: Fri, 30 Oct 2026 08:13:36 +0000 Subject: [PATCH 13/31] isp1040: add the dual-channel ISP12160 part Fuel uses two independent SCSI buses behind one ISP12160 function. Reuse the ISP1040 command engine and let the part trait own the firmware, register and channel differences. A bus reset reaches every attached target as unit attention rather than a media change. Keep the reset attention on the shared target interface so a controller can notify each device through the same bus. The ISP12160 is its only caller: the WD33C93A, AIC-7880 and ISP1040 models do not call it, so their targets behave as before. A bus reset restores each disk's write cache bit per channel: a reset of one channel leaves the other's disks alone, and a chip or soft reset restores both. The engine changes reach the Octane's ISP1040: mailbox commands now run with dispatch held, and the target interface gains the bus reset. The Octane's IRIX 6.5.30 SCSI disk (Octane2, R10000, 256 MiB, V12, jitv2, release build, headless, NAT) boots to the login prompt and runs commands on this branch's tip and on main c6dea8c4: login after 126 s on the branch and 109 s on main in one run each (host load makes the times indicative only), the same `hinv` listing of both integral controllers and the disk, and the same command output. --- HACKING.md | 6 + src/dev/isp1040.rs | 395 ++++++++++++++++++++++++------- src/dev/isp1040/tests.rs | 488 ++++++++++++++++++++++++++++++++++++++- src/dev/isp12160.rs | 338 +++++++++++++++++++++++++++ src/dev/mod.rs | 1 + src/scsi.rs | 52 ++++- src/scsi/bus.rs | 11 +- 7 files changed, 1197 insertions(+), 94 deletions(-) create mode 100644 src/dev/isp12160.rs diff --git a/HACKING.md b/HACKING.md index 54373e395..b271ee97d 100644 --- a/HACKING.md +++ b/HACKING.md @@ -98,6 +98,12 @@ way: `scsi::attach_from_config` walks the `[scsi.N]` tables for every machine, where the key is a slot (controller × 16 + id, see `config::scsi_slot`), and the machine supplies only a route from controller number to bus. +`ScsiTarget::bus_reset` defaults to no action. Disk targets record reset as +unit attention. Only the ISP12160 calls it; the other controllers keep +their earlier behaviour of not reporting resets to targets. +The ISP12160 releases its controller state lock before walking either bus. +Target callbacks run under that bus lock and must not reacquire it. + ## 2. Data path and endianness **Word-transparent architecture.** Host `u32`/`u64` values are bit diff --git a/src/dev/isp1040.rs b/src/dev/isp1040.rs index 1dc60951f..ae655f888 100644 --- a/src/dev/isp1040.rs +++ b/src/dev/isp1040.rs @@ -50,8 +50,9 @@ //! again when the hold ends: an approximation, a chip that had fetched it //! would plausibly move on. A parked entry the guest rewrites meanwhile is //! answered incomplete. A snapshot carries each bus's remaining hold and -//! the parked positions. The ISP12160 would widen `BUSES` and decode the -//! bus from mailbox 2 and its request entries; the per-bus state is there. +//! the parked positions. Only the ISP1040's bus reset starts a hold-off: +//! the ISP12160 (two buses, per-bus state is there) keeps its own bus-reset +//! handling, and its hold-off is not modelled. //! * Only logical unit 0 of a present target exists. A command for another //! unit is passed to the target with that unit number, which answers as //! SCSI says (INQUIRY: no device at this unit; anything else: check @@ -81,7 +82,7 @@ use parking_lot::{Condvar, Mutex, MutexGuard}; use crate::devlog::LogModule; use crate::irq::IrqLine; -use crate::pci::{Bar, PciBusDevice, PciConfig, PciDma, PCI_INTA}; +use crate::pci::{Bar, PciBusDevice, PciConfig, PciConfigPolicy, PciDma, PciRegisterPolicy, PCI_INTA}; use crate::scsi::{scsi_cmd, ScsiBus, ScsiDataLength, ScsiIdentities, ScsiRequest, CDROM_IDENTITY, DISK_IDENTITY}; use crate::snapshot::{get_field, hex_u16, hex_u32, load_u16_slice, toml_u16, toml_u32, toml_u8, u16_slice_to_toml, u8_slice_to_toml}; @@ -115,7 +116,7 @@ pub mod reg { pub const WINDOW_SIZE: u32 = 0x100; } -const MAILBOX_COUNT: usize = 8; +pub(crate) const MAILBOX_COUNT: usize = 8; const MAILBOX_END: u32 = reg::MAILBOX_BASE + MAILBOX_COUNT as u32 * 2; const BANK_END: u32 = 0xc0; const BANK_WORDS: usize = ((BANK_END - reg::BANK_BASE) / 2) as usize; @@ -163,6 +164,8 @@ pub mod mbox { pub const INIT_REQ_QUEUE: u16 = 0x0010; pub const INIT_RSP_QUEUE: u16 = 0x0011; pub const ABORT_COMMAND: u16 = 0x0015; + pub const ABORT_DEVICE: u16 = 0x0016; + pub const ABORT_TARGET: u16 = 0x0017; pub const BUS_RESET: u16 = 0x0018; pub const GET_FIRMWARE_STATUS: u16 = 0x001f; pub const GET_TARGET_PARAMETERS: u16 = 0x0028; @@ -187,13 +190,14 @@ pub mod mbox { pub const INVALID: u16 = 0x4001; pub const HOST_INTERFACE_ERROR: u16 = 0x4002; pub const TEST_FAILED: u16 = 0x4003; + pub const COMMAND_ERROR: u16 = 0x4005; pub const PARAM_ERROR: u16 = 0x4006; } const RAM_WORDS: usize = 0x10000; /// The command budget the adapter announces in its firmware status. pub const MAX_REQUESTS: u16 = 512; -const FW_VERSION: [u16; 3] = [1, 0, 0]; +pub(crate) const FW_VERSION: [u16; 3] = [1, 0, 0]; pub const HOST_CMD_SHIFT: u32 = 12; pub const RISC_FLAG_HOST_INT: u16 = 0x0080; @@ -229,6 +233,7 @@ pub const SCSI_STATUS_SENSE_VALID: u16 = 0x0200; pub const IOCB_COMPLETE: u16 = 0x0000; pub const IOCB_INCOMPLETE: u16 = 0x0001; pub const IOCB_RESET: u16 = 0x0004; +pub const IOCB_ABORTED: u16 = 0x0005; pub const IOCB_DATA_UNDERRUN: u16 = 0x0015; pub const STATE_GOT_BUS: u16 = 0x0100; @@ -256,13 +261,14 @@ const DEFAULT_SELECTION_TIMEOUT: u16 = 250; /// source for a device, so it runs on the adapter's clock, host wall time /// less the time the machine was stopped, like the selection timeout. const BUS_RESET_DELAY_UNIT: Duration = Duration::from_secs(1); -/// The SCSI buses this part drives. The hold-off, the parked commands and the -/// selection are all kept per bus, so a two-bus part widens this and decodes -/// the bus from its own request entries and from mailbox 2. -const BUSES: usize = 1; +/// The most SCSI buses any part drives (the ISP12160's two; the ISP1040 has +/// one, `IspPart::channels`). The hold-off, the parked commands and the +/// selection are all kept per bus. Only the ISP1040's bus reset starts a +/// hold-off: the ISP12160 keeps its own bus-reset handling. +const BUSES: usize = 2; const CHECK_CONDITION: u8 = 0x02; -const MAX_TARGET: u8 = 15; +pub(crate) const MAX_TARGET: u8 = 15; const MAX_LUN: u8 = 7; /// A completion held for response-ring room. @@ -374,6 +380,49 @@ enum Prepared { Run(Job), } +/// Part-specific register and firmware behavior; the ring engine owns the state. +/// Defaults preserve the ISP1040 contract, including wrapping firmware RAM. +pub(crate) trait IspPart: Send + Sync { + fn name(&self) -> &'static str; + fn channels(&self) -> usize { 1 } + fn pci_config(&self, bars: [Bar; 6]) -> PciConfig { + PciConfig::new(PCI_VENDOR_QLOGIC, PCI_DEVICE_ISP1040, PCI_CLASS_SCSI, PCI_REVISION_ISP1040, bars, PCI_INTA) + } + fn firmware_version(&self) -> [u16; 3] { FW_VERSION } + fn active_bank(&self, cfg1: u16) -> Bank { + if cfg1 & CFG1_BANK_SELECT != 0 { Bank::Sxp } else { Bank::Risc } + } + fn checks_ram_bounds(&self) -> bool { false } + fn has_abort_commands(&self) -> bool { false } + fn target_on(&self, _channel: u8, target: u8) -> u8 { target } + fn decode_target(&self, target: u8) -> (usize, u8) { (0, target) } + fn id_high(&self) -> u16 { PCI_DEVICE_ISP1040 } + /// The ISP1040 register revision runs one ahead of its PCI revision. + fn bus_cfg0(&self) -> u16 { PCI_REVISION_ISP1040 as u16 + 1 } + fn cfg1_write_mask(&self) -> u16 { CFG1_WRITE_MASK } + fn unmodelled_dma_register(&self, _off: u32) -> bool { false } + fn completion_phases(&self, _moved: u64, _total: u64) -> u16 { STATE_ALL_PHASES & !STATE_GOT_SENSE } + fn mailbox_ready(&self, _semaphore: u16) -> bool { true } + /// Extra target-parameter results, beyond the common asynchronous/narrow reply. + fn clear_target_parameters(&self, _mailbox: &mut [u16; MAILBOX_COUNT]) {} + fn set_reset_delay(&self, st: &mut State) { st.reset_delay = st.mailbox[1]; } + /// Queue target notifications for the worker; the CPU never locks a bus. + fn reset_targets(&self, _st: &mut State) -> bool { false } + /// Handle extra mailbox commands, leaving result signalling to the engine. + fn execute_mailbox(&self, _core: &Core, _st: &mut MutexGuard<'_, State>) -> bool { false } + fn save_extra(&self, _st: &State, _t: &mut toml::map::Map) {} + fn load_extra(&self, _st: &mut State, _v: &toml::Value) {} +} + +struct Isp1040Part; + +impl IspPart for Isp1040Part { + fn name(&self) -> &'static str { "ISP1040" } +} + +/// The register window selected by BUS_CFG1, independent of its encoding. +pub(crate) enum Bank { Risc, Sxp, Sxp1, Dma } + /// A command waiting out its bus's post-reset hold-off. Only the ring /// position is kept: the entry, its data segments included, is read again /// from the guest's ring when the hold ends. @@ -392,7 +441,15 @@ struct Parked { started: bool, } -struct State { +pub(crate) struct State { + pub(crate) reset_targets: bool, + pub(crate) pending_target_resets: VecDeque<(u8, Option)>, + pub(crate) active: Option<(u8, u8, u32)>, + cancelled: bool, + pub(crate) outstanding: Vec<(u8, u8, u32)>, + pub(crate) sxp1_bank: [u16; BANK_WORDS], + pub(crate) dma_bank: [u16; BANK_WORDS], + pub(crate) reset_delay1: u16, ctl: Ctl, bus_cfg1: u16, bus_ctrl: u16, @@ -402,11 +459,11 @@ struct State { risc_flags: u16, dma_cmd_cfg: u16, dma_data_cfg: u16, - mailbox: [u16; MAILBOX_COUNT], + pub(crate) mailbox: [u16; MAILBOX_COUNT], risc_bank: [u16; BANK_WORDS], sxp_bank: [u16; BANK_WORDS], risc_ram: Vec, - reset_delay: u16, + pub(crate) reset_delay: u16, /// Mailbox command 0x0031's value, in [`SELECTION_TIMEOUT_UNIT`]s. selection_timeout: u16, rqst_base: u64, @@ -434,6 +491,14 @@ struct State { impl State { fn new() -> Self { let mut s = State { + reset_targets: false, + pending_target_resets: VecDeque::new(), + active: None, + cancelled: false, + outstanding: Vec::new(), + sxp1_bank: [0; BANK_WORDS], + dma_bank: [0; BANK_WORDS], + reset_delay1: 0, ctl: Ctl::default(), bus_cfg1: 0, bus_ctrl: 0, @@ -485,6 +550,12 @@ impl State { self.mailbox = [0, SIG1, SIG2, SIG3, SIG4, 0, 0, 0]; self.risc_bank = [0; BANK_WORDS]; self.sxp_bank = [0; BANK_WORDS]; + self.sxp1_bank = [0; BANK_WORDS]; + self.dma_bank = [0; BANK_WORDS]; + self.reset_delay1 = 0; + self.active = None; + self.cancelled = false; + self.outstanding.clear(); // The driver re-downloads firmware after every reset. self.risc_ram.iter_mut().for_each(|w| *w = 0); self.reset_delay = 0; @@ -499,6 +570,8 @@ impl State { self.resp_seen = 0; self.resp_cons = 0; self.pending.clear(); + self.reset_targets = false; + self.pending_target_resets.clear(); self.quiet_until = [None; BUSES]; self.parked.clear(); } @@ -524,9 +597,15 @@ impl State { self.resp_len != 0 && (u32::from(self.resp_prod) + 1) % u32::from(self.resp_len) != u32::from(self.resp_cons) } - fn active_bank(&mut self) -> &mut [u16; BANK_WORDS] { - if self.bus_cfg1 & CFG1_BANK_SELECT != 0 { &mut self.sxp_bank } else { &mut self.risc_bank } + fn active_bank(&mut self, bank: Bank) -> &mut [u16; BANK_WORDS] { + match bank { + Bank::Risc => &mut self.risc_bank, + Bank::Sxp => &mut self.sxp_bank, + Bank::Sxp1 => &mut self.sxp1_bank, + Bank::Dma => &mut self.dma_bank, + } } + } /// The adapter. Its state lives in a [`Core`] shared with the thread that @@ -543,8 +622,8 @@ impl std::ops::Deref for Isp1040 { /// Lock order: `line_order`, then `st`. Nothing else is taken with `st` /// held except the leaf locks (`notes`, `dma_misses`, `odd_accesses`): DMA, -/// the targets and `fence` are reached with it released. `fence` is held -/// across one guest-memory store at a time, and `targets` across one +/// either bus and `fence` are reached with it released. `fence` is held +/// across one guest-memory store at a time, and either bus across one /// target command; neither is held with the other. pub struct Core { st: Mutex, @@ -560,13 +639,17 @@ pub struct Core { wake: Condvar, /// Serialises driving the interrupt line; see [`Core::sync_irq`]. line_order: Mutex<()>, - /// The targets on the bus, by SCSI id. - bus: ScsiBus, - /// A bus reset the targets have not yet seen. `st` and `bus` are never - /// held together, so the reset (taken under `st`) is passed on by the - /// next command to take `bus`, before it runs: no command can see the - /// targets as they were. See [`Core::scsi`]. - targets_reset: AtomicBool, + /// Generation for commands cancelled by aborts without resetting the chip. + cancel: AtomicU64, + /// Immutable part policy, shared by register, worker and snapshot paths. + part: &'static dyn IspPart, + /// The controller's one bus, or both channels of the ISP12160. + buses: [ScsiBus; 2], + /// A bus reset the targets on that bus have not yet seen. `st` and the + /// buses are never held together, so the reset (taken under `st`) is + /// passed on by the next command to take the bus, before it runs: no + /// command can see the targets as they were. See [`Core::scsi`]. + targets_reset: [AtomicBool; 2], dma: OnceLock>, irq: OnceLock, /// Which of the Octane's two controllers this is (0 internal, 1 @@ -603,7 +686,11 @@ impl Isp1040 { /// The controller at `index` (0 internal, 1 external) on `bus`. pub fn new_at(index: u8, bus: ScsiBus) -> Self { - Self { core: Arc::new(Core::new_at(index, bus)), worker: Mutex::new(None) } + Self { core: Arc::new(Core::new_at(index, &Isp1040Part, [bus, Self::new_bus()])), worker: Mutex::new(None) } + } + + pub(crate) fn new_12160(bus0: ScsiBus, bus1: ScsiBus) -> Self { + Self { core: Arc::new(Core::new_at(0, &crate::dev::isp12160::Isp12160Part, [bus0, bus1])), worker: Mutex::new(None) } } /// Start the ring thread (idempotent) and let it take whatever was @@ -617,7 +704,7 @@ impl Isp1040 { let c = self.core.clone(); *worker = Some( std::thread::Builder::new() - .name(format!("ISP1040-{}", self.index)) + .name(format!("{}-{}", self.core.model_name(), self.index)) .spawn(move || c.run_ring()) .expect("spawn isp1040 thread"), ); @@ -641,13 +728,17 @@ impl Isp1040 { /// Name the adapter's long-lived locks in the lock registry, which /// holds only a weak reference to them. pub fn register_locks(&self) { - for (what, probe) in [ + let mut locks: Vec<(&str, fn(&Core) -> bool)> = vec![ ("st", (|c: &Core| c.st.is_locked()) as fn(&Core) -> bool), ("fence", |c| c.fence.is_locked()), - ("targets", |c| c.bus.is_locked()), - ] { + ("targets", |c| c.buses[0].is_locked()), + ]; + if self.core.part.channels() > 1 { + locks.push(("targets1", |c| c.buses[1].is_locked())); + } + for (what, probe) in locks { let core = Arc::downgrade(&self.core); - crate::locks::register_lock(format!("isp1040-{}::{what}", self.index), move || { + crate::locks::register_lock(format!("{}-{}::{what}", self.core.model_name().to_ascii_lowercase(), self.index), move || { core.upgrade().is_some_and(|c| probe(&c)) }); } @@ -662,27 +753,27 @@ impl Drop for Isp1040 { } impl Core { - fn new_at(index: u8, bus: ScsiBus) -> Self { + fn new_at(index: u8, part: &'static dyn IspPart, buses: [ScsiBus; 2]) -> Self { let st = State::new(); let gen = st.ctl.gen; + let bars = [Bar::Io(reg::WINDOW_SIZE), Bar::Mem(reg::WINDOW_SIZE), Bar::Unused, Bar::Unused, Bar::Unused, Bar::Unused]; + let pci = part.pci_config(bars); Self { + part, + cancel: AtomicU64::new(0), st: Mutex::new(st), gen: AtomicU64::new(gen), fence: Mutex::new(()), wake: Condvar::new(), line_order: Mutex::new(()), - bus, - targets_reset: AtomicBool::new(false), + buses, + targets_reset: [AtomicBool::new(false), AtomicBool::new(false)], dma: OnceLock::new(), irq: OnceLock::new(), index, // BAR 0 is the I/O window and BAR 1 its memory-space alias, one // register file. - pci: PciConfig::new( - PCI_VENDOR_QLOGIC, PCI_DEVICE_ISP1040, PCI_CLASS_SCSI, PCI_REVISION_ISP1040, - [Bar::Io(reg::WINDOW_SIZE), Bar::Mem(reg::WINDOW_SIZE), Bar::Unused, Bar::Unused, Bar::Unused, Bar::Unused], - PCI_INTA, - ), + pci, notes: Mutex::new(Vec::new()), dma_misses: Mutex::new(std::collections::BTreeSet::new()), odd_accesses: Mutex::new(0), @@ -692,6 +783,8 @@ impl Core { } } + fn model_name(&self) -> &'static str { self.part.name() } + pub fn index(&self) -> u8 { self.index } /// A machine reset: the chip as at power-on (no firmware, no queues), @@ -702,6 +795,7 @@ impl Core { { let mut st = self.st.lock(); st.reset(); + if self.part.reset_targets(&mut st) { self.kick(&mut st); } self.publish_gen(&st); self.wake.notify_all(); } @@ -929,7 +1023,7 @@ impl Core { } /// Tell the ring thread the ring may have moved. - fn kick(&self, st: &mut State) { + pub(crate) fn kick(&self, st: &mut State) { st.ctl.work = true; self.wake.notify_all(); } @@ -957,11 +1051,16 @@ impl Core { /// The SCSI bus: attach targets to it and ask what is on it. pub fn bus(&self) -> &ScsiBus { - &self.bus + &self.buses[0] + } + + pub(crate) fn bus_for_channel(&self, channel: u8) -> Option<&ScsiBus> { + ((channel as usize) < self.part.channels()).then(|| &self.buses[channel as usize]) } - fn has_target(&self, id: u8) -> bool { - self.bus.has_target(id) + fn bus_for_target(&self, target: u8) -> (&ScsiBus, u8) { + let (channel, id) = self.part.decode_target(target); + (&self.buses[channel], id) } // ---- DMA --------------------------------------------------------- @@ -1045,12 +1144,18 @@ impl Core { /// A RISC RAM load or dump of up to 128 KiB, with the state unlocked /// for the DMA. The whole transfer is read before a word of it is kept, /// so a refused DMA leaves the array as it was. Words travel - /// little-endian. This part has no edge to its array: a transfer past - /// the end folds to the start. - fn ram_transfer(&self, st: &mut MutexGuard<'_, State>, load: bool) -> u16 { + /// little-endian. The ISP1040 wraps past the end; the ISP12160 rejects + /// transfers that would cross it. + pub(crate) fn ram_transfer(&self, st: &mut MutexGuard<'_, State>, load: bool) -> u16 { let base = st.mailbox[1]; let count = st.mailbox[4] as usize; - let host = Self::mbox_host_addr(&st.mailbox); + if self.part.checks_ram_bounds() && base as usize + count > RAM_WORDS { + return mbox::PARAM_ERROR; + } + let mut host = Self::mbox_host_addr(&st.mailbox); + if st.mailbox[0] == mbox::LOAD_RAM_A64 { + host |= ((st.mailbox[7] as u64) << 32) | ((st.mailbox[6] as u64) << 48); + } if count == 0 { return mbox::COMPLETE; } @@ -1077,6 +1182,20 @@ impl Core { /// Synchronous: the protocol has no way to leave a command pending. fn mailbox_command(&self, st: &mut MutexGuard<'_, State>) { let cmd = st.mailbox[0]; + if self.part.has_abort_commands() && matches!(cmd, mbox::ABORT_COMMAND | mbox::ABORT_DEVICE | mbox::ABORT_TARGET | mbox::BUS_RESET) { + self.with_dispatch_held(st, |st| self.execute_mailbox_command(st)); + } else { + self.execute_mailbox_command(st); + } + } + + fn execute_mailbox_command(&self, st: &mut MutexGuard<'_, State>) { + let cmd = st.mailbox[0]; + if self.part.execute_mailbox(self, st) { + st.semaphore = SEMAPHORE_LOCK; + st.bus_status |= STATUS_INT_REQ | STATUS_RISC_INT_REQ; + return; + } match cmd { mbox::NOP => st.mailbox[0] = mbox::COMPLETE, mbox::WRITE_RAM_WORD => { @@ -1098,7 +1217,19 @@ impl Core { // zero. mbox::VERIFY_CHECKSUM => { let start = st.mailbox[1]; + if self.part.checks_ram_bounds() && start as usize + 4 > RAM_WORDS { + st.mailbox[0] = mbox::PARAM_ERROR; + st.semaphore = SEMAPHORE_LOCK; + st.bus_status |= STATUS_INT_REQ | STATUS_RISC_INT_REQ; + return; + } let len = st.risc_ram[start.wrapping_add(3) as usize]; + if self.part.checks_ram_bounds() && start as usize + len as usize > RAM_WORDS { + st.mailbox[0] = mbox::PARAM_ERROR; + st.semaphore = SEMAPHORE_LOCK; + st.bus_status |= STATUS_INT_REQ | STATUS_RISC_INT_REQ; + return; + } let mut sum = 0u16; for i in 0..len { sum = sum.wrapping_add(st.risc_ram[start.wrapping_add(i) as usize]); @@ -1111,9 +1242,7 @@ impl Core { // Accepted, never executed. mbox::EXEC_FIRMWARE => st.mailbox[0] = mbox::COMPLETE, mbox::ABOUT_FIRMWARE => { - st.mailbox[1] = FW_VERSION[0]; - st.mailbox[2] = FW_VERSION[1]; - st.mailbox[3] = FW_VERSION[2]; + st.mailbox[1..4].copy_from_slice(&self.part.firmware_version()); st.mailbox[6] = 0; st.mailbox[0] = mbox::COMPLETE; } @@ -1128,6 +1257,7 @@ impl Core { mbox::GET_TARGET_PARAMETERS => { st.mailbox[2] = 0; st.mailbox[3] = 0; + self.part.clear_target_parameters(&mut st.mailbox); st.mailbox[0] = mbox::COMPLETE; } // Setters for state nothing reads back; answering invalid is a @@ -1149,7 +1279,7 @@ impl Core { st.mailbox[0] = mbox::COMPLETE; } mbox::SET_RESET_DELAY_PARAMS => { - st.reset_delay = st.mailbox[1]; + self.part.set_reset_delay(st); st.mailbox[0] = mbox::COMPLETE; } mbox::INIT_REQ_QUEUE | mbox::INIT_REQ_QUEUE_A64 | mbox::INIT_RSP_QUEUE | mbox::INIT_RSP_QUEUE_A64 => { @@ -1204,7 +1334,7 @@ impl Core { mbox::BUS_RESET => { let now = self.now_ns(st); let bus = usize::from(st.mailbox[2]); - if bus >= BUSES { + if bus >= self.part.channels() { // A bus this part does not have: nothing to quiet or // retire, and the datasheet's per-bus hold leaves the // real bus alone. Noted so a driver that leaves mailbox 2 @@ -1228,7 +1358,7 @@ impl Core { st.pending.push_back(Pending { handle: p.handle, completion: IOCB_RESET, ..Default::default() }); } } - self.targets_reset.store(true, Ordering::Release); + self.targets_reset[bus].store(true, Ordering::Release); let delay = BUS_RESET_DELAY_UNIT * u32::from(st.mailbox[1]); st.quiet_until[bus] = (!delay.is_zero()).then(|| now.saturating_add(delay.as_nanos() as i64)); self.kick(st); @@ -1245,6 +1375,67 @@ impl Core { st.bus_status |= STATUS_INT_REQ | STATUS_RISC_INT_REQ; } + /// Prevent the worker from taking another entry while a mailbox drops `st`. + fn with_dispatch_held(&self, st: &mut MutexGuard<'_, State>, f: impl FnOnce(&mut MutexGuard<'_, State>) -> R) -> R { + struct Hold<'a, 'b> { + core: &'a Core, + st: &'a mut MutexGuard<'b, State>, + } + impl Drop for Hold<'_, '_> { + fn drop(&mut self) { + self.st.ctl.hold -= 1; + self.core.kick(self.st); + } + } + st.ctl.hold += 1; + let held = Hold { core: self, st }; + f(held.st) + } + + /// A reset of `channel`'s bus: each disk on it puts its mode parameters back before the next + /// command, see [`ScsiTarget::reset_mode_parameters`]. A channel the part lacks is ignored. + pub(crate) fn note_bus_reset(&self, channel: usize) { + if let Some(flag) = self.targets_reset.get(channel) { flag.store(true, Ordering::Release); } + } + + /// Notify targets with `st` released; a blocked target must not stall register access. + fn reset_targets(&self, st: &mut MutexGuard<'_, State>, channel: u8, target: Option) { + let Some(bus) = self.bus_for_channel(channel) else { return }; + self.with_dispatch_held(st, |st| MutexGuard::unlocked(st, || bus.for_each(|id, dev| { + if target.is_none_or(|target| target as usize == id) { + dev.bus_reset(); + } + }))); + } + + pub(crate) fn retire_outstanding(&self, st: &mut State, matches: impl Fn((u8, u8, u32)) -> bool, completion: u16) -> usize { + let mut retired = 0; + let mut keep = Vec::new(); + for item in st.outstanding.drain(..) { + if matches(item) { + st.pending.push_back(Pending { handle: item.2, completion, ..Pending::default() }); + retired += 1; + } else { + keep.push(item); + } + } + st.outstanding = keep; + if retired != 0 { self.kick(st); } + retired + } + + /// Queue one cancellation reply, then fence its remaining guest-memory stores. + pub(crate) fn cancel_active(&self, st: &mut MutexGuard<'_, State>, completion: u16) { + if st.cancelled { return; } + if let Some((_, _, handle)) = st.active { + self.cancel.fetch_add(1, Ordering::AcqRel); + st.cancelled = true; + st.pending.push_back(Pending { handle, completion, ..Pending::default() }); + self.kick(st); + MutexGuard::unlocked(st, || self.retire_stores()); + } + } + fn host_command(&self, st: &mut MutexGuard<'_, State>, val: u16) { match val >> HOST_CMD_SHIFT { 0x0 => {} @@ -1255,6 +1446,10 @@ impl Core { 0x3 => st.risc_flags &= !(RISC_FLAG_PAUSED | RISC_FLAG_RESET), // "A mailbox command is ready": the trigger to process one. 0x5 => { + if !self.part.mailbox_ready(st.semaphore) { + self.note("mailbox submitted with an unacknowledged result".into()); + return; + } st.risc_flags |= RISC_FLAG_HOST_INT; self.mailbox_command(st); } @@ -1440,17 +1635,19 @@ impl Core { /// Run a CDB against the target; `data` is what a write sends it. None /// when a reset cancelled the command while it waited for the target. - fn scsi(&self, gen: u64, target: u8, cdb: &[u8], data: Option>) -> Option<(Vec, u8)> { - let mut targets = self.bus.lock(); - if self.targets_reset.swap(false, Ordering::AcqRel) { + fn scsi(&self, gen: u64, cancel: u64, target: u8, cdb: &[u8], data: Option>) -> Option<(Vec, u8)> { + let (channel, _) = self.part.decode_target(target); + let (bus, id) = self.bus_for_target(target); + let mut targets = bus.lock(); + if self.targets_reset[channel].swap(false, Ordering::AcqRel) { // The bus reset puts each target's mode parameters back; see // [`ScsiTarget::reset_mode_parameters`]. targets.iter_mut().flatten().for_each(|t| t.reset_mode_parameters()); } - if self.gen.load(Ordering::Acquire) != gen { + if self.gen.load(Ordering::Acquire) != gen || self.cancel.load(Ordering::Acquire) != cancel { return None; } - let Some(dev) = targets.get_mut(target as usize).and_then(|t| t.as_mut()) else { + let Some(dev) = targets.get_mut(id as usize).and_then(|t| t.as_mut()) else { return Some((Vec::new(), CHECK_CONDITION)); }; let req = ScsiRequest { cdb: cdb.to_vec(), data_len: ScsiDataLength::Unlimited, data_in: data }; @@ -1481,14 +1678,15 @@ impl Core { let lun = (device & 0xff) as u8; // A bus this part does not have. - if usize::from(channel) >= BUSES { + if channel as usize >= self.part.channels() { return Prepared::Answered(Self::incomplete(handle, 0, 0)); } // GOT_BUS alone is Linux's bad-target; the timeout bit beside it is // what lets the empty slot be treated as benign. - if target > MAX_TARGET || !self.has_target(target) { + if target > MAX_TARGET || !self.buses[channel as usize].has_target(target) { return Prepared::Absent(channel as u8, Self::incomplete(handle, STATE_GOT_BUS, STATUS_FLAG_TIMEOUT)); } + let id = target; if lun > MAX_LUN { return Prepared::Answered(Self::incomplete(handle, STATE_GOT_BUS, STATUS_FLAG_TIMEOUT)); } @@ -1497,9 +1695,9 @@ impl Core { cdb[1] = (cdb[1] & 0x1f) | (lun << 5); let cdb = &cdb[..(cdb_len as usize).clamp(1, CDB_FIELD_LEN)]; let block = self - .bus + .buses[channel as usize] .lock() - .get(target as usize) + .get(id as usize) .and_then(|t| t.as_ref()) .and_then(|t| t.block_size()) .map(|b| b as usize) @@ -1507,6 +1705,7 @@ impl Core { .unwrap_or(BLOCK_SIZE); let (xfer, to_dev) = Self::cdb_transfer(cdb, block); let (segs, described) = self.read_segments(ring, addr, entry_count, seg_count, xfer as u64, a64); + let target = self.part.target_on(channel as u8, target); Prepared::Run(Job { handle, target, cdb: cdb.to_vec(), xfer, to_dev, segs, described }) } @@ -1522,7 +1721,7 @@ impl Core { /// None too when a reset cancelled the command (`gen` is the generation /// it was read under): what it had not yet stored in host memory is not /// stored, and no completion is built. - fn run_job(&self, job: Job, gen: u64) -> Option { + fn run_job(&self, job: Job, gen: u64, cancel: u64) -> Option { let Job { handle, target, cdb, xfer, to_dev, segs, described } = job; // The bar has lights for SCSI ids 0-5. if let (Some(hb), 0..=5) = (self.heartbeat.get(), target) { @@ -1540,9 +1739,10 @@ impl Core { buf.extend_from_slice(&chunk); } moved = buf.len() as u64; - status = self.scsi(gen, target, &cdb, Some(buf))?.1; + if self.cancel.load(Ordering::Acquire) != cancel { return None; } + status = self.scsi(gen, cancel, target, &cdb, Some(buf))?.1; } else { - let (data, s) = self.scsi(gen, target, &cdb, None)?; + let (data, s) = self.scsi(gen, cancel, target, &cdb, None)?; status = s; // Offer the target's bytes to the descriptors in order. let take = (data.len() as u64).min(described).min(xfer as u64) as usize; @@ -1550,7 +1750,7 @@ impl Core { for &(a, l) in &segs { if off >= take { break; } let n = (l as usize).min(take - off); - if !self.fenced(gen, || self.dma_write(a, &data[off..off + n]))? { + if !self.fenced(gen, || self.cancel.load(Ordering::Acquire) == cancel && self.dma_write(a, &data[off..off + n]))? { self.note(format!("payload DMA for handle {handle:#x} refused by the fabric; command stopped")); return None; } @@ -1561,12 +1761,13 @@ impl Core { // Auto-sense runs only after a CHECK CONDITION. let mut sense = Vec::new(); + if self.cancel.load(Ordering::Acquire) != cancel { return None; } if status == CHECK_CONDITION { - sense = self.scsi(gen, target, &[scsi_cmd::REQUEST_SENSE, 0, 0, 0, SENSE_MAX as u8, 0], None)?.0; + sense = self.scsi(gen, cancel, target, &[scsi_cmd::REQUEST_SENSE, 0, 0, 0, SENSE_MAX as u8, 0], None)?.0; sense.truncate(SENSE_MAX); } let mut scsi_status = status as u16; - let mut state_flags = STATE_ALL_PHASES & !STATE_GOT_SENSE; + let mut state_flags = self.part.completion_phases(moved, described.max(xfer as u64)); if !sense.is_empty() { scsi_status |= SCSI_STATUS_SENSE_VALID; state_flags |= STATE_GOT_SENSE; @@ -1663,6 +1864,17 @@ impl Core { st.ctl.work = true; return st; } + if std::mem::take(&mut st.reset_targets) { + for channel in 0..self.part.channels() { + self.note_bus_reset(channel); + self.reset_targets(&mut st, channel as u8, None); + } + continue; + } + if let Some((channel, target)) = st.pending_target_resets.pop_front() { + self.reset_targets(&mut st, channel, target); + continue; + } // Completions held for room go first, in order, as the driver // frees it; while any is held the request ring waits. if !st.pending.is_empty() { @@ -1804,8 +2016,16 @@ impl Core { } Prepared::Run(job) => { st.total_cmds += 1; + st.active = Some((job.target, job.cdb.get(1).copied().unwrap_or(0) >> 5, job.handle)); + st.cancelled = false; st.ctl.busy = true; - let done = MutexGuard::unlocked(&mut st, || self.run_job(job, ring.gen)); + let cancel = self.cancel.load(Ordering::Acquire); + let mut done = MutexGuard::unlocked(&mut st, || self.run_job(job, ring.gen, cancel)); + if st.cancelled { done = None; } + else if self.part.has_abort_commands() && done.is_none() && st.ctl.gen == ring.gen { + if let Some(active) = st.active { st.outstanding.push(active); } + } + st.active = None; st.ctl.busy = false; self.wake.notify_all(); // A reset cancelled the command. @@ -1838,13 +2058,16 @@ impl Core { return st.mailbox[((off - reg::MAILBOX_BASE) / 2) as usize]; } if (reg::BANK_BASE..BANK_END).contains(&off) { - return st.active_bank()[((off - reg::BANK_BASE) / 2) as usize]; + let bank = self.part.active_bank(st.bus_cfg1); + return st.active_bank(bank)[((off - reg::BANK_BASE) / 2) as usize]; + } + if self.part.unmodelled_dma_register(off) { + return 0; } match off { reg::ID_LOW => PCI_VENDOR_QLOGIC, - reg::ID_HIGH => PCI_DEVICE_ISP1040, - // The revision field runs one ahead of the PCI byte. - reg::BUS_CFG0 => PCI_REVISION_ISP1040 as u16 + 1, + reg::ID_HIGH => self.part.id_high(), + reg::BUS_CFG0 => self.part.bus_cfg0(), reg::BUS_CFG1 => st.bus_cfg1, reg::BUS_CTRL => st.bus_ctrl, reg::BUS_STATUS => st.bus_status, @@ -1862,6 +2085,10 @@ impl Core { } fn reg_write(&self, off: u32, val: u16) { + if self.part.unmodelled_dma_register(off) { + self.note(format!("write to ISP12160 DMA register {off:#04x} is not modelled")); + return; + } let mut st = self.st.lock(); if off == reg::RESPONSE_QUEUE { // The consumer acknowledgement: recorded, never over the @@ -1884,15 +2111,17 @@ impl Core { return; } if (reg::BANK_BASE..BANK_END).contains(&off) { - st.active_bank()[((off - reg::BANK_BASE) / 2) as usize] = val; + let bank = self.part.active_bank(st.bus_cfg1); + st.active_bank(bank)[((off - reg::BANK_BASE) / 2) as usize] = val; return; } match off { reg::BUS_CFG1 => { - if val & !CFG1_WRITE_MASK != 0 { - self.note(format!("bus configuration 1 bits {:#06x} are not modelled", val & !CFG1_WRITE_MASK)); + let mask = self.part.cfg1_write_mask(); + if val & !mask != 0 { + self.note(format!("bus configuration 1 bits {:#06x} are not modelled", val & !mask)); } - st.bus_cfg1 = val & CFG1_WRITE_MASK; + st.bus_cfg1 = val & mask; } // Applied after the enables are stored, so the reset wins. reg::BUS_CTRL => { @@ -1900,6 +2129,7 @@ impl Core { // The command in flight is cancelled, not waited for. if val & CTRL_SOFT_RESET != 0 { st.reset(); + if self.part.reset_targets(&mut st) { self.kick(&mut st); } self.publish_gen(&st); self.wake.notify_all(); drop(st); @@ -2005,7 +2235,7 @@ impl crate::traits::Device for Isp1040 { impl crate::scsi_monitor::ScsiTargets for Isp1040 { fn scsi_bus(&self) -> &ScsiBus { - &self.bus + &self.buses[0] } } @@ -2041,14 +2271,15 @@ impl crate::traits::Saveable for Isp1040 { // A hold-off already over but not acted on still crosses: the // commands parked behind it start when the load re-arms it at 0. let quiet_left = st.quiet_until.map(|until| until.map(|u| u - now)); - Self::save_locked(st, left, quiet_left) + self.save_locked(st, left, quiet_left) }) } fn load_state(&self, v: &toml::Value) -> Result<(), String> { self.quiesced(|st| { let now = self.now_ns(st); - Self::load_locked(st, v, now); + self.load_locked(st, v, now); + self.cancel.store(0, Ordering::Release); self.publish_gen(st); }); self.sync_irq(); @@ -2057,7 +2288,7 @@ impl crate::traits::Saveable for Isp1040 { } impl Isp1040 { - fn save_locked(st: &State, selecting_left: Option, quiet_left: [Option; BUSES]) -> toml::Value { + fn save_locked(&self, st: &State, selecting_left: Option, quiet_left: [Option; BUSES]) -> toml::Value { let mut t = toml::map::Map::new(); for (k, v) in [ ("bus_cfg1", st.bus_cfg1), ("bus_ctrl", st.bus_ctrl), ("bus_status", st.bus_status), @@ -2130,16 +2361,18 @@ impl Isp1040 { m.insert("sense".into(), u8_slice_to_toml(&sel.done.sense)); t.insert("selecting".into(), toml::Value::Table(m)); } + self.part.save_extra(st, &mut t); toml::Value::Table(t) } /// Unprocessed request entries survive: the ring thread is kicked and /// runs them after the load. - fn load_locked(st: &mut State, v: &toml::Value, now: i64) { + fn load_locked(&self, st: &mut State, v: &toml::Value, now: i64) { { let ctl = std::mem::take(&mut st.ctl); *st = State::new(); st.ctl = ctl; + self.part.load_extra(st, v); st.ctl.gen += 1; st.ctl.rqst_epoch += 1; st.ctl.resp_epoch += 1; diff --git a/src/dev/isp1040/tests.rs b/src/dev/isp1040/tests.rs index 8a96b9d66..0bb7e084c 100644 --- a/src/dev/isp1040/tests.rs +++ b/src/dev/isp1040/tests.rs @@ -2,7 +2,8 @@ //! and ring contract. Host memory is a flat byte array behind a fake `PciDma`; the disk //! is a small raw image. //! -//! Ported from tests/qtest/qlogic-isp-test.c in jamesbraid/qemu-sgi-origami. +//! Ported from tests/qtest/qlogic-isp-test.c and tests/qtest/sn0-isp12160-test.c +//! in jamesbraid/qemu-sgi-origami. use super::*; use crate::irq::LevelProbe; @@ -59,15 +60,16 @@ fn image(name: &str, blocks: usize) -> std::path::PathBuf { } impl Rig { - /// A chip on fake memory, no disks. - fn bare() -> Rig { + fn on(isp: Isp1040) -> Rig { let mem = FakeMem::new(); - let isp = Isp1040::new(); Device::start(&isp); isp.connect_dma(mem.clone()); Rig { isp, mem } } + /// A chip on fake memory, no disks. + fn bare() -> Rig { Rig::on(Isp1040::new()) } + /// A chip with a disk of `blocks` blocks at target `id`. fn with_disk(name: &str, id: u8, blocks: usize) -> Rig { let r = Rig::bare(); @@ -1616,6 +1618,403 @@ fn a_controller_built_on_a_filled_bus_answers_for_its_target() { let _ = std::fs::remove_file(img); } + +struct MarkerTarget(u8); + +impl crate::scsi::ScsiTarget for MarkerTarget { + fn request(&mut self, _req: &crate::scsi::ScsiRequest) -> std::io::Result { + Ok(crate::scsi::ScsiResponse { status: 0, data: vec![self.0] }) + } +} + +/// Target 0 on channel 1 must stay distinct from target 0 on channel 0. +#[test] +fn the_isp12160_routes_ring_requests_to_separate_buses() { + let bus0 = Isp1040::new_bus(); + let bus1 = Isp1040::new_bus(); + bus0.attach(0, Box::new(MarkerTarget(0x10))).unwrap(); + bus1.attach(0, Box::new(MarkerTarget(0x20))).unwrap(); + let r = Rig::on(Isp1040::new_12160(bus0, bus1)); + r.init_rings(4, 4); + + r.put_entry(0, &request(0x1234, 0x80, &[0x12, 0, 0, 0, 36, 0], &[(BUF, 36)])); + r.doorbell(1); + assert_eq!(r.mem.get(BUF, 1), vec![0x20], "channel 1 reaches its target 0"); + + r.put_entry(1, &request(0x1235, 0, &[0x12, 0, 0, 0, 36, 0], &[(BUF + 64, 36)])); + r.doorbell(2); + assert_eq!(r.mem.get(BUF + 64, 1), vec![0x10], "channel 0 keeps its own target 0"); +} + +struct ResetProbe(Arc); + +impl crate::scsi::ScsiTarget for ResetProbe { + fn request(&mut self, _req: &crate::scsi::ScsiRequest) -> std::io::Result { + Ok(crate::scsi::ScsiResponse { status: 0, data: Vec::new() }) + } + + fn bus_reset(&mut self) { + self.0.fetch_add(1, Ordering::Relaxed); + } +} + +/// A snapshot retains both the chip-wide reset and the selected target reset, +/// and the resumed worker delivers both through the original channel buses. +#[test] +fn the_isp12160_preserves_queued_bus_and_target_resets_across_save_load() { + let src = Isp1040::new_12160(Isp1040::new_bus(), Isp1040::new_bus()); + { + let mut state = src.st.lock(); + state.reset_targets = true; + state.pending_target_resets.push_back((1, Some(2))); + } + let saved = src.save_state(); + + let resets0 = Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let resets1 = Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let selected_reset = Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let bus0 = Isp1040::new_bus(); + let bus1 = Isp1040::new_bus(); + bus0.attach(1, Box::new(ResetProbe(resets0.clone()))).unwrap(); + bus1.attach(1, Box::new(ResetProbe(resets1.clone()))).unwrap(); + bus1.attach(2, Box::new(ResetProbe(selected_reset.clone()))).unwrap(); + let dst = Isp1040::new_12160(bus0, bus1); + dst.load_state(&saved).unwrap(); + Device::start(&dst); + dst.wait_idle(); + Device::stop(&dst); + + assert_eq!(resets0.load(Ordering::Relaxed), 1); + assert_eq!(resets1.load(Ordering::Relaxed), 1); + assert_eq!(selected_reset.load(Ordering::Relaxed), 2, "the full reset and queued target reset both run"); + assert_eq!(saved.get("reset_targets").and_then(toml::Value::as_bool), Some(true)); + assert_eq!(saved.get("target_resets").and_then(toml::Value::as_array).map(Vec::len), Some(1)); + let after = dst.save_state(); + assert_eq!(after.get("reset_targets").and_then(toml::Value::as_bool), Some(false)); + assert_eq!(after.get("target_resets").and_then(toml::Value::as_array).map(Vec::len), Some(0)); +} + +// ---- ISP12160 second channel and recovery commands ----------------------- + +fn rig12160() -> Rig { Rig::on(Isp1040::new_12160(Isp1040::new_bus(), Isp1040::new_bus())) } + +/// A 64-block image whose first block is `fill`, so a read says which disk answered. +fn filled_image(name: &str, fill: u8) -> std::path::PathBuf { + let p = image(name, 64); + let mut data = vec![0u8; 64 * BLOCK_SIZE]; + data[..BLOCK_SIZE].fill(fill); + std::fs::write(&p, data).expect("test image"); + p +} + +/// A 12160 with a different disk at target 1 on each channel. +fn disks_on_both_channels(name: &str) -> Rig { + let r = rig12160(); + for channel in 0..2u8 { + let img = filled_image(&format!("{name}{channel}"), 0xa0 + channel); + r.isp.bus_for_channel(channel).unwrap().attach_image(1, &img, false).expect("attach"); + } + r +} + +/// Issues entries one at a time on a 16-entry ring and returns each completion. +struct Seq<'a> { + r: &'a Rig, + n: u16, +} + +impl<'a> Seq<'a> { + fn new(r: &'a Rig) -> Self { + r.init_rings(16, 16); + Seq { r, n: 0 } + } + fn run(&mut self, handle: u32, target: u8, cdb: &[u8], segs: &[(u32, u32)]) -> [u8; ENTRY_SIZE] { + let slot = self.n; + self.r.put_entry(slot as u32, &request(handle, target, cdb, segs)); + self.n += 1; + self.r.doorbell(self.n); + self.r.status(slot as u32) + } +} + +const TUR: [u8; 6] = [0; 6]; + +/// Counts bus resets and answers any command with a full block pair, so a read +/// whose second buffer segment is unreachable stays outstanding. +struct CountingDisk(Arc); + +impl crate::scsi::ScsiTarget for CountingDisk { + fn request(&mut self, _req: &crate::scsi::ScsiRequest) -> std::io::Result { + Ok(crate::scsi::ScsiResponse { status: 0, data: vec![0; 2 * BLOCK_SIZE] }) + } + fn bus_reset(&mut self) { self.0.fetch_add(1, Ordering::Relaxed); } +} + +struct Held { + r: Rig, + /// Bus resets seen by the counting disks at target 1 on channel 0, target 1 on channel 1, + /// and target 2 on channel 1. + ch0: Arc, + ch1: Arc, + ch1_other: Arc, +} + +impl Held { + fn new() -> Held { + let counters: Vec<_> = (0..3).map(|_| Arc::new(std::sync::atomic::AtomicUsize::new(0))).collect(); + let (bus0, bus1) = (Isp1040::new_bus(), Isp1040::new_bus()); + bus0.attach(1, Box::new(CountingDisk(counters[0].clone()))).unwrap(); + bus1.attach(1, Box::new(CountingDisk(counters[1].clone()))).unwrap(); + bus1.attach(2, Box::new(CountingDisk(counters[2].clone()))).unwrap(); + let r = Rig::on(Isp1040::new_12160(bus0, bus1)); + r.init_rings(16, 16); + Held { r, ch0: counters[0].clone(), ch1: counters[1].clone(), ch1_other: counters[2].clone() } + } + /// A read whose payload store the fabric refuses: the adapter keeps it outstanding, which + /// is what the abort and reset commands act on. + fn hold(&self, slot: u32, handle: u32, target: u8) { + self.r.put_entry(slot, &request(handle, target, &cdb10(scsi_cmd::READ_10, 0, 2), + &[(BUF, BLOCK_SIZE as u32), (MEM as u32, BLOCK_SIZE as u32)])); + self.r.doorbell(slot as u16 + 1); + assert_eq!(g32(&self.r.status(slot), 4), 0, "handle {handle:#x} is held, not completed"); + } + fn command(&self, regs: &[u16]) -> u16 { + let status = self.r.cmd(regs); + self.r.isp.wait_idle(); + status + } + /// The completion posted in response slot `slot`: (handle, completion code). + fn answer(&self, slot: u32) -> (u32, u16) { + let s = self.r.status(slot); + (g32(&s, 4), g16(&s, 10)) + } +} + +const CH1: u16 = 0x8000; + +/// A driver sizing its queues and recognising the part reads these before any ring exists. +#[test] +fn the_isp12160_reports_its_own_identity_firmware_and_unmapped_registers() { + let r = rig12160(); + assert_eq!(r.r(reg::ID_HIGH), 0x1216); + assert_eq!(r.r(reg::BUS_CFG0), 0, "no revision field on this part"); + r.w(reg::DMA_CMD_CFG, 0xffff); + r.w(reg::DMA_DATA_CFG, 0xffff); + assert_eq!((r.r(reg::DMA_CMD_CFG), r.r(reg::DMA_DATA_CFG)), (0, 0), "the flat ISP1040 DMA blocks are not here"); + + // Mailbox 6 is seeded because an untouched output would also read zero. + r.w(r.mb(6), 0x1234); + r.w(r.mb(0), mbox::ABOUT_FIRMWARE); + r.w(reg::HOST_CMD, 0x5 << HOST_CMD_SHIFT); + assert_eq!(r.r(r.mb(0)), mbox::COMPLETE); + assert_eq!([r.r(r.mb(1)), r.r(r.mb(2))], [10, 4]); + assert_eq!(r.r(r.mb(6)), 0, "the attribute word is cleared, not echoed"); + r.ack(); + r.w(r.mb(0), mbox::GET_FIRMWARE_STATUS); + r.w(reg::HOST_CMD, 0x5 << HOST_CMD_SHIFT); + assert_eq!(r.r(r.mb(0)), mbox::COMPLETE); + assert_eq!([r.r(r.mb(1)), r.r(r.mb(2))], [0, MAX_REQUESTS]); + r.ack(); +} + +/// The driver switches register banks through bus configuration 1 bits 9:8; each of the +/// four selections must show its own words, and no other bit may stick. +#[test] +fn the_isp12160_selects_four_distinct_banks_and_keeps_only_the_bank_field() { + let r = rig12160(); + let selections = [0x0000, 0x0100, 0x0200, 0x0300]; + for sel in selections { + r.w(reg::BUS_CFG1, sel); + r.w(reg::BANK_BASE, 0x1000 + sel); + r.w(reg::BANK_BASE + 0x20, 0x2000 + sel); + } + for sel in selections { + r.w(reg::BUS_CFG1, sel); + assert_eq!(r.r(reg::BANK_BASE), 0x1000 + sel, "bank {sel:#x}"); + assert_eq!(r.r(reg::BANK_BASE + 0x20), 0x2000 + sel, "bank {sel:#x}"); + } + r.w(reg::BUS_CFG1, 0xb2c5); + assert_eq!(r.r(reg::BUS_CFG1), 0x0200); + + // A soft reset hands a re-initialising driver empty banks, not the previous run's words. + r.w(reg::BUS_CTRL, CTRL_SOFT_RESET); + assert_eq!(r.r(reg::BUS_CFG1), 0); + for sel in selections { + r.w(reg::BUS_CFG1, sel); + assert_eq!(r.r(reg::BANK_BASE), 0, "bank {sel:#x} after reset"); + } +} + +/// A second command submitted before the driver acknowledged the first result must not +/// run, or it would overwrite the answer the driver has not read. +#[test] +fn the_isp12160_ignores_a_mailbox_submission_while_the_result_is_unacknowledged() { + let r = rig12160(); + r.w(r.mb(0), mbox::NOP); + r.w(reg::HOST_CMD, 0x5 << HOST_CMD_SHIFT); + assert_eq!(r.r(r.mb(0)), mbox::COMPLETE); + assert_eq!(r.r(reg::SEMAPHORE) & SEMAPHORE_LOCK, SEMAPHORE_LOCK); + + r.w(r.mb(1), 0x9999); + r.w(r.mb(0), mbox::ABOUT_FIRMWARE); + r.w(reg::HOST_CMD, 0x5 << HOST_CMD_SHIFT); + assert_eq!(r.r(r.mb(0)), mbox::ABOUT_FIRMWARE, "the second submission did not run"); + assert_eq!(r.r(r.mb(1)), 0x9999); + + r.ack(); + r.w(reg::HOST_CMD, 0x5 << HOST_CMD_SHIFT); + assert_eq!(r.r(r.mb(0)), mbox::COMPLETE); + assert_eq!(r.r(r.mb(1)), 10, "acknowledged, the same submission runs"); + r.ack(); +} + +/// An empty slot on either channel times out on its own bus; a driver probing channel 1 +/// must see an absent target, not a hang or the channel 0 answer. +#[test] +fn a_missing_target_times_out_on_each_channel() { + let r = rig12160(); + r.init_rings(16, 16); + r.clock(0); + // An absent target answers once its selection timeout has run out on + // the adapter's clock. + let mut now = 0; + for (slot, (handle, target)) in [(0x31, 5), (0x32, 0x80 | 5)].into_iter().enumerate() { + r.put_entry(slot as u32, &request(handle, target, &TUR, &[])); + r.doorbell(slot as u16 + 1); + assert_ne!(g32(&r.status(slot as u32), 4), handle, "nothing before the selection timeout"); + now += DEFAULT_SELECTION_TIMEOUT as i64 * MS; + r.clock(now); + let s = r.status(slot as u32); + assert_eq!(g32(&s, 4), handle); + assert_eq!(g16(&s, 10), IOCB_INCOMPLETE); + assert_eq!(g16(&s, 12), STATE_GOT_BUS); + assert_eq!(g16(&s, 14), STATUS_FLAG_TIMEOUT); + } +} + +/// A command refused on channel 1 must not disturb its neighbour on channel 0, and +/// a write must land on the disk of the channel it named. +#[test] +fn a_refused_command_on_channel_1_leaves_channel_0_and_the_media_alone() { + let r = disks_on_both_channels("refuse"); + let mut seq = Seq::new(&r); + // Collect each disk's power-on unit attention so the reads below see real answers. + seq.run(0xa0, 1, &TUR, &[]); + seq.run(0xa1, 0x81, &TUR, &[]); + + r.mem.put(BUF, &[0x5c; BLOCK_SIZE]); + let beyond_disk = seq.run(0xb1, 0x81, &cdb10(scsi_cmd::READ_10, 1000, 1), &[(BUF, BLOCK_SIZE as u32)]); + assert_eq!(g16(&beyond_disk, 8) & 0xff, CHECK_CONDITION as u16); + assert_eq!(r.mem.get(BUF, BLOCK_SIZE), vec![0x5c; BLOCK_SIZE], "the refused read stored nothing"); + + let good = seq.run(0xb0, 1, &cdb10(scsi_cmd::READ_10, 0, 1), &[(BUF + 0x1000, BLOCK_SIZE as u32)]); + assert_eq!(g16(&good, 10), IOCB_COMPLETE); + assert_eq!(r.mem.get(BUF + 0x1000, BLOCK_SIZE), vec![0xa0; BLOCK_SIZE], "channel 0 still reads its own disk"); + + r.mem.put(BUF + 0x2000, &[0x77; BLOCK_SIZE]); + seq.run(0xc0, 0x81, &cdb10(scsi_cmd::WRITE_10, 3, 1), &[(BUF + 0x2000, BLOCK_SIZE as u32)]); + seq.run(0xc1, 0x81, &cdb10(scsi_cmd::READ_10, 3, 1), &[(BUF + 0x3000, BLOCK_SIZE as u32)]); + seq.run(0xc2, 1, &cdb10(scsi_cmd::READ_10, 3, 1), &[(BUF + 0x4000, BLOCK_SIZE as u32)]); + assert_eq!(r.mem.get(BUF + 0x3000, BLOCK_SIZE), vec![0x77; BLOCK_SIZE], "channel 1 holds what was written"); + assert_eq!(r.mem.get(BUF + 0x4000, BLOCK_SIZE), vec![0; BLOCK_SIZE], "channel 0's block 3 is untouched"); +} + +/// Abort device names a channel in the top bit of its selector; the same target number +/// held on the other bus must stay outstanding. +#[test] +fn abort_device_retires_only_the_named_channels_commands() { + let h = Held::new(); + h.hold(0, 0x41, 1); + h.hold(1, 0x42, 0x81); + assert_eq!(h.command(&[mbox::ABORT_DEVICE, CH1 | 0x0100, 0, 0]), mbox::COMPLETE); + assert_eq!(h.answer(0), (0x42, IOCB_ABORTED)); + assert_eq!(h.answer(1), (0, 0), "channel 0's command is still held"); + // The survivor is still the adapter's to abort. + assert_eq!(h.command(&[mbox::ABORT_DEVICE, 0x0100, 0, 0]), mbox::COMPLETE); + assert_eq!(h.answer(1), (0x41, IOCB_ABORTED)); +} + +/// Abort command names one handle: two commands to the same channel, target and unit differ +/// only by it, and a selector that matches a held handle on the wrong channel is an error. +#[test] +fn abort_command_matches_the_handle_and_the_whole_selector() { + let h = Held::new(); + h.hold(0, 0x51, 0x81); + h.hold(1, 0x52, 0x81); + assert_eq!(h.command(&[mbox::ABORT_COMMAND, CH1 | 0x0100, 0, 0x52]), mbox::COMPLETE); + assert_eq!(h.answer(0), (0x52, IOCB_ABORTED)); + assert_eq!(h.answer(1), (0, 0), "its neighbour on the same target is untouched"); + + assert_eq!(h.command(&[mbox::ABORT_COMMAND, 0x0100, 0, 0x51]), mbox::COMMAND_ERROR, + "a held handle named on the wrong channel is not a match"); + assert_eq!(h.answer(1), (0, 0)); + assert_eq!(h.command(&[mbox::ABORT_COMMAND, CH1 | 0x0100, 0, 0x51]), mbox::COMPLETE); + assert_eq!(h.answer(1), (0x51, IOCB_ABORTED)); +} + +/// A driver clearing a device it already drained must read success, while naming a handle +/// nothing is using must read as an error rather than claim a retirement. +#[test] +fn aborting_nothing_is_complete_but_aborting_an_unknown_handle_is_refused() { + let h = Held::new(); + assert_eq!(h.command(&[mbox::ABORT_DEVICE, CH1 | 0x0100, 0, 0]), mbox::COMPLETE); + assert_eq!(h.command(&[mbox::ABORT_COMMAND, CH1 | 0x0100, 0, 0x77]), mbox::COMMAND_ERROR); + assert_eq!(h.answer(0), (0, 0), "neither posted a completion"); +} + +/// Target reset reaches one target on one channel; the same number on the other channel and +/// the neighbour on the same channel must not see it, and the reset target keeps serving. +#[test] +fn target_reset_is_scoped_to_one_target_on_one_channel() { + let h = Held::new(); + h.hold(0, 0x41, 1); + h.hold(1, 0x42, 0x81); + assert_eq!(h.command(&[mbox::ABORT_TARGET, CH1 | 0x0100, 0, 0]), mbox::COMPLETE); + assert_eq!(h.answer(0), (0x42, IOCB_RESET)); + assert_eq!(h.answer(1), (0, 0)); + assert_eq!([h.ch0.load(Ordering::Relaxed), h.ch1.load(Ordering::Relaxed), h.ch1_other.load(Ordering::Relaxed)], + [0, 1, 0]); + + h.r.put_entry(2, &request(0x43, 0x81, &TUR, &[])); + h.r.doorbell(3); + assert_eq!(h.answer(1), (0x43, IOCB_COMPLETE)); +} + +/// Bus reset takes its channel from mailbox 2; mailbox 1 is the reset delay and here holds a value +/// that is not even a channel. The whole named bus resets, the other one does not. +#[test] +fn bus_reset_takes_its_channel_from_mailbox_two_and_resets_every_target_on_it() { + let h = Held::new(); + h.hold(0, 0x41, 1); + h.hold(1, 0x42, 0x81); + assert_eq!(h.command(&[mbox::BUS_RESET, 3, 1]), mbox::COMPLETE); + assert_eq!(h.answer(0), (0x42, IOCB_RESET)); + assert_eq!(h.answer(1), (0, 0)); + assert_eq!([h.ch0.load(Ordering::Relaxed), h.ch1.load(Ordering::Relaxed), h.ch1_other.load(Ordering::Relaxed)], + [0, 1, 1]); + assert_eq!(h.command(&[mbox::BUS_RESET, 0, 2]), mbox::PARAM_ERROR, "there is no third bus"); +} + +/// A bus reset leaves a unit attention on the disks of that channel only; the driver's +/// first command afterwards is expected to collect it. +#[test] +fn a_bus_reset_leaves_a_unit_attention_on_the_reset_channel_only() { + let r = disks_on_both_channels("uareset"); + let mut seq = Seq::new(&r); + seq.run(0x10, 1, &TUR, &[]); + seq.run(0x11, 0x81, &TUR, &[]); + assert_eq!(g16(&seq.run(0x12, 0x81, &TUR, &[]), 8) & 0xff, 0, "attention was collected"); + + assert_eq!(r.cmd(&[mbox::BUS_RESET, 0, 1]), mbox::COMPLETE); + r.isp.wait_idle(); + let after = seq.run(0x13, 0x81, &TUR, &[]); + assert_eq!(g16(&after, 8) & 0xff, CHECK_CONDITION as u16); + assert_eq!(after[34] & 0x0f, 0x06, "unit attention"); + let other = seq.run(0x14, 1, &TUR, &[]); + assert_eq!(g16(&other, 8) & 0xff, 0, "channel 0 was not reset"); + assert_eq!(g16(&seq.run(0x15, 0x81, &TUR, &[]), 8) & 0xff, 0, "reported once"); +} + // ---- the hold-off after a bus reset --------------------------------------- // // Datasheet, "Bus-reset hold-off (0x0018)": after the reset the chip starts no @@ -1651,6 +2050,87 @@ fn a_bus_reset_puts_a_disks_write_cache_back() { assert_eq!(r.mem.get(BUF + 6, 1)[0] & 0x04, 0x04, "WCE is back"); } +/// A 12160 with a cached disk at target 1 on each channel whose guests cleared WCE on both. +fn cleared_write_cache_on_both_channels(name: &str) -> Rig { + let r = rig12160(); + for channel in 0..2u8 { + let bus = r.isp.bus_for_channel(channel).unwrap(); + bus.attach_cached_image(1, &image(&format!("{name}{channel}"), 64)).expect("attach"); + bus.select_write_cache(1, false); + assert!(!bus.write_cache_reported(1), "channel {channel}: the guest cleared it"); + } + r +} + +/// The write cache bit MODE SENSE(6) of the caching page reports for target 1 on `channel`. A TUR +/// goes first: it collects the reset's unit attention, and consumes the reset flag as a guest's +/// first command does, so the mode sense that follows returns its page. +fn wce_after_next_command(seq: &mut Seq, channel: u8, handle: u32) -> bool { + let target = channel << 7 | 1; + seq.run(handle, target, &TUR, &[]); + let done = seq.run(handle + 0x100, target, &[0x1a, 0x08, 0x08, 0, 24, 0], &[(BUF, 24)]); + assert_eq!(g16(&done, 10), IOCB_COMPLETE); + assert_eq!(seq.r.mem.get(BUF + 4, 1)[0], 0x08, "caching page"); + seq.r.mem.get(BUF + 6, 1)[0] & 0x04 != 0 +} + +/// Each channel's disks put their write cache back on that channel's reset and not on the +/// other's, so the flag has to be set for the channel the mailbox names. +#[test] +fn a_12160_bus_reset_of_channel_1_restores_write_cache_on_channel_1_only() { + let r = cleared_write_cache_on_both_channels("wce12160-ch1"); + let mut seq = Seq::new(&r); + assert_eq!(r.cmd(&[mbox::BUS_RESET, 0, 1]), mbox::COMPLETE); + r.isp.wait_idle(); + assert!(!wce_after_next_command(&mut seq, 0, 0x31), "channel 0 was not reset"); + assert!(wce_after_next_command(&mut seq, 1, 0x32), "channel 1 was reset"); +} + +#[test] +fn a_12160_bus_reset_of_channel_0_restores_write_cache_on_channel_0_only() { + let r = cleared_write_cache_on_both_channels("wce12160-ch0"); + let mut seq = Seq::new(&r); + assert_eq!(r.cmd(&[mbox::BUS_RESET, 0, 0]), mbox::COMPLETE); + r.isp.wait_idle(); + assert!(!wce_after_next_command(&mut seq, 1, 0x33), "channel 1 was not reset"); + assert!(wce_after_next_command(&mut seq, 0, 0x34), "channel 0 was reset"); +} + +/// Both reset the chip's SCSI buses, which the disks see as a bus reset on each channel. +#[test] +fn a_12160_chip_reset_restores_write_cache_on_both_channels() { + let r = cleared_write_cache_on_both_channels("wce12160-chip"); + r.isp.reset(); + r.isp.wait_idle(); + let mut seq = Seq::new(&r); + assert!(wce_after_next_command(&mut seq, 0, 0x35), "channel 0"); + assert!(wce_after_next_command(&mut seq, 1, 0x36), "channel 1"); +} + +#[test] +fn a_12160_soft_reset_restores_write_cache_on_both_channels() { + let r = cleared_write_cache_on_both_channels("wce12160-soft"); + r.w(reg::BUS_CTRL, CTRL_SOFT_RESET); + r.isp.wait_idle(); + let mut seq = Seq::new(&r); + assert!(wce_after_next_command(&mut seq, 0, 0x37), "channel 0"); + assert!(wce_after_next_command(&mut seq, 1, 0x38), "channel 1"); +} + +/// The single-bus part has no bus 1: a mailbox that names it is ignored, and the one bus it has +/// keeps its disks' write cache as the guest set it. +#[test] +fn an_isp1040_bus_reset_naming_bus_1_leaves_the_reset_flag_unset() { + let r = Rig::bare(); + r.isp.bus().attach_cached_image(1, &image("wce-bus1", 64)).expect("attach"); + r.init_rings(4, 4); + r.isp.bus().select_write_cache(1, false); + bus_reset(&r, 0, 1); + r.isp.wait_idle(); + assert!(!r.isp.targets_reset[0].load(Ordering::Acquire)); + assert!(!r.isp.targets_reset[1].load(Ordering::Acquire)); +} + /// A TUR for the disk at target 1. fn tur1(handle: u32) -> [u8; ENTRY_SIZE] { request(handle, 1, &[0, 0, 0, 0, 0, 0], &[]) diff --git a/src/dev/isp12160.rs b/src/dev/isp12160.rs new file mode 100644 index 000000000..9fcf55ab1 --- /dev/null +++ b/src/dev/isp12160.rs @@ -0,0 +1,338 @@ +//! The QLogic ISP12160: two SCSI buses behind one PCI function. +//! +//! The ISP12160 shares the ISP1040 ring, DMA, and mailbox mechanism, while +//! selecting targets from either channel. Each channel remains an explicit +//! [`ScsiBus`], including for monitor and run-time disc operations. +//! +//! Ported from hw/scsi/qlogic-isp.c and include/hw/scsi/qlogic-isp.h +//! in jamesbraid/qemu-sgi-origami. + +use std::collections::VecDeque; +use std::ops::Deref; + +use parking_lot::MutexGuard; + +use crate::dev::isp1040::{ + mbox, reg, Bank, Core, Isp1040, IspPart, State, CFG1_BANK_SELECT, FW_VERSION, IOCB_ABORTED, IOCB_RESET, MAILBOX_COUNT, + MAX_REQUESTS, MAX_TARGET, PCI_CLASS_SCSI, PCI_VENDOR_QLOGIC, SEMAPHORE_LOCK, STATE_ALL_PHASES, +}; +use crate::irq::IrqLine; +use crate::pci::{Bar, PciBusDevice, PciConfig, PciConfigPolicy, PciDma, PciRegisterPolicy, PCI_INTA}; +use crate::scsi::{ScsiBus, ScsiIdentities, ScsiIdentity}; +use crate::snapshot::{get_field, hex_u16, hex_u32, load_u16_slice, toml_u16, toml_u32, toml_u8, u16_slice_to_toml}; +use crate::traits::{Device, Resettable, Saveable}; +use crate::workstation::DiscChanger; + +const ISP_DISK: ScsiIdentity = ScsiIdentity::new(b"SGI ", b"IRIS EMULATED ", b"1.0 "); +const ISP_CDROM: ScsiIdentity = ISP_DISK; + +pub(crate) struct Isp12160Part; + +impl IspPart for Isp12160Part { + fn name(&self) -> &'static str { "ISP12160" } + fn channels(&self) -> usize { 2 } + fn pci_config(&self, bars: [Bar; 6]) -> PciConfig { + let registers = [ + PciRegisterPolicy { reg_dword: 13, initial: 0x44, writable_mask: 0 }, + PciRegisterPolicy { reg_dword: 17, initial: 0x0001_0001, writable_mask: 0 }, + PciRegisterPolicy { reg_dword: 18, initial: 0, writable_mask: 0x100 }, + ]; + PciConfig::new_with_policy( + PCI_VENDOR_QLOGIC, + 0x1216, + PCI_CLASS_SCSI, + 6, + bars, + PCI_INTA, + PciConfigPolicy { + status: 0x02b0, + status_w1c_mask: 0, + command_mask: 0x0156, + latency_mask: 0xff, + cache_line_mask: 0xff, + registers: ®isters, + }, + ) + } + fn firmware_version(&self) -> [u16; 3] { [10, 4, FW_VERSION[2]] } + fn active_bank(&self, cfg1: u16) -> Bank { + match (cfg1 >> 8) & 3 { + 1 => Bank::Sxp, + 2 => Bank::Sxp1, + 3 => Bank::Dma, + _ => Bank::Risc, + } + } + fn checks_ram_bounds(&self) -> bool { true } + fn has_abort_commands(&self) -> bool { true } + fn target_on(&self, channel: u8, target: u8) -> u8 { target | (channel << 4) } + fn decode_target(&self, target: u8) -> (usize, u8) { ((target >> 4) as usize, target & 0x0f) } + fn id_high(&self) -> u16 { 0x1216 } + fn bus_cfg0(&self) -> u16 { 0 } + fn cfg1_write_mask(&self) -> u16 { 0x0300 } + fn unmodelled_dma_register(&self, off: u32) -> bool { + matches!(off, reg::DMA_CMD_CFG | reg::DMA_CMD_CTRL | reg::DMA_DATA_CFG | reg::DMA_DATA_CTRL) + } + fn completion_phases(&self, moved: u64, total: u64) -> u16 { + let mut flags = 0x1700; + if moved != 0 { + flags |= 0x0800; + if moved == total { flags |= 0x4000; } + } + flags + } + fn mailbox_ready(&self, semaphore: u16) -> bool { semaphore & SEMAPHORE_LOCK == 0 } + fn clear_target_parameters(&self, mailbox: &mut [u16; MAILBOX_COUNT]) { mailbox[6] = 0; } + fn set_reset_delay(&self, st: &mut State) { + st.reset_delay = st.mailbox[1]; + st.reset_delay1 = st.mailbox[2]; + } + fn reset_targets(&self, st: &mut State) -> bool { + st.reset_targets = true; + true + } + fn execute_mailbox(&self, core: &Core, st: &mut MutexGuard<'_, State>) -> bool { + let cmd = st.mailbox[0]; + match cmd { + mbox::LOAD_RAM_A64 => st.mailbox[0] = core.ram_transfer(st, true), + mbox::ABORT_COMMAND | mbox::ABORT_DEVICE | mbox::ABORT_TARGET => { + let device = st.mailbox[1]; + let valid_target = (device >> 8) & 0x7f <= MAX_TARGET as u16; + let target = ((device >> 8) & 0x7f) as u8 | (((device >> 15) as u8) << 4); + let lun = device as u8; + let handle = ((st.mailbox[2] as u32) << 16) | st.mailbox[3] as u32; + let hit = valid_target && st.active.is_some_and(|(t, l, h)| t == target + && (cmd == mbox::ABORT_TARGET || l == lun) && (cmd != mbox::ABORT_COMMAND || h == handle)); + let completion = if cmd == mbox::ABORT_TARGET { IOCB_RESET } else { IOCB_ABORTED }; + if hit { core.cancel_active(st, completion); } + let retired = core.retire_outstanding(st, |(t, l, h)| valid_target && t == target + && (cmd == mbox::ABORT_TARGET || l == lun) && (cmd != mbox::ABORT_COMMAND || h == handle), completion); + if cmd == mbox::ABORT_TARGET && valid_target { + st.pending_target_resets.push_back((target >> 4, Some(target & 15))); + } + st.mailbox[0] = if cmd == mbox::ABORT_COMMAND && !hit && retired == 0 { mbox::COMMAND_ERROR } else { mbox::COMPLETE }; + } + // Mailbox 1 is the reset delay; mailbox 2 selects the bus. + mbox::BUS_RESET => { + let channel = st.mailbox[2]; + if channel > 1 { st.mailbox[0] = mbox::PARAM_ERROR; } + else { + if st.active.is_some_and(|(target, _, _)| target >> 4 == channel as u8) { + core.cancel_active(st, IOCB_RESET); + } + core.retire_outstanding(st, |(target, _, _)| target >> 4 == channel as u8, IOCB_RESET); + st.pending_target_resets.push_back((channel as u8, None)); + core.note_bus_reset(usize::from(channel)); + core.kick(st); + st.mailbox[0] = mbox::COMPLETE; + } + } + _ => return false, + } + true + } + fn save_extra(&self, st: &State, t: &mut toml::map::Map) { + t.insert("reset_targets".into(), toml::Value::Boolean(st.reset_targets)); + t.insert("target_resets".into(), toml::Value::Array(st.pending_target_resets.iter().map(|(channel, target)| { + toml::Value::Array(vec![hex_u16(*channel as u16), hex_u16(target.unwrap_or(0xff) as u16)]) + }).collect())); + t.insert("sxp1_bank".into(), u16_slice_to_toml(&st.sxp1_bank)); + t.insert("dma_bank".into(), u16_slice_to_toml(&st.dma_bank)); + t.insert("reset_delay1".into(), hex_u16(st.reset_delay1)); + t.insert("outstanding".into(), toml::Value::Array(st.outstanding.iter().map(|&(target, lun, handle)| { + toml::Value::Array(vec![hex_u16(target as u16), hex_u16(lun as u16), hex_u32(handle)]) + }).collect())); + } + fn load_extra(&self, st: &mut State, v: &toml::Value) { + st.reset_targets = get_field(v, "reset_targets").and_then(|x| x.as_bool()).unwrap_or(false); + if let Some(toml::Value::Array(items)) = get_field(v, "target_resets") { + for item in items.iter().take(64) { + if let toml::Value::Array(fields) = item { + if fields.len() == 2 { + if let (Some(channel), Some(target)) = (toml_u8(&fields[0]), toml_u8(&fields[1])) { + if channel < 2 && (target < 16 || target == 0xff) { + st.pending_target_resets.push_back((channel, (target < 16).then_some(target))); + } + } + } + } + } + } + let u16f = |k: &str| get_field(v, k).and_then(toml_u16).unwrap_or(0); + st.reset_delay1 = u16f("reset_delay1"); + if let Some(m) = get_field(v, "sxp1_bank") { load_u16_slice(m, &mut st.sxp1_bank); } + if let Some(m) = get_field(v, "dma_bank") { load_u16_slice(m, &mut st.dma_bank); } + if let Some(toml::Value::Array(items)) = get_field(v, "outstanding") { + for item in items.iter().take(MAX_REQUESTS as usize) { + if let toml::Value::Array(fields) = item { + if fields.len() == 3 { + if let (Some(target), Some(lun), Some(handle)) = + (toml_u8(&fields[0]), toml_u8(&fields[1]), toml_u32(&fields[2])) { + if target < 32 && lun < 8 { st.outstanding.push((target, lun, handle)); } + } + } + } + } + } + } +} + +/// The two-channel controller. The shared ISP engine owns both buses. +pub struct Isp12160(Isp1040); + +impl Default for Isp12160 { + fn default() -> Self { Self::new(Self::new_bus(), Self::new_bus()) } +} + +impl Isp12160 { + /// Light the status bar's SCSI activity bits in `heartbeat`. + pub fn set_heartbeat(&self, heartbeat: std::sync::Arc) { + self.0.set_heartbeat(heartbeat); + } + + /// A 16-id bus with the ISP controller's disk and CD-ROM identities. + pub fn new_bus() -> ScsiBus { + ScsiBus::new(16, ScsiIdentities { disk: ISP_DISK, cdrom: ISP_CDROM }) + } + + /// The controller with its two channel buses already populated. + pub fn new(bus0: ScsiBus, bus1: ScsiBus) -> Self { + Self(Isp1040::new_12160(bus0, bus1)) + } + + /// Borrow one channel for the shared monitor commands. + pub fn channel(&self, channel: u8) -> Option> { + (channel < 2).then_some(Channel { adapter: self, channel }) + } + + pub fn register_locks(&self) { self.0.register_locks(); } +} + +impl Deref for Isp12160 { + type Target = Core; + fn deref(&self) -> &Core { &self.0 } +} + +/// One channel, with SCSI ids 0–15 as the monitor presents them. +pub struct Channel<'a> { + adapter: &'a Isp12160, + channel: u8, +} + +impl crate::scsi_monitor::ScsiTargets for Channel<'_> { + fn scsi_bus(&self) -> &ScsiBus { + self.adapter.bus_for_channel(self.channel).expect("valid ISP12160 channel") + } +} + +/// The machine API flattens the ISP12160's channels to ids 0–31. +/// Monitor commands use a `Channel` instead, so their ids remain 0–15. +impl DiscChanger for Isp12160 { + fn disc_status(&self) -> Vec<(usize, String, Vec, u64, u64)> { + (0..2).flat_map(|channel| { + crate::scsi_monitor::disc_status(&self.channel(channel).expect("valid ISP12160 channel")) + .into_iter() + .map(move |(id, current, discs, physical, logical)| (id + channel as usize * 16, current, discs, physical, logical)) + }).collect() + } + + fn load_disc(&self, id: usize, path: String) -> Result { + let (channel, target) = Self::split_id(id)?; + crate::scsi_monitor::load_disc(&self.channel(channel).unwrap(), target, path) + } + + fn eject_disc(&self, id: usize) -> Result { + let (channel, target) = Self::split_id(id)?; + crate::scsi_monitor::eject_disc(&self.channel(channel).unwrap(), target) + } + + fn eject_to_empty(&self, id: usize) -> Result<(), String> { + let (channel, target) = Self::split_id(id)?; + crate::scsi_monitor::eject_to_empty(&self.channel(channel).unwrap(), target) + } +} + +impl Isp12160 { + fn split_id(id: usize) -> Result<(u8, usize), String> { + if id >= 32 { return Err(format!("SCSI ID {id} is out of range")); } + Ok(((id / 16) as u8, id % 16)) + } +} + +impl Device for Isp12160 { + fn step(&self, cycles: u64) { Device::step(&self.0, cycles) } + fn start(&self) { Device::start(&self.0) } + fn stop(&self) { Device::stop(&self.0) } + fn is_running(&self) -> bool { Device::is_running(&self.0) } + fn get_clock(&self) -> u64 { Device::get_clock(&self.0) } + + fn register_commands(&self) -> Vec<(String, String)> { + let usage = crate::scsi_monitor::SCSI_USAGE; + vec![ + ("scsi".into(), format!("SCSI (QLogic ISP12160, channel 0): {usage} | scsi notes")), + ("scsi0".into(), "Alias for `scsi` (ISP12160 channel 0)".into()), + ("cow".into(), "COW overlays on ISP12160 channel 0: cow status | cow commit [id] | cow reset [id]".into()), + ("scsi1".into(), format!("SCSI (QLogic ISP12160, channel 1): {usage}")), + ("cow1".into(), "COW overlays on ISP12160 channel 1: same subcommands as `cow`".into()), + ] + } + + fn execute_command(&self, cmd: &str, args: &[&str], mut w: Box) -> Result<(), String> { + use std::io::Write as _; + let (channel, cow) = match cmd { + "scsi" | "scsi0" | "cow" => (0, cmd.starts_with("cow")), + "scsi1" | "cow1" => (1, cmd.starts_with("cow")), + _ => return Err("Command not found".into()), + }; + let ctl = self.channel(channel).expect("valid ISP12160 channel"); + if cow { + return crate::scsi_monitor::cow_command(&ctl, args, &mut *w); + } + if let Some(done) = crate::scsi_monitor::scsi_subcommand(&ctl, args, &mut *w) { + return done; + } + if channel == 0 && args.first().copied() == Some("notes") { + let _ = writeln!(w, "QLogic ISP12160: {} commands", self.total_commands()); + for note in self.notes() { let _ = writeln!(w, " note: {note}"); } + return Ok(()); + } + Err(format!("Usage: {}", crate::scsi_monitor::SCSI_USAGE)) + } +} + +impl Resettable for Isp12160 { + fn power_on(&self) { + self.0.reset(); + PciBusDevice::config(&self.0).reset(); + } +} + +impl Saveable for Isp12160 { + fn save_state(&self) -> toml::Value { Saveable::save_state(&self.0) } + fn load_state(&self, v: &toml::Value) -> Result<(), String> { Saveable::load_state(&self.0, v) } +} + +impl PciBusDevice for Isp12160 { + fn config(&self) -> &PciConfig { PciBusDevice::config(&self.0) } + fn bar_read(&self, bar: usize, off: u32, width: usize) -> u32 { PciBusDevice::bar_read(&self.0, bar, off, width) } + fn bar_write(&self, bar: usize, off: u32, val: u32, width: usize) { PciBusDevice::bar_write(&self.0, bar, off, val, width) } + fn connect_interrupt(&self, line: IrqLine) { PciBusDevice::connect_interrupt(&self.0, line) } + fn connect_dma(&self, dma: std::sync::Arc) { PciBusDevice::connect_dma(&self.0, dma) } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// The machine disc API uses target 17 for channel 1, target 1. + #[test] + fn disc_status_flattens_the_second_channels_target_ids() { + let bus0 = Isp12160::new_bus(); + let bus1 = Isp12160::new_bus(); + bus1.attach_device(1, "", true, Vec::new(), false, None, false).unwrap(); + let isp = Isp12160::new(bus0, bus1); + + let status = DiscChanger::disc_status(&isp); + assert_eq!(status.iter().map(|drive| drive.0).collect::>(), vec![17]); + } +} diff --git a/src/dev/mod.rs b/src/dev/mod.rs index fdd54e4b1..c8e3177b3 100644 --- a/src/dev/mod.rs +++ b/src/dev/mod.rs @@ -60,3 +60,4 @@ pub mod vino; pub mod virtio; pub mod wd33c93a; pub mod z85c30; +pub mod isp12160; diff --git a/src/scsi.rs b/src/scsi.rs index bfb2ffda8..eb82e3347 100644 --- a/src/scsi.rs +++ b/src/scsi.rs @@ -162,6 +162,7 @@ pub struct ScsiDevice { kind: DeviceKind, buffer: Vec, // Internal buffer for WRITE_BUFFER/READ_BUFFER commands pending_sense: [u8; 18], // Stored sense data, served on REQUEST_SENSE + pending_attention: Option, /// Physical sector size: always 2048 for CD-ROM (Yellow Book), 512 for HDD. /// Used for TOC lead-out LBA (CD spec requires TOC in physical sectors). phys_block_size: u64, @@ -222,6 +223,7 @@ impl ScsiDevice { kind: if is_cdrom { DeviceKind::Cdrom } else { DeviceKind::Disk }, buffer: vec![0u8; SCSI_BUFFER_SIZE], pending_sense: [0u8; 18], + pending_attention: None, phys_block_size: if is_cdrom { CDROM_SECTOR } else { 512 }, // CD-ROM drives default to 2048-byte logical blocks (Sony CDU-76S behaviour). // HDD defaults to 512. dksc switches CD-ROM to 512 for EFS, back to 2048 for ISO. @@ -242,6 +244,7 @@ impl ScsiDevice { kind: DeviceKind::Cdrom, buffer: vec![0u8; SCSI_BUFFER_SIZE], pending_sense: [0u8; 18], + pending_attention: None, phys_block_size: CDROM_SECTOR, logical_block_size: 2048, identity, @@ -326,6 +329,10 @@ impl ScsiDevice { ScsiResponse { status: 0x02, data: vec![] } } + fn take_attention(&mut self) -> Option { + self.pending_attention.take().or_else(|| self.media.take_changed().then_some(0x28)) + } + fn exec_test_unit_ready(&mut self, _cdb: &[u8]) -> Result { if self.media.take_changed() { // Sense key 0x06 UNIT_ATTENTION, ASC 0x28 "Not Ready to Ready Transition / Medium Changed" @@ -1118,11 +1125,10 @@ impl ScsiTarget for ScsiDevice { } // Unit attention fires before any other command (except REQUEST_SENSE/INQUIRY). - if req.cdb[0] != scsi_cmd::REQUEST_SENSE - && req.cdb[0] != scsi_cmd::INQUIRY - && self.media.take_changed() - { - return Ok(self.check_condition(0x06, 0x28, 0x00)); + if req.cdb[0] != scsi_cmd::REQUEST_SENSE && req.cdb[0] != scsi_cmd::INQUIRY { + if let Some(asc) = self.take_attention() { + return Ok(self.check_condition(0x06, asc, 0x00)); + } } let mut response = match req.cdb[0] { @@ -1268,6 +1274,12 @@ impl ScsiTarget for ScsiDevice { fn block_size(&self) -> Option { Some(self.logical_block_size) } + fn bus_reset(&mut self) { + self.pending_sense = [0; 18]; + self.pending_sense[0] = 0x70; + self.pending_attention = Some(0x29); + } + fn reset_block_size(&mut self, cdrom_512: bool) { if cdrom_512 && self.is_cdrom() { self.logical_block_size = 512; @@ -1867,12 +1879,38 @@ mod tests { cd.changer().unwrap().load(&chd.display().to_string()).unwrap(); assert!(cd.has_media()); assert_eq!(cd.size(), 16 * 2048, "the CHD's size, not the file's"); - // The medium change is reported once, then the disc reads. + // A reset is reported before a media change that was already pending. + crate::scsi::ScsiTarget::bus_reset(&mut cd); + let reset = cd.request(&request(&[0, 0, 0, 0, 0, 0], None)).unwrap(); + assert_eq!(reset.status, 0x02, "unit attention after the reset"); + let reset_sense = cd.request(&request(&[0x03, 0, 0, 0, 18, 0], None)).unwrap(); + assert_eq!(reset_sense.data[12], 0x29, "reset attention comes first"); + let tur = cd.request(&request(&[0, 0, 0, 0, 0, 0], None)).unwrap(); - assert_eq!(tur.status, 0x02, "unit attention after the change"); + assert_eq!(tur.status, 0x02, "the pending media change follows"); + let sense = cd.request(&request(&[0x03, 0, 0, 0, 18, 0], None)).unwrap(); + assert_eq!(sense.data[12], 0x28, "media change keeps its ASC"); let read = cd.request(&request(&[0x28, 0, 0, 0, 0, 3, 0, 0, 1, 0], None)).unwrap(); assert_eq!(read.status, 0x00); assert_eq!(read.data, vec![0x42u8; 2048]); let _ = std::fs::remove_dir_all(dir); } + #[test] + fn bus_reset_reports_unit_attention_once_and_keeps_block_size() { + let mut dev = cdrom(); + dev.logical_block_size = 512; + dev.set_sense(0x05, 0x20, 0x00); + crate::scsi::ScsiTarget::bus_reset(&mut dev); + + let old_sense = run(&mut dev, &[0x03, 0, 0, 0, 18, 0]); + assert_eq!((old_sense.data[2], old_sense.data[12]), (0, 0), "reset clears previous sense"); + let response = run(&mut dev, &[0x00, 0, 0, 0, 0, 0]); + assert_eq!(response.status, 0x02, "a bus reset arms unit attention"); + let sense = run(&mut dev, &[0x03, 0, 0, 0, 18, 0]); + assert_eq!(sense.data[12], 0x29, "the reset attention has the reset ASC"); + + let response = run(&mut dev, &[0x00, 0, 0, 0, 0, 0]); + assert_eq!(response.status, 0x00, "the attention is reported once"); + assert_eq!(dev.block_sizes().1, 512, "bus reset leaves the selected block size alone"); + } } diff --git a/src/scsi/bus.rs b/src/scsi/bus.rs index 4283d763f..cb978d19a 100644 --- a/src/scsi/bus.rs +++ b/src/scsi/bus.rs @@ -51,8 +51,15 @@ pub trait ScsiTarget: Send { /// The logical block size, for a target that has one. fn block_size(&self) -> Option { None } - /// A controller's bus reset: block size back to what the controller - /// leaves, with a CD-ROM at 512 bytes if `cdrom_512`. + /// A SCSI bus reset reached this target, which owes the initiator a + /// unit attention on its next command. The ISP12160 calls this when the + /// guest resets either of its buses; the WD33C93A, AIC-7880 and ISP1040 + /// models never told their targets about a reset and still do not. + fn bus_reset(&mut self) {} + + /// The AIC-7880's bus reset: block size back to what that controller + /// leaves, with a CD-ROM at 512 bytes if `cdrom_512`. Only the AIC-7880 + /// calls this. fn reset_block_size(&mut self, _cdrom_512: bool) {} /// Put the mode parameters a guest can change back to what the target From 85e639b4798fca73d1e77e57e48612e4c6795804 Mon Sep 17 00:00:00 2001 From: James Braid Date: Fri, 30 Oct 2026 08:13:36 +0000 Subject: [PATCH 14/31] dp83843: add the Fuel Ethernet PHY Fuel fits a DP83843 at MII management address 1, where the Octane has an ICS1890. The IOC3 Ethernet controller needs the same packet and link machinery with either PHY, so keep that machinery shared: move the Octane's ICS1890 register model behind a PHY interface, and supply the DP83843's register and negotiation behavior through it. The MAC keeps its existing packet path. The interface carries the snapshot hooks too: the ICS1890 keeps the register save and load it had in the IOC3 model, and the DP83843 saves its registers and negotiation timers the same way, so the shared Ethernet block saves whichever PHY it holds. --- src/dev/dp83843.rs | 573 ++++++++++++++++++++++++++++++++++++++ src/dev/ics1890.rs | 423 ++++++++++++++++++++++++++++ src/dev/ioc3_eth.rs | 473 ++++--------------------------- src/dev/ioc3_eth/tests.rs | 36 +++ src/dev/mod.rs | 2 + 5 files changed, 1092 insertions(+), 415 deletions(-) create mode 100644 src/dev/dp83843.rs create mode 100644 src/dev/ics1890.rs diff --git a/src/dev/dp83843.rs b/src/dev/dp83843.rs new file mode 100644 index 000000000..8d3bb40c5 --- /dev/null +++ b/src/dev/dp83843.rs @@ -0,0 +1,573 @@ +//! The DP83843 MII model exposes the control, status and auto-negotiation +//! registers used by the Fuel IOC3 Ethernet port. A guest starts negotiation +//! through MII; the backend link level determines the simulated partner result. +//! +//! # What is approximated here +//! +//! Link settling and partner responses are software models. BMSR reports the +//! current link level rather than the hardware's latched-low status. +//! +//! Ported from hw/net/dp83840.c in jamesbraid/qemu-sgi-origami. + +use std::time::{Duration, Instant}; + +use crate::snapshot::{field, field_bool, field_u32, field_u64, field_u8, toml_u16, StateTable}; +pub mod reg { + pub const BASIC_CONTROL: u8 = 0x0; + pub const BASIC_STATUS: u8 = 0x1; + pub const PHY_ID1: u8 = 0x2; + pub const PHY_ID2: u8 = 0x3; + pub const AUTO_NEG_ADVERTISE: u8 = 0x4; + pub const AUTO_NEG_PARTNER: u8 = 0x5; + pub const AUTO_NEG_EXPANSION: u8 = 0x6; + pub const NEXT_PAGE_TRANSMIT: u8 = 0x7; + pub const PHY_STATUS: u8 = 0x10; + pub const INTERRUPT_CONTROL: u8 = 0x11; + pub const INTERRUPT_STATUS: u8 = 0x12; + pub const DISCONNECT_COUNTER: u8 = 0x13; + pub const FALSE_CARRIER_COUNTER: u8 = 0x14; + pub const RECEIVE_ERROR_COUNTER: u8 = 0x15; + pub const PCS_CONFIG: u8 = 0x16; + pub const LOOPBACK: u8 = 0x17; + pub const TEN_BASE_T_STATUS: u8 = 0x18; + pub const PHY_CONTROL: u8 = 0x19; +} +const SETTLE: Duration = Duration::from_secs(2); +const BREAK: Duration = Duration::from_millis(1500); + +/// Management state for one transceiver. No board GPIO reset is connected. +pub struct Dp83843 { + configured_addr: u8, + regs: [u16; 32], + backend_up: bool, + done: bool, + forced: bool, + resolved: u8, + armed: bool, + deadline: Option, + complete: Option, + pending: bool, + restarting: bool, + pub unimplemented_accesses: u32, +} +impl Dp83843 { + pub fn new(addr: u8) -> Self { + let mut p = Self { + configured_addr: addr & 31, + regs: [0; 32], + backend_up: true, + done: false, + forced: false, + resolved: 0, + armed: false, + deadline: None, + complete: None, + pending: false, + restarting: false, + unimplemented_accesses: 0, + }; + p.reset(); + p + } + pub fn reset(&mut self) { + self.regs = [0; 32]; + self.regs[reg::BASIC_CONTROL as usize] = 0x3100 | if self.configured_addr == 0 { 0x400 } else { 0 }; + self.regs[reg::AUTO_NEG_ADVERTISE as usize] = 0x1e1; + self.regs[reg::NEXT_PAGE_TRANSMIT as usize] = 0x2001; + self.regs[reg::PCS_CONFIG as usize] = 0x20; + self.regs[reg::PHY_CONTROL as usize] = u16::from(self.configured_addr); + self.done = false; + self.forced = false; + self.resolved = 0; + self.armed = false; + self.deadline = None; + self.complete = None; + self.pending = false; + self.restarting = false; + } + pub fn address(&self) -> u8 { + (self.regs[reg::PHY_CONTROL as usize] & 31) as u8 + } + pub fn backend_up(&self) -> bool { + self.backend_up + } + pub fn deadline(&self) -> Option { + self.deadline + } + fn autoneg(&self) -> bool { + self.regs[reg::BASIC_CONTROL as usize] & 0x1000 != 0 + } + fn supported(&self) -> bool { + self.regs[reg::PHY_CONTROL as usize] & 0x220 == 0 + && self.regs[reg::TEN_BASE_T_STATUS as usize] & 0x1a20 == 0 + && self.regs[reg::LOOPBACK as usize] & 0x3f80 == 0 + } + fn link(&self) -> bool { + self.backend_up + && self.supported() + && self.regs[reg::BASIC_CONTROL as usize] & 0x800 == 0 + && if self.autoneg() { + self.done && self.resolved != 0 + } else { + self.forced + } + } + pub fn data_path(&self) -> bool { + self.supported() && self.regs[reg::BASIC_CONTROL as usize] & 0xc00 == 0 + } + pub fn wire_enabled(&self) -> bool { + self.data_path() && self.link() + } + fn loopback_selected(&self) -> bool { + self.regs[reg::BASIC_CONTROL as usize] & 0x4000 != 0 || self.regs[reg::LOOPBACK as usize] & 0x30 != 0 + } + pub fn in_loopback(&self) -> bool { + self.data_path() && self.loopback_selected() + } + fn status(&self) -> u16 { + let mut v = if self.autoneg() { 0x200 } else { 0 }; + if self.loopback_selected() { + v |= 8; + } + if !self.link() { + return v; + } + v |= 1; + let (speed, full) = if self.autoneg() { + v |= 0x20; + (self.resolved <= 2, self.resolved == 1 || self.resolved == 3) + } else { + (self.regs[reg::BASIC_CONTROL as usize] & 0x2000 != 0, self.regs[reg::BASIC_CONTROL as usize] & 0x100 != 0) + }; + if !speed { + v |= 2; + } + if full { + v |= 4; + } + if self.regs[reg::BASIC_CONTROL as usize] & 0x400 == 0 { + v |= 0x800; + } + v + } + fn event(&mut self, before: u16) { + let after = self.status(); + if self.regs[reg::INTERRUPT_CONTROL as usize] & 2 != 0 && ((before ^ after) & 1 != 0 || !before & after & 0x20 != 0) { + self.pending = true; + } + } + fn resolve(&mut self) { + self.resolved = 0; + if self.autoneg() && self.supported() { + for (i, bit) in [0x100, 0x80, 0x40, 0x20].iter().enumerate() { + if self.regs[reg::AUTO_NEG_ADVERTISE as usize] & bit != 0 { + self.resolved = i as u8 + 1; + break; + } + } + } + if self.resolved == 0 { + crate::dlog!(crate::devlog::LogModule::Ioc, "dp83843: no common supported advertised mode"); + } + } + pub fn sync(&mut self, now: Instant) { + if self.deadline.is_some_and(|d| now >= d) { + let before = self.status(); + if let Some(end) = self.complete.take() { + self.regs[reg::BASIC_CONTROL as usize] &= !0x200; + self.deadline = Some(end); + if now < end { + return; + } + } + self.deadline = None; + self.restarting = false; + self.done = true; + self.resolve(); + self.event(before); + } + } + pub fn set_backend_link(&mut self, up: bool, now: Instant) { + self.sync(now); + let before = self.status(); + self.backend_up = up; + self.event(before); + } + fn unimplemented(&mut self, r: u8) { + self.unimplemented_accesses += 1; + crate::dlog!(crate::devlog::LogModule::Ioc, "dp83843: unsupported register or control at {r:#x}"); + } + pub fn read(&mut self, r: u8, now: Instant) -> u16 { + self.sync(now); + let v = match r { + reg::BASIC_CONTROL => self.regs[reg::BASIC_CONTROL as usize], + reg::BASIC_STATUS => { + if !self.armed && !self.done && self.autoneg() { + self.armed = true; + self.deadline = Some(now + SETTLE); + } + let status = self.status(); + 0x7849 | if status & 1 != 0 { 4 } else { 0 } | (status & 0x20) + } + reg::PHY_ID1 => 0x2000, + reg::PHY_ID2 => 0x5c10, + reg::AUTO_NEG_ADVERTISE | reg::NEXT_PAGE_TRANSMIT => self.regs[r as usize], + reg::AUTO_NEG_PARTNER => { + if self.done { + 0x1e1 + } else { + 0 + } + } + reg::AUTO_NEG_EXPANSION => 4 | u16::from(self.done), + reg::PHY_STATUS => { + let v = self.status() | if self.pending { 0x100 } else { 0 }; + self.pending = false; + v + } + reg::INTERRUPT_CONTROL => self.regs[reg::INTERRUPT_CONTROL as usize], + reg::INTERRUPT_STATUS => { + let v = if self.pending { 0x8000 } else { 0 }; + self.pending = false; + v + } + reg::DISCONNECT_COUNTER..=reg::RECEIVE_ERROR_COUNTER => { + let v = self.regs[r as usize]; + self.regs[r as usize] = 0; + v + } + reg::PCS_CONFIG => { + let v = self.regs[reg::PCS_CONFIG as usize]; + if self.autoneg() { + v & !0x4000 + } else if self.regs[reg::PHY_CONTROL as usize] & 0x20 != 0 { + v | 0x4000 + } else { + v + } + } + reg::LOOPBACK => { + self.regs[reg::LOOPBACK as usize] + | if self.regs[reg::PHY_CONTROL as usize] & 0x20 != 0 { + 0x1000 + } else { + 0 + } + } + reg::TEN_BASE_T_STATUS => { + self.regs[reg::TEN_BASE_T_STATUS as usize] + | if self.status() & 4 != 0 || self.regs[reg::PHY_CONTROL as usize] & 0x200 != 0 { + 0x80 + } else { + 0 + } + } + reg::PHY_CONTROL => self.regs[reg::PHY_CONTROL as usize], + _ => { + self.unimplemented(r); + 0 + } + }; + if self.regs[reg::INTERRUPT_CONTROL as usize] & 1 != 0 { + self.pending = true; + } + v + } + pub fn write(&mut self, r: u8, val: u16, now: Instant) { + self.sync(now); + let before = self.status(); + match r { + reg::BASIC_CONTROL => { + let val = val & 0xff80; + if val & 0x8000 != 0 { + self.reset(); + return; + } + if val & 0x80 != 0 { + self.unimplemented(r); + } + self.regs[reg::LOOPBACK as usize] &= !0x380; + if val & 0x4000 != 0 { + self.regs[reg::LOOPBACK as usize] |= 0x30; + } + if val & 0x400 == 0 && val & 0x1200 == 0x1200 { + self.done = false; + self.resolved = 0; + self.forced = false; + self.armed = true; + self.restarting = true; + self.deadline = Some(now + BREAK); + self.complete = Some(now + BREAK + SETTLE); + self.regs[reg::BASIC_CONTROL as usize] = val; + } else if val & 0x400 == 0 && (self.restarting || self.forced && !self.autoneg()) { + if val & 0x1000 == 0 { + self.restarting = false; + self.complete = None; + self.deadline = None; + self.forced = val & 0x800 == 0; + } + self.regs[reg::BASIC_CONTROL as usize] = val & !0x200; + } else { + self.restarting = false; + self.complete = None; + self.deadline = None; + self.forced = false; + self.done = val & 0x400 == 0; + self.armed = self.done; + self.regs[reg::BASIC_CONTROL as usize] = val & !0x8200; + } + if !self.autoneg() { + self.resolved = 0; + self.done = false; + self.forced = self.supported() && self.regs[reg::BASIC_CONTROL as usize] & 0xc00 == 0; + } else if self.done && self.resolved == 0 { + self.resolve(); + } + } + reg::BASIC_STATUS..=reg::PHY_ID2 | reg::AUTO_NEG_PARTNER | reg::AUTO_NEG_EXPANSION | reg::PHY_STATUS | reg::INTERRUPT_STATUS => {} + reg::AUTO_NEG_ADVERTISE => self.regs[reg::AUTO_NEG_ADVERTISE as usize] = val & 0xa5ff, + reg::NEXT_PAGE_TRANSMIT => self.regs[reg::NEXT_PAGE_TRANSMIT as usize] = (val & 0xb7ff) | (self.regs[reg::NEXT_PAGE_TRANSMIT as usize] & 0x800), + reg::INTERRUPT_CONTROL => self.regs[reg::INTERRUPT_CONTROL as usize] = val & 3, + reg::DISCONNECT_COUNTER..=reg::RECEIVE_ERROR_COUNTER => self.regs[r as usize] = val, + reg::PCS_CONFIG => { + if val & !0x20 != 0 { + self.unimplemented(r); + } + let mask = if self.autoneg() || self.regs[reg::PHY_CONTROL as usize] & 0x20 != 0 { + 0xbfef + } else { + 0xffef + }; + self.regs[reg::PCS_CONFIG as usize] = (self.regs[reg::PCS_CONFIG as usize] & !mask) | (val & mask); + } + reg::LOOPBACK => { + if val & 0x4000 != 0 { + self.unimplemented(r); + } + self.regs[reg::LOOPBACK as usize] = val & 0x7fb0; + } + reg::TEN_BASE_T_STATUS => { + if val & 0x55c != 0 { + self.unimplemented(r); + } + self.regs[reg::TEN_BASE_T_STATUS as usize] = val & 0x1ffc; + } + reg::PHY_CONTROL => { + if val & !31 != 0 { + self.unimplemented(r); + } + self.regs[reg::PHY_CONTROL as usize] = val & 0x3bff; + } + _ => self.unimplemented(r), + } + if !self.supported() { + self.resolved = 0; + self.forced = false; + self.unimplemented(r); + } + self.event(before); + if self.regs[reg::INTERRUPT_CONTROL as usize] & 1 != 0 { + self.pending = true; + } + } +} +// The registers and negotiation progress for a snapshot. The two instants +// are saved as the time they had left. +impl Dp83843 { + fn instant_left(at: Option, now: Instant) -> u64 { + // Nanoseconds left plus one; zero for none. + at.map_or(0, |d| d.saturating_duration_since(now).as_nanos() as u64 + 1) + } + + fn instant_from(left: u64, now: Instant) -> Option { + (left != 0).then(|| now + Duration::from_nanos(left - 1)) + } + + pub(crate) fn save_state(&self, now: Instant) -> toml::Value { + StateTable::new() + .u8("configured_addr", self.configured_addr) + .u16s("regs", &self.regs) + .bool("backend_up", self.backend_up) + .bool("done", self.done) + .bool("forced", self.forced) + .u8("resolved", self.resolved) + .bool("armed", self.armed) + .u64("deadline", Self::instant_left(self.deadline, now)) + .u64("complete", Self::instant_left(self.complete, now)) + .bool("pending", self.pending) + .bool("restarting", self.restarting) + .u32("unimplemented_accesses", self.unimplemented_accesses) + .build() + } + + pub(crate) fn load_state(&mut self, v: &toml::Value, now: Instant) -> Result<(), String> { + let items = field(v, "regs")?.as_array().ok_or("regs: not a list")?; + if items.len() != 32 { + return Err(format!("regs: {} entries, expected 32", items.len())); + } + let mut regs = [0u16; 32]; + for (r, item) in regs.iter_mut().zip(items) { + *r = toml_u16(item).ok_or("regs: an entry is out of range")?; + } + *self = Self { + configured_addr: field_u8(v, "configured_addr")?, + regs, + backend_up: field_bool(v, "backend_up")?, + done: field_bool(v, "done")?, + forced: field_bool(v, "forced")?, + resolved: field_u8(v, "resolved")?, + armed: field_bool(v, "armed")?, + deadline: Self::instant_from(field_u64(v, "deadline")?, now), + complete: Self::instant_from(field_u64(v, "complete")?, now), + pending: field_bool(v, "pending")?, + restarting: field_bool(v, "restarting")?, + unimplemented_accesses: field_u32(v, "unimplemented_accesses")?, + }; + Ok(()) + } +} + +impl crate::dev::ioc3_eth::MiiPhy for Dp83843 { + fn address(&self) -> u8 { Dp83843::address(self) } + fn deadline(&self) -> Option { Dp83843::deadline(self) } + fn wire_enabled(&self) -> bool { Dp83843::wire_enabled(self) } + fn in_loopback(&self) -> bool { Dp83843::in_loopback(self) } + fn sync(&mut self, now: Instant) { Dp83843::sync(self, now) } + fn read(&mut self, register: u8, now: Instant) -> u16 { Dp83843::read(self, register, now) } + fn write(&mut self, register: u8, value: u16, now: Instant) { Dp83843::write(self, register, value, now) } + fn set_backend_link(&mut self, up: bool, now: Instant) { Dp83843::set_backend_link(self, up, now) } + fn power_on(&mut self) { Dp83843::reset(self) } + fn save_state(&self, now: Instant) -> toml::Value { Dp83843::save_state(self, now) } + fn load_state(&mut self, v: &toml::Value, now: Instant) -> Result<(), String> { Dp83843::load_state(self, v, now) } +} + +#[cfg(test)] +mod tests { + // Cases ported from tests/qtest/sgi-ioc3-eth-test.c in jamesbraid/qemu-sgi-origami. + use super::*; + + /// The guest observes link and negotiation changes through MII status registers. + #[test] + fn the_phy_restarts_negotiation_and_resets_the_link() { + let now = Instant::now(); + let mut p = Dp83843::new(1); + assert_eq!(p.read(reg::BASIC_STATUS, now) & 0x24, 0); + assert!(!p.wire_enabled()); + assert_eq!(p.read(reg::BASIC_STATUS, now + Duration::from_secs(2)) & 0x24, 0x24); + p.write(reg::BASIC_CONTROL, 0x3300, now + Duration::from_secs(3)); + for (millis, restart, link) in [(4499, true, false), (4500, false, false), (6500, false, true)] { + let at = now + Duration::from_millis(millis); + assert_eq!(p.read(reg::BASIC_CONTROL, at) & 0x200 != 0, restart); + assert_eq!(p.read(reg::BASIC_STATUS, at) & 4 != 0, link); + assert_eq!(p.wire_enabled(), link); + } + let at = now + Duration::from_secs(7); + p.set_backend_link(false, at); + p.write(reg::PHY_CONTROL, 7, at); + p.write(reg::BASIC_CONTROL, 0x8000, at); + assert_eq!(p.address(), 1); + assert_eq!(p.read(reg::BASIC_STATUS, at) & 4, 0); + assert_eq!(p.read(reg::BASIC_STATUS, at + Duration::from_secs(2)) & 4, 0); + p.set_backend_link(true, at + Duration::from_secs(2)); + assert!(p.wire_enabled()); + } + + /// A PROM probing the PHY reads the next-page register's reset value and + /// writes it back; the read-only toggle bit must survive every write. + #[test] + fn the_next_page_register_keeps_its_toggle_bit_and_resets_to_0x2001() { + let now = Instant::now(); + let mut p = Dp83843::new(1); + assert_eq!(p.read(reg::NEXT_PAGE_TRANSMIT, now), 0x2001); + for bit in 0..16 { + let value = 1u16 << bit; + p.write(reg::NEXT_PAGE_TRANSMIT, value, now); + assert_eq!(p.read(reg::NEXT_PAGE_TRANSMIT, now), value & 0xb7ff, "bit {bit}"); + } + p.write(reg::NEXT_PAGE_TRANSMIT, 0xffff, now); + assert_eq!(p.read(reg::NEXT_PAGE_TRANSMIT, now), 0xb7ff); + p.write(reg::BASIC_CONTROL, 0x8000, now); + assert_eq!(p.read(reg::NEXT_PAGE_TRANSMIT, now), 0x2001); + } + + /// The PROM's PHY probe expects zero loopback and masked 10BASE-T status + /// registers, plus the standard BMSR and ANER capability bits, both at + /// power-on and after a BMCR reset. + #[test] + fn the_phy_reads_the_probe_values_the_prom_expects_before_and_after_reset() { + let now = Instant::now(); + let mut p = Dp83843::new(1); + for round in 0..2 { + assert_eq!(p.read(reg::BASIC_STATUS, now) & 0x7849, 0x7849, "round {round}"); + assert_eq!(p.read(reg::AUTO_NEG_EXPANSION, now) & 4, 4); + assert_eq!(p.read(reg::LOOPBACK, now), 0); + assert_eq!(p.read(reg::TEN_BASE_T_STATUS, now) & 0xc77f, 0); + p.write(reg::BASIC_CONTROL, 0x2100, now); + p.write(reg::LOOPBACK, 0x20, now); + p.write(reg::BASIC_CONTROL, 0x8000, now); + } + assert_eq!(p.unimplemented_accesses, 0); + } + + /// Setting BMCR loopback makes the PHY report loopback in the LBR bits the + /// MAC consults, and leaving it must drop the stale bypass bits. + #[test] + fn bmcr_loopback_selects_the_loopback_register_and_clears_the_bypass_bits() { + let now = Instant::now(); + let mut p = Dp83843::new(1); + p.write(reg::LOOPBACK, 0x0180, now); + p.write(reg::BASIC_CONTROL, 0x7300, now); + assert_eq!(p.read(reg::LOOPBACK, now) & 0x03b0, 0x0030); + assert!(p.in_loopback()); + let lbr = p.read(reg::LOOPBACK, now); + p.write(reg::LOOPBACK, lbr | 0x0180, now); + p.write(reg::BASIC_CONTROL, 0x3100, now); + assert_eq!(p.read(reg::LOOPBACK, now) & 0x0380, 0); + p.write(reg::BASIC_CONTROL, 0x8000, now); + assert_eq!(p.read(reg::LOOPBACK, now), 0); + assert!(!p.in_loopback()); + } + + /// Drivers poll the error counters and expect each read to return the count + /// since the previous read, not a running total. + #[test] + fn the_error_counters_clear_when_read() { + let now = Instant::now(); + let mut p = Dp83843::new(1); + for r in [reg::DISCONNECT_COUNTER, reg::FALSE_CARRIER_COUNTER, reg::RECEIVE_ERROR_COUNTER] { + p.write(r, 0xbeef, now); + assert_eq!(p.read(r, now), 0xbeef); + assert_eq!(p.read(r, now), 0); + } + } + + /// A guest that restarts negotiation twice in a row must see the break + /// interval from the second write, not a shortened one from the first. + #[test] + fn a_second_restart_request_rearms_the_break_interval() { + let now = Instant::now(); + let ms = |m: u64| now + Duration::from_millis(m); + let mut p = Dp83843::new(1); + p.write(reg::BASIC_CONTROL, 0x3200, ms(0)); + p.write(reg::BASIC_CONTROL, 0x3200, ms(750)); + assert_ne!(p.read(reg::BASIC_CONTROL, ms(2249)) & 0x200, 0); + assert_eq!(p.read(reg::BASIC_CONTROL, ms(2250)) & 0x200, 0); + assert_eq!(p.read(reg::BASIC_STATUS, ms(4249)) & 0x20, 0); + assert_eq!(p.read(reg::BASIC_STATUS, ms(4250)) & 0x20, 0x20); + } + + /// A snapshot keeps the registers, the negotiation in progress and its + /// time left. + #[test] + fn a_snapshot_round_trips_a_negotiation_in_progress() { + let now = Instant::now(); + let mut src = Dp83843::new(1); + src.write(reg::BASIC_CONTROL, 0x3300, now); + let v = src.save_state(now); + let mut dst = Dp83843::new(1); + dst.load_state(&v, now).unwrap(); + assert_eq!(v, dst.save_state(now)); + assert_eq!(src.deadline(), dst.deadline()); + assert!(dst.load_state(&toml::Value::Table(Default::default()), now).is_err()); + } +} diff --git a/src/dev/ics1890.rs b/src/dev/ics1890.rs new file mode 100644 index 000000000..04cc6706a --- /dev/null +++ b/src/dev/ics1890.rs @@ -0,0 +1,423 @@ +//! The ICS1890 transceiver fitted to Octane's IOC3. +//! +//! This module is shared by the Octane IOC3 Ethernet model. It was moved here +//! from `ioc3_eth.rs`; its management-register behavior follows the existing +//! Octane behavior and Integrated Circuit Systems, "ICS1890 10Base-T/100Base-TX +//! Integrated PHYceiver", document ICS1890RevG, 21 October 1997. +//! +//! Ported from hw/net/ics1890.c in jamesbraid/qemu-sgi-origami. + +use std::time::{Duration, Instant}; + +use crate::dev::ioc3_eth::MiiPhy; +use crate::snapshot::{field_bool, field_u16, field_u32, field_u64, field_u8, StateTable}; + +/// Ordinary and command-override writable masks, by register. +fn phy_masks(r: usize) -> (u16, u16) { + match r { + 0 => (0xff80, 0), + 1 => (0, 0x7f80), + 2 | 3 => (0, 0xffff), + 4 => (0x21e0, 0x1c1f), + 6 => (0, 0xffe0), + 0x10 => (0x802d, 0), + 0x12 => (0x002f, 0), + 0x13 => (0x0011, 0), + _ => (0, 0), + } +} + +/// The ICS1890 as the Octane carries it: address 1, node mode, software +/// priority, revision 3. +pub struct Ics1890 { + addr: u8, + bmcr: u16, + bmsr_caps: u16, + id1: u16, + id2: u16, + anar: u16, + partner: u16, + exp: u16, + ext_ctl: u16, + tenbt: u16, + ext2: u16, + override_armed: bool, + page_received: bool, + link_low_latch: bool, + last_link: bool, + backend_up: bool, + nego_done: bool, + deadline: Option, + delay: Duration, + /// Accesses to registers the part does not implement. + pub unimplemented_accesses: u32, +} + +impl Ics1890 { + pub fn new(addr: u8, delay: Duration) -> Self { + let mut p = Self { + addr, + bmcr: 0, + bmsr_caps: 0, + id1: 0, + id2: 0, + anar: 0, + partner: 0, + exp: 0, + ext_ctl: 0, + tenbt: 0, + ext2: 0, + override_armed: false, + page_received: false, + link_low_latch: false, + last_link: false, + backend_up: true, + nego_done: false, + deadline: None, + delay, + unimplemented_accesses: 0, + }; + p.reset(Instant::now()); + p + } + + pub fn address(&self) -> u8 { + self.addr + } + + /// Hardware and software reset: defaults, no override, no latches, and a + /// fresh negotiation if enabled. + pub fn reset(&mut self, now: Instant) { + self.bmcr = 0x3000; + self.bmsr_caps = 0x7800; + self.id1 = 0x0015; + self.id2 = 0xf423; + self.anar = 0x01e1; + self.partner = 0; + self.exp = 0; + self.ext_ctl = 0x0008; + self.tenbt = 0; + self.ext2 = 0x0001; + self.override_armed = false; + self.page_received = false; + self.link_low_latch = false; + self.nego_done = false; + self.deadline = None; + self.last_link = false; + self.start_negotiation(now); + } + + fn autoneg(&self) -> bool { + self.bmcr & 0x1000 != 0 + } + fn power_down(&self) -> bool { + self.bmcr & 0x0800 != 0 + } + fn isolated(&self) -> bool { + self.bmcr & 0x0400 != 0 + } + + /// The live link: down in power-down, down without a backend, and with + /// negotiation enabled up only once it has completed. + fn link(&self) -> bool { + !self.power_down() && self.backend_up && (!self.autoneg() || self.nego_done) + } + + fn start_negotiation(&mut self, now: Instant) { + self.nego_done = false; + self.partner = 0; + self.page_received = false; + self.deadline = None; + if self.autoneg() && self.backend_up && !self.power_down() { + // The synthetic peer advertises 0x01e1: IEEE 802.3, no fault. + let common = self.anar & 0x01e0; + if self.anar & 0x1f == 1 && common != 0 && self.anar & 0x2000 == 0 { + self.deadline = Some(now + self.delay); + } + } + } + + /// When a negotiation in progress completes, if one is. + pub fn deadline(&self) -> Option { + self.deadline + } + + /// Advance the negotiation deadline and note link transitions. Reads never + /// start or postpone negotiation; this only observes the clock. + pub fn sync(&mut self, now: Instant) { + if let Some(d) = self.deadline { + if now >= d && !self.power_down() && self.backend_up { + self.deadline = None; + self.nego_done = true; + self.partner = 0x01e1; + self.page_received = true; + } + } + let link = self.link(); + if self.last_link && !link { + self.link_low_latch = true; + } + self.last_link = link; + } + + /// The packet backend's cable state. + pub fn set_backend_link(&mut self, up: bool, now: Instant) { + self.sync(now); + if self.backend_up == up { + return; + } + self.backend_up = up; + if up { + // Recovery starts a fresh negotiation. + self.start_negotiation(now); + } else { + self.deadline = None; + self.nego_done = false; + } + self.sync(now); + } + + /// BMCR loopback turns frames round inside the part. Isolate and + /// power-down take precedence. + pub fn in_loopback(&self) -> bool { + self.bmcr & 0x4000 != 0 && !self.power_down() && !self.isolated() + } + + /// Frames may cross the MII at all. + pub fn data_path(&self) -> bool { + !self.power_down() && !self.isolated() + } + + /// Frames may go to or come from the packet backend. + pub fn wire_enabled(&self) -> bool { + self.data_path() && self.link() + } + + fn resolved(&self) -> (bool, bool) { + // (100 Mb/s, full duplex) + if self.autoneg() { + let common = self.anar & self.partner & 0x01e0; + if common & 0x100 != 0 { + (true, true) + } else if common & 0x80 != 0 { + (true, false) + } else if common & 0x40 != 0 { + (false, true) + } else { + (false, false) + } + } else { + (self.bmcr & 0x2000 != 0, self.bmcr & 0x0100 != 0) + } + } + + pub fn read(&mut self, r: u8, now: Instant) -> u16 { + self.sync(now); + match r { + 0 => self.bmcr, + 1 => { + let mut v = 0x0009 | self.bmsr_caps; + if self.autoneg() && self.nego_done { + v |= 0x0020; + } + if self.link() && !self.link_low_latch { + v |= 0x0004; + } + self.link_low_latch = false; + v + } + 2 => self.id1, + 3 => self.id2, + 4 => self.anar, + 5 => self.partner, + 6 => { + let v = (self.exp & 0xffe0) + | if self.nego_done { 1 } else { 0 } + | if self.page_received { 2 } else { 0 }; + self.page_received = false; + v + } + 0x10 => { + // Bit 4 is a fixed reserved read choice (stock firmware + // expects it set); bit 15 reads the override state. + (self.ext_ctl & 0x002d) + | (u16::from(self.addr & 0x1f) << 6) + | 0x0010 + | if self.override_armed { 0x8000 } else { 0 } + } + 0x11 => { + let link = self.link(); + let (speed, full) = self.resolved(); + let mut v = 0; + if link { + v |= 1; + if speed { + v |= 0x8000; + } + if full { + v |= 0x4000; + } + } + if self.nego_done { + v |= 0x0010 | (7 << 11); + } + v + } + 0x12 => (self.tenbt & 0x002f) | 0x0010, + // Node mode strap in bit 14, software bit 0. + 0x13 => (self.ext2 & 0x0011) | 0x4000, + _ => { + self.unimplemented_accesses += 1; + 0 + } + } + } + + pub fn write(&mut self, r: u8, val: u16, now: Instant) { + self.sync(now); + let r = usize::from(r); + let armed = self.override_armed; + match r { + 0 => { + if val & 0x8000 != 0 { + self.reset(now); + self.sync(now); + return; + } + let old = self.bmcr; + // Reset and restart are self-clearing. + self.bmcr = val & 0x7d80; + let restart = val & 0x0200 != 0; + let pd_now = self.power_down(); + if pd_now && old & 0x0800 == 0 { + // Entering power-down clears the latching status. + self.link_low_latch = false; + self.page_received = false; + self.deadline = None; + } else if !pd_now && old & 0x0800 != 0 { + // Release: enabled negotiation starts afresh. + self.start_negotiation(now); + } else if self.autoneg() && (restart || old & 0x1000 == 0) { + self.start_negotiation(now); + } else if !self.autoneg() { + self.nego_done = false; + self.deadline = None; + } + if armed { + self.override_armed = false; + } + } + 1 | 2 | 3 | 4 | 6 | 0x10 | 0x12 | 0x13 => { + let (ord, ovr) = phy_masks(r); + let mask = ord | if armed { ovr } else { 0 }; + let cell = match r { + 1 => &mut self.bmsr_caps, + 2 => &mut self.id1, + 3 => &mut self.id2, + 4 => &mut self.anar, + 6 => &mut self.exp, + 0x10 => &mut self.ext_ctl, + 0x12 => &mut self.tenbt, + _ => &mut self.ext2, + }; + let before = *cell; + *cell = (before & !mask) | (val & mask); + // The status caps live in bits 14..7 only. + if r == 1 { + self.bmsr_caps &= 0x7f80; + } + if r == 4 && self.anar != before && self.autoneg() { + self.start_negotiation(now); + } + if armed { + self.override_armed = false; + } else if r == 0x10 && val & 0x8000 != 0 { + self.override_armed = true; + } + } + _ => { + // Partner ability, Quick Poll and unimplemented registers + // ignore writes. The part is addressed, so a write still + // consumes an armed override. + if r != 5 && r != 0x11 { + self.unimplemented_accesses += 1; + } + if armed { + self.override_armed = false; + } + } + } + self.sync(now); + } +} + +// The transceiver's registers and auto-negotiation progress for a +// snapshot. Its one deadline is saved as the time it had left. +impl Ics1890 { + pub(crate) fn save_state(&self, now: Instant) -> toml::Value { + StateTable::new() + .u8("addr", self.addr) + .u16("bmcr", self.bmcr) + .u16("bmsr_caps", self.bmsr_caps) + .u16("id1", self.id1) + .u16("id2", self.id2) + .u16("anar", self.anar) + .u16("partner", self.partner) + .u16("exp", self.exp) + .u16("ext_ctl", self.ext_ctl) + .u16("tenbt", self.tenbt) + .u16("ext2", self.ext2) + .bool("override_armed", self.override_armed) + .bool("page_received", self.page_received) + .bool("link_low_latch", self.link_low_latch) + .bool("last_link", self.last_link) + .bool("backend_up", self.backend_up) + .bool("nego_done", self.nego_done) + // Nanoseconds left plus one; zero for none. + .u64("deadline", self.deadline.map_or(0, |d| d.saturating_duration_since(now).as_nanos() as u64 + 1)) + .u64("delay", self.delay.as_nanos() as u64) + .u32("unimplemented_accesses", self.unimplemented_accesses) + .build() + } + + pub(crate) fn load_state(&mut self, v: &toml::Value, now: Instant) -> Result<(), String> { + let deadline = field_u64(v, "deadline")?; + *self = Self { + addr: field_u8(v, "addr")?, + bmcr: field_u16(v, "bmcr")?, + bmsr_caps: field_u16(v, "bmsr_caps")?, + id1: field_u16(v, "id1")?, + id2: field_u16(v, "id2")?, + anar: field_u16(v, "anar")?, + partner: field_u16(v, "partner")?, + exp: field_u16(v, "exp")?, + ext_ctl: field_u16(v, "ext_ctl")?, + tenbt: field_u16(v, "tenbt")?, + ext2: field_u16(v, "ext2")?, + override_armed: field_bool(v, "override_armed")?, + page_received: field_bool(v, "page_received")?, + link_low_latch: field_bool(v, "link_low_latch")?, + last_link: field_bool(v, "last_link")?, + backend_up: field_bool(v, "backend_up")?, + nego_done: field_bool(v, "nego_done")?, + deadline: (deadline != 0).then(|| now + Duration::from_nanos(deadline - 1)), + delay: Duration::from_nanos(field_u64(v, "delay")?), + unimplemented_accesses: field_u32(v, "unimplemented_accesses")?, + }; + Ok(()) + } +} + +impl MiiPhy for Ics1890 { + fn address(&self) -> u8 { Ics1890::address(self) } + fn deadline(&self) -> Option { Ics1890::deadline(self) } + fn wire_enabled(&self) -> bool { Ics1890::wire_enabled(self) } + fn in_loopback(&self) -> bool { Ics1890::in_loopback(self) } + fn sync(&mut self, now: Instant) { Ics1890::sync(self, now) } + fn read(&mut self, register: u8, now: Instant) -> u16 { Ics1890::read(self, register, now) } + fn write(&mut self, register: u8, value: u16, now: Instant) { Ics1890::write(self, register, value, now) } + fn set_backend_link(&mut self, up: bool, now: Instant) { Ics1890::set_backend_link(self, up, now) } + fn power_on(&mut self) { Ics1890::reset(self, Instant::now()) } + fn save_state(&self, now: Instant) -> toml::Value { Ics1890::save_state(self, now) } + fn load_state(&mut self, v: &toml::Value, now: Instant) -> Result<(), String> { Ics1890::load_state(self, v, now) } +} diff --git a/src/dev/ioc3_eth.rs b/src/dev/ioc3_eth.rs index cbe3c0c88..4b9e9fa63 100644 --- a/src/dev/ioc3_eth.rs +++ b/src/dev/ioc3_eth.rs @@ -1,5 +1,5 @@ -//! The Ethernet MAC inside IOC3, with the ICS1890 transceiver the Octane -//! carries on its management bus. +//! The Ethernet MAC inside IOC3, with the Octane ICS1890 or Fuel DP83843 +//! transceiver on its management bus. //! //! IOC3 forwards the register offsets 0xf0..=0x150 of its register BAR here //! through [`Ioc3EthPort`]. The MAC is SGI's own design with no vendor @@ -17,7 +17,7 @@ //! Ring entries and descriptor words are big-endian bytes at bus addresses, //! frame bytes are in wire order; the Bridge applies any programmed swap. //! -//! Frames go through iris's network engine like the O2's MEC does: see +//! Frames use iris's shared network link. The machine attaches it through //! [`Ioc3Eth::attach`]. //! //! Approximations (the QEMU model this was ported from has the same ones, @@ -38,8 +38,8 @@ //! sends whenever the PHY's data path is open (not powered down or //! isolated), link or not. //! -//! Ported from hw/net/sgi-ioc3-eth.c and hw/net/ics1890.c in -//! jamesbraid/qemu-sgi-origami. +//! Ported from hw/net/sgi-ioc3-eth.c in jamesbraid/qemu-sgi-origami. The +//! Octane PHY model lives in `ics1890.rs`. use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; use std::sync::{Arc, OnceLock, Weak}; @@ -48,8 +48,9 @@ use parking_lot::Mutex; use std::time::{Duration, Instant}; use crate::dev::ioc3_port::Ioc3EthPort; +use crate::dev::ics1890::Ics1890; use crate::pci::{PciDma, PciDmaStatus}; -use crate::snapshot::{field, field_bool, field_u16, field_u32, field_u32s, field_u64, field_u8, StateTable}; +use crate::snapshot::{field, field_bool, field_u16, field_u32, field_u32s, field_u64, StateTable}; use crate::traits::{Device, Resettable, Saveable}; #[cfg(test)] @@ -163,6 +164,23 @@ pub const PHY_ADDR: u8 = 1; /// How long the synthetic link partner takes to complete negotiation. pub const NEGOTIATION_DELAY: Duration = Duration::from_millis(100); +/// The management and packet-path behavior that varies between IOC3 PHYs. +pub(crate) trait MiiPhy: Send { + fn address(&self) -> u8; + fn deadline(&self) -> Option; + fn wire_enabled(&self) -> bool; + fn in_loopback(&self) -> bool; + fn sync(&mut self, now: Instant); + fn read(&mut self, register: u8, now: Instant) -> u16; + fn write(&mut self, register: u8, value: u16, now: Instant); + fn set_backend_link(&mut self, up: bool, now: Instant); + fn power_on(&mut self); + /// The transceiver's registers and negotiation progress for a snapshot; + /// its deadline is saved as the time it had left. + fn save_state(&self, now: Instant) -> toml::Value; + fn load_state(&mut self, v: &toml::Value, now: Instant) -> Result<(), String>; +} + fn ridx(off: u32) -> usize { ((off - MAC_CONTROL) / 4) as usize } @@ -171,345 +189,6 @@ fn ridx(off: u32) -> usize { // ICS1890 transceiver // --------------------------------------------------------------------------- -/// Ordinary and command-override writable masks, by register. -fn phy_masks(r: usize) -> (u16, u16) { - match r { - 0 => (0xff80, 0), - 1 => (0, 0x7f80), - 2 | 3 => (0, 0xffff), - 4 => (0x21e0, 0x1c1f), - 6 => (0, 0xffe0), - 0x10 => (0x802d, 0), - 0x12 => (0x002f, 0), - 0x13 => (0x0011, 0), - _ => (0, 0), - } -} - -/// The ICS1890 as the Octane carries it: address 1, node mode, software -/// priority, revision 3. -pub struct Ics1890 { - addr: u8, - bmcr: u16, - bmsr_caps: u16, - id1: u16, - id2: u16, - anar: u16, - partner: u16, - exp: u16, - ext_ctl: u16, - tenbt: u16, - ext2: u16, - override_armed: bool, - page_received: bool, - link_low_latch: bool, - last_link: bool, - backend_up: bool, - nego_done: bool, - deadline: Option, - delay: Duration, - /// Accesses to registers the part does not implement. - pub unimplemented_accesses: u32, -} - -impl Ics1890 { - pub fn new(addr: u8, delay: Duration) -> Self { - let mut p = Self { - addr, - bmcr: 0, - bmsr_caps: 0, - id1: 0, - id2: 0, - anar: 0, - partner: 0, - exp: 0, - ext_ctl: 0, - tenbt: 0, - ext2: 0, - override_armed: false, - page_received: false, - link_low_latch: false, - last_link: false, - backend_up: true, - nego_done: false, - deadline: None, - delay, - unimplemented_accesses: 0, - }; - p.reset(Instant::now()); - p - } - - pub fn address(&self) -> u8 { - self.addr - } - - /// Hardware and software reset: defaults, no override, no latches, and a - /// fresh negotiation if enabled. - pub fn reset(&mut self, now: Instant) { - self.bmcr = 0x3000; - self.bmsr_caps = 0x7800; - self.id1 = 0x0015; - self.id2 = 0xf423; - self.anar = 0x01e1; - self.partner = 0; - self.exp = 0; - self.ext_ctl = 0x0008; - self.tenbt = 0; - self.ext2 = 0x0001; - self.override_armed = false; - self.page_received = false; - self.link_low_latch = false; - self.nego_done = false; - self.deadline = None; - self.last_link = false; - self.start_negotiation(now); - } - - fn autoneg(&self) -> bool { - self.bmcr & 0x1000 != 0 - } - fn power_down(&self) -> bool { - self.bmcr & 0x0800 != 0 - } - fn isolated(&self) -> bool { - self.bmcr & 0x0400 != 0 - } - - /// The live link: down in power-down, down without a backend, and with - /// negotiation enabled up only once it has completed. - fn link(&self) -> bool { - !self.power_down() && self.backend_up && (!self.autoneg() || self.nego_done) - } - - fn start_negotiation(&mut self, now: Instant) { - self.nego_done = false; - self.partner = 0; - self.page_received = false; - self.deadline = None; - if self.autoneg() && self.backend_up && !self.power_down() { - // The synthetic peer advertises 0x01e1: IEEE 802.3, no fault. - let common = self.anar & 0x01e0; - if self.anar & 0x1f == 1 && common != 0 && self.anar & 0x2000 == 0 { - self.deadline = Some(now + self.delay); - } - } - } - - /// When a negotiation in progress completes, if one is. - pub fn deadline(&self) -> Option { - self.deadline - } - - /// Advance the negotiation deadline and note link transitions. Reads never - /// start or postpone negotiation; this only observes the clock. - pub fn sync(&mut self, now: Instant) { - if let Some(d) = self.deadline { - if now >= d && !self.power_down() && self.backend_up { - self.deadline = None; - self.nego_done = true; - self.partner = 0x01e1; - self.page_received = true; - } - } - let link = self.link(); - if self.last_link && !link { - self.link_low_latch = true; - } - self.last_link = link; - } - - /// The packet backend's cable state. - pub fn set_backend_link(&mut self, up: bool, now: Instant) { - self.sync(now); - if self.backend_up == up { - return; - } - self.backend_up = up; - if up { - // Recovery starts a fresh negotiation. - self.start_negotiation(now); - } else { - self.deadline = None; - self.nego_done = false; - } - self.sync(now); - } - - /// BMCR loopback turns frames round inside the part. Isolate and - /// power-down take precedence. - pub fn in_loopback(&self) -> bool { - self.bmcr & 0x4000 != 0 && !self.power_down() && !self.isolated() - } - - /// Frames may cross the MII at all. - pub fn data_path(&self) -> bool { - !self.power_down() && !self.isolated() - } - - /// Frames may go to or come from the packet backend. - pub fn wire_enabled(&self) -> bool { - self.data_path() && self.link() - } - - fn resolved(&self) -> (bool, bool) { - // (100 Mb/s, full duplex) - if self.autoneg() { - let common = self.anar & self.partner & 0x01e0; - if common & 0x100 != 0 { - (true, true) - } else if common & 0x80 != 0 { - (true, false) - } else if common & 0x40 != 0 { - (false, true) - } else { - (false, false) - } - } else { - (self.bmcr & 0x2000 != 0, self.bmcr & 0x0100 != 0) - } - } - - pub fn read(&mut self, r: u8, now: Instant) -> u16 { - self.sync(now); - match r { - 0 => self.bmcr, - 1 => { - let mut v = 0x0009 | self.bmsr_caps; - if self.autoneg() && self.nego_done { - v |= 0x0020; - } - if self.link() && !self.link_low_latch { - v |= 0x0004; - } - self.link_low_latch = false; - v - } - 2 => self.id1, - 3 => self.id2, - 4 => self.anar, - 5 => self.partner, - 6 => { - let v = (self.exp & 0xffe0) - | if self.nego_done { 1 } else { 0 } - | if self.page_received { 2 } else { 0 }; - self.page_received = false; - v - } - 0x10 => { - // Bit 4 is a fixed reserved read choice (stock firmware - // expects it set); bit 15 reads the override state. - (self.ext_ctl & 0x002d) - | (u16::from(self.addr & 0x1f) << 6) - | 0x0010 - | if self.override_armed { 0x8000 } else { 0 } - } - 0x11 => { - let link = self.link(); - let (speed, full) = self.resolved(); - let mut v = 0; - if link { - v |= 1; - if speed { - v |= 0x8000; - } - if full { - v |= 0x4000; - } - } - if self.nego_done { - v |= 0x0010 | (7 << 11); - } - v - } - 0x12 => (self.tenbt & 0x002f) | 0x0010, - // Node mode strap in bit 14, software bit 0. - 0x13 => (self.ext2 & 0x0011) | 0x4000, - _ => { - self.unimplemented_accesses += 1; - 0 - } - } - } - - pub fn write(&mut self, r: u8, val: u16, now: Instant) { - self.sync(now); - let r = usize::from(r); - let armed = self.override_armed; - match r { - 0 => { - if val & 0x8000 != 0 { - self.reset(now); - self.sync(now); - return; - } - let old = self.bmcr; - // Reset and restart are self-clearing. - self.bmcr = val & 0x7d80; - let restart = val & 0x0200 != 0; - let pd_now = self.power_down(); - if pd_now && old & 0x0800 == 0 { - // Entering power-down clears the latching status. - self.link_low_latch = false; - self.page_received = false; - self.deadline = None; - } else if !pd_now && old & 0x0800 != 0 { - // Release: enabled negotiation starts afresh. - self.start_negotiation(now); - } else if self.autoneg() && (restart || old & 0x1000 == 0) { - self.start_negotiation(now); - } else if !self.autoneg() { - self.nego_done = false; - self.deadline = None; - } - if armed { - self.override_armed = false; - } - } - 1 | 2 | 3 | 4 | 6 | 0x10 | 0x12 | 0x13 => { - let (ord, ovr) = phy_masks(r); - let mask = ord | if armed { ovr } else { 0 }; - let cell = match r { - 1 => &mut self.bmsr_caps, - 2 => &mut self.id1, - 3 => &mut self.id2, - 4 => &mut self.anar, - 6 => &mut self.exp, - 0x10 => &mut self.ext_ctl, - 0x12 => &mut self.tenbt, - _ => &mut self.ext2, - }; - let before = *cell; - *cell = (before & !mask) | (val & mask); - // The status caps live in bits 14..7 only. - if r == 1 { - self.bmsr_caps &= 0x7f80; - } - if r == 4 && self.anar != before && self.autoneg() { - self.start_negotiation(now); - } - if armed { - self.override_armed = false; - } else if r == 0x10 && val & 0x8000 != 0 { - self.override_armed = true; - } - } - _ => { - // Partner ability, Quick Poll and unimplemented registers - // ignore writes. The part is addressed, so a write still - // consumes an armed override. - if r != 5 && r != 0x11 { - self.unimplemented_accesses += 1; - } - if armed { - self.override_armed = false; - } - } - } - self.sync(now); - } -} - // --------------------------------------------------------------------------- // Checksums and the multicast hash // --------------------------------------------------------------------------- @@ -616,7 +295,7 @@ struct State { mii_data_w: u16, /// Apart from its index: an index-only write must not disarm. rx_timer_armed: bool, - phy: Ics1890, + phy: Box, irq_asserted: bool, /// Level changes not yet delivered to the callback, in order. irq_events: Vec, @@ -626,14 +305,14 @@ struct State { } impl State { - fn new(delay: Duration) -> Self { + fn new(phy: Box) -> Self { Self { regs: [0; NREGS], int_deferred: 0, mii_data: MII_ABSENT, mii_data_w: 0, rx_timer_armed: false, - phy: Ics1890::new(PHY_ADDR, delay), + phy, irq_asserted: false, irq_events: Vec::new(), frame: vec![0; FRAME_MAX], @@ -642,6 +321,20 @@ impl State { } } + fn power_on(&mut self) { + self.phy.power_on(); + self.regs = [0; NREGS]; + self.int_deferred = 0; + self.mii_data = MII_ABSENT; + self.mii_data_w = 0; + self.rx_timer_armed = false; + self.irq_asserted = false; + self.irq_events.clear(); + self.frame.fill(0); + self.tx_frames = 0; + self.rx_frames = 0; + } + fn get(&self, off: u32) -> u32 { self.regs[ridx(off)] } @@ -1025,6 +718,7 @@ type IrqFn = Box; /// through the Bridge and HEART, which never call back into this device. pub struct Ioc3Eth { mac: [u8; 6], + thread_prefix: &'static str, st: Mutex, dma: Mutex>>, irq: Mutex>, @@ -1062,9 +756,19 @@ impl Ioc3Eth { /// As [`Ioc3Eth::new`], with the PHY's synthetic negotiation delay set. pub fn with_negotiation_delay(mac: [u8; 6], delay: Duration) -> Arc { + Self::with_phy(mac, Box::new(Ics1890::new(PHY_ADDR, delay)), "ip30-eth") + } + + /// Fuel uses the DP83843 at management address 1. + pub fn new_fuel(mac: [u8; 6]) -> Arc { + Self::with_phy(mac, Box::new(crate::dev::dp83843::Dp83843::new(PHY_ADDR)), "ioc3-eth") + } + + fn with_phy(mac: [u8; 6], phy: Box, thread_prefix: &'static str) -> Arc { Arc::new_cyclic(|me| Self { mac, - st: Mutex::new(State::new(delay)), + thread_prefix, + st: Mutex::new(State::new(phy)), dma: Mutex::new(None), irq: Mutex::new(None), irq_order: Mutex::new(()), @@ -1101,13 +805,7 @@ impl Ioc3Eth { /// with the state saying low. pub fn reset(&self) { let _order = self.irq_order.lock(); - { - let mut st = self.st.lock(); - let delay = st.phy.delay; - let backend_up = st.phy.backend_up; - *st = State::new(delay); - st.phy.set_backend_link(backend_up, Instant::now()); - } + self.st.lock().power_on(); if let Some(cb) = self.irq.lock().as_ref() { cb(false); } @@ -1175,8 +873,9 @@ impl Ioc3Eth { fn spawn_nat(&self) { let slot = self.backend.clone(); let Some(mut engine) = slot.lock().take() else { return }; + let thread_name = format!("{}-net", self.thread_prefix); let t = std::thread::Builder::new() - .name("ip30-eth-net".into()) + .name(thread_name) .spawn(move || { engine.run(); *slot.lock() = Some(engine); @@ -1281,8 +980,9 @@ impl Device for Ioc3Eth { } let wake = self.rx_wake.lock().get_or_insert_with(crate::net::Wake::new).clone(); let (me, running) = (self.me.clone(), self.running.clone()); + let thread_name = format!("{}-rx", self.thread_prefix); let rx = std::thread::Builder::new() - .name("ip30-eth-rx".into()) + .name(thread_name) .spawn(move || Self::rx_loop(me, running, wake)) .expect("ioc3 eth rx thread"); self.threads.lock().push(rx); @@ -1377,63 +1077,6 @@ impl Ioc3EthPort for Ioc3Eth { } } -// The transceiver's registers and auto-negotiation progress for a -// snapshot. Its one deadline is saved as the time it had left. -impl Ics1890 { - pub(crate) fn save_state(&self, now: Instant) -> toml::Value { - StateTable::new() - .u8("addr", self.addr) - .u16("bmcr", self.bmcr) - .u16("bmsr_caps", self.bmsr_caps) - .u16("id1", self.id1) - .u16("id2", self.id2) - .u16("anar", self.anar) - .u16("partner", self.partner) - .u16("exp", self.exp) - .u16("ext_ctl", self.ext_ctl) - .u16("tenbt", self.tenbt) - .u16("ext2", self.ext2) - .bool("override_armed", self.override_armed) - .bool("page_received", self.page_received) - .bool("link_low_latch", self.link_low_latch) - .bool("last_link", self.last_link) - .bool("backend_up", self.backend_up) - .bool("nego_done", self.nego_done) - // Nanoseconds left plus one; zero for none. - .u64("deadline", self.deadline.map_or(0, |d| d.saturating_duration_since(now).as_nanos() as u64 + 1)) - .u64("delay", self.delay.as_nanos() as u64) - .u32("unimplemented_accesses", self.unimplemented_accesses) - .build() - } - - pub(crate) fn load_state(&mut self, v: &toml::Value, now: Instant) -> Result<(), String> { - let deadline = field_u64(v, "deadline")?; - *self = Self { - addr: field_u8(v, "addr")?, - bmcr: field_u16(v, "bmcr")?, - bmsr_caps: field_u16(v, "bmsr_caps")?, - id1: field_u16(v, "id1")?, - id2: field_u16(v, "id2")?, - anar: field_u16(v, "anar")?, - partner: field_u16(v, "partner")?, - exp: field_u16(v, "exp")?, - ext_ctl: field_u16(v, "ext_ctl")?, - tenbt: field_u16(v, "tenbt")?, - ext2: field_u16(v, "ext2")?, - override_armed: field_bool(v, "override_armed")?, - page_received: field_bool(v, "page_received")?, - link_low_latch: field_bool(v, "link_low_latch")?, - last_link: field_bool(v, "last_link")?, - backend_up: field_bool(v, "backend_up")?, - nego_done: field_bool(v, "nego_done")?, - deadline: (deadline != 0).then(|| now + Duration::from_nanos(deadline - 1)), - delay: Duration::from_nanos(field_u64(v, "delay")?), - unimplemented_accesses: field_u32(v, "unimplemented_accesses")?, - }; - Ok(()) - } -} - // The Ethernet block's state for a snapshot: the MAC's registers, the // MII port and the transceiver. Frames in flight between the receive // thread and the host network are not state: with the threads stopped diff --git a/src/dev/ioc3_eth/tests.rs b/src/dev/ioc3_eth/tests.rs index 64a54bb4f..d0ea63891 100644 --- a/src/dev/ioc3_eth/tests.rs +++ b/src/dev/ioc3_eth/tests.rs @@ -814,6 +814,42 @@ fn the_last_reference_released_on_another_thread_stops_the_mac() { assert!(!running.load(Ordering::SeqCst), "the last release stops it"); } +#[test] +fn fuel_phy_mdio_decoder_and_machine_reset() { + let eth = Ioc3Eth::new_fuel(STATION); + assert_eq!(eth.mac(), STATION); + let read = |phy: u32, r: u32| { + eth.write32(MII_CONTROL, MII_READ | (phy << 5) | r); + eth.read32(MII_DATA_R) + }; + assert_eq!(read(1, 2), 0x2000); + assert_eq!(read(1, 3), 0x5c10); + assert_eq!(read(0, 2), 0xffff); + eth.write32(MII_DATA_W, 7); + eth.write32(MII_CONTROL, (1 << 5) | 0x19); + assert_eq!(read(1, 2), 0xffff); + assert_eq!(read(7, 2), 0x2000); + // A wrong-address reset has no effect. + eth.write32(MII_DATA_W, 0x8000); + eth.write32(MII_CONTROL, 1 << 5); + assert_eq!(read(7, 2), 0x2000); + eth.reset(); + assert_eq!(read(1, 3), 0x5c10); + assert_eq!(read(7, 2), 0xffff); + // The MAC soft reset leaves PHY programming intact. + eth.write32(MII_DATA_W, 0x21); + eth.write32(MII_CONTROL, (1 << 5) | 4); + eth.write32(MAC_CONTROL, MCR_RESET); + assert_eq!(read(1, 4), 0x21); + // Forced mode exposes the configured backend cable without waiting. + eth.write32(MII_DATA_W, 0x2100); + eth.write32(MII_CONTROL, 1 << 5); + for up in [false, true] { + eth.set_backend_link(up); + assert_eq!(read(1, 1) & 4 != 0, up); + } +} + /// A guest that does nothing but ACK: the whole host side of a lock-step /// TFTP read (transmit descriptor, NAT thread, built-in server, receive /// thread, receive ring), with the guest's own software left out. Every diff --git a/src/dev/mod.rs b/src/dev/mod.rs index c8e3177b3..a8392b11e 100644 --- a/src/dev/mod.rs +++ b/src/dev/mod.rs @@ -60,4 +60,6 @@ pub mod vino; pub mod virtio; pub mod wd33c93a; pub mod z85c30; +pub mod dp83843; +pub mod ics1890; pub mod isp12160; From daaf4d79258f7ea383e8f1da7b65ab50bae37765 Mon Sep 17 00:00:00 2001 From: James Braid Date: Fri, 30 Oct 2026 08:13:36 +0000 Subject: [PATCH 15/31] timekeeper: add the M48T35 clock and NVRAM Fuel stores its clock and nonvolatile configuration in an M48T35 on the IOC3 byte bus. Keep the register overlay and battery-backed bytes together so latching and persistence share one owner. --- src/dev/mod.rs | 1 + src/dev/timekeeper.rs | 701 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 702 insertions(+) create mode 100644 src/dev/timekeeper.rs diff --git a/src/dev/mod.rs b/src/dev/mod.rs index a8392b11e..d8f7d6b72 100644 --- a/src/dev/mod.rs +++ b/src/dev/mod.rs @@ -63,3 +63,4 @@ pub mod z85c30; pub mod dp83843; pub mod ics1890; pub mod isp12160; +pub mod timekeeper; diff --git a/src/dev/timekeeper.rs b/src/dev/timekeeper.rs new file mode 100644 index 000000000..95a2557a5 --- /dev/null +++ b/src/dev/timekeeper.rs @@ -0,0 +1,701 @@ +//! The Fuel M48T35 Timekeeper exposes 32 KiB of byte-wide NVRAM and calendar +//! registers at the end of that window. The guest reads the clock independently +//! of processor byte order, and software can stage clock changes through the +//! control register before committing them. +//! +//! # What is approximated here +//! +//! Host time and a persisted sidecar stand in for the battery-backed clock. +//! The model does not reproduce the physical oscillator or battery behavior. +//! +//! Ported from hw/rtc/m48t35.c in jamesbraid/qemu-sgi-origami. + +use crate::traits::{BusDevice, BusRead8, Device, Resettable, BUS_ERR, BUS_OK}; +use parking_lot::Mutex; +use std::path::{Path, PathBuf}; +use std::sync::Arc; +use std::time::{Instant, SystemTime, UNIX_EPOCH}; + +pub const SIZE: usize = 0x8000; +pub mod reg { + pub const CONTROL: usize = 0x7ff8; + pub const SECONDS: usize = 0x7ff9; + pub const MINUTES: usize = 0x7ffa; + pub const HOURS: usize = 0x7ffb; + pub const CLOCK_WRITE: usize = 0x7ffc; + pub const DAY: usize = 0x7ffd; + pub const MONTH: usize = 0x7ffe; + pub const YEAR: usize = 0x7fff; +} +pub const CONTROL: usize = reg::CONTROL; +const CENTURY: i64 = 36525 * 86400; +const MAGIC: &[u8; 8] = b"M48TREF1"; +fn bcd(v: i64) -> u8 { + ((v / 10) * 16 + v % 10) as u8 +} +fn unbcd(v: u8) -> i64 { + i64::from((v >> 4) * 10 + (v & 15)) +} +fn month_days(y: i64, m: i64) -> i64 { + match m { + 2 => 28 + i64::from(y % 4 == 0), + 4 | 6 | 9 | 11 => 30, + _ => 31, + } +} +#[derive(Clone, Copy)] +struct Calendar { + year: i64, + month: i64, + day: i64, + hour: i64, + minute: i64, + second: i64, +} +impl Calendar { + fn seconds(self) -> i64 { + let year = (self.year + (self.month - 1).div_euclid(12)).rem_euclid(100); + let month = (self.month - 1).rem_euclid(12) + 1; + let mut days = year * 365 + (year + 3) / 4 + self.day - 1; + for m in 1..month { + days += month_days(year, m); + } + (days * 86400 + self.hour * 3600 + self.minute * 60 + self.second).rem_euclid(CENTURY) + } + fn from_seconds(t: i64) -> Self { + let t = t.rem_euclid(CENTURY); + let mut days = t / 86400; + let mut year = 0; + while days >= 365 + i64::from(year % 4 == 0) { + days -= 365 + i64::from(year % 4 == 0); + year += 1; + } + let mut month = 1; + while days >= month_days(year, month) { + days -= month_days(year, month); + month += 1; + } + Self { + year, + month, + day: days + 1, + hour: t / 3600 % 24, + minute: t / 60 % 60, + second: t % 60, + } + } +} +fn saved_calendar(image: &[u8]) -> Option { + if image.len() != SIZE { + return None; + } + let values = [ + image[0x7ff9] & 0x7f, + image[0x7ffa], + image[0x7ffb], + image[0x7ffd], + image[0x7ffe], + image[0x7fff], + ]; + let limits = [(0, 59), (0, 59), (0, 23), (1, 31), (1, 12), (0, 99)]; + let valid = values + .iter() + .zip(limits) + .all(|(&v, (lo, hi))| v & 15 <= 9 && v >> 4 <= 9 && (lo..=hi).contains(&unbcd(v))); + if !valid { + return None; + } + Some(Calendar { + second: unbcd(values[0]), + minute: unbcd(values[1]), + hour: unbcd(values[2]), + day: unbcd(values[3]), + month: unbcd(values[4]), + year: unbcd(values[5]), + }) +} +struct State { + cells: Vec, + offset: i64, + stopped: Option, + frozen: Option, + staged: Option<(Calendar, bool)>, + file: Option, + #[cfg(test)] + extra: i64, +} +impl State { + fn running(&self, host: i64) -> i64 { + self.stopped + .unwrap_or_else(|| (host + self.offset).rem_euclid(CENTURY)) + } + fn calendar(&self, host: i64) -> Calendar { + Calendar::from_seconds(self.frozen.unwrap_or_else(|| self.running(host))) + } +} + +/// Fuel's 1968-based clock. Ordinary storage starts blank; the guest owns it. +/// The lower candidate clock addresses are independent RAM, with no aliases. +pub struct Timekeeper { + initial_epoch: i64, + st: Mutex, + clock: Arc i64 + Send + Sync>, +} +impl Timekeeper { + pub fn fuel() -> Result { + let host = SystemTime::now() + .duration_since(UNIX_EPOCH) + .map_err(|e| e.to_string())? + .as_secs() as i64; + // Gregorian host date, used only to initialize the part's own calendar. + let mut days = host / 86400; + let mut year = 1970; + loop { + let n = 365 + i64::from(year % 4 == 0 && (year % 100 != 0 || year % 400 == 0)); + if days < n { + break; + } + days -= n; + year += 1; + } + let mut month = 1; + while days >= month_days(year, month) { + days -= month_days(year, month); + month += 1; + } + let t = Self::fuel_at( + year, + month, + days + 1, + host / 3600 % 24, + host / 60 % 60, + host % 60, + )?; + Ok(t) + } + /// Open authoritative saved clock state before applying new-store year limits. + pub fn fuel_with_nvram_file(path: &Path) -> Result { + Self::backed(path, Self::fuel()) + } + fn backed(path: &Path, initial: Result) -> Result { + let saved = match std::fs::read(path) { + Ok(v) => v, + Err(e) if e.kind() == std::io::ErrorKind::NotFound => Vec::new(), + Err(e) => return Err(e.to_string()), + }; + let clock = if saved_calendar(&saved).is_some() { + // This seed is replaced by the authoritative image before exposure. + Self::fuel_at(1968, 1, 1, 0, 0, 0)? + } else { + initial? + }; + clock.set_nvram_file(path)?; + Ok(clock) + } + /// A deterministic initial date; subsequent seconds follow monotonic host time. + pub fn fuel_at( + year: i64, + month: i64, + day: i64, + hour: i64, + minute: i64, + second: i64, + ) -> Result { + if !(1968..=2067).contains(&year) { + return Err("Timekeeper: initial year is outside 1968-2067".into()); + } + if !(1..=12).contains(&month) + || !(1..=month_days(year, month)).contains(&day) + || !(0..24).contains(&hour) + || !(0..60).contains(&minute) + || !(0..60).contains(&second) + { + return Err("Timekeeper: invalid initial calendar".into()); + } + let epoch = Calendar { + year: year - 1968, + month, + day, + hour, + minute, + second, + } + .seconds(); + let at = Instant::now(); + Ok(Self { + initial_epoch: epoch, + st: Mutex::new(State { + cells: vec![0; SIZE], + offset: epoch, + stopped: None, + frozen: None, + staged: None, + file: None, + #[cfg(test)] + extra: 0, + }), + clock: Arc::new(move || at.elapsed().as_secs() as i64), + }) + } + fn host(&self, st: &State) -> i64 { + #[cfg(not(test))] + let _ = st; + (self.clock)() + { + #[cfg(test)] + { + st.extra + } + #[cfg(not(test))] + { + 0 + } + } + } + #[cfg(test)] + fn advance_for_test(&self, seconds: i64) { + self.st.lock().extra += seconds; + } + + fn image(st: &State, host: i64, saved_at: i64) -> (Vec, [u8; 16]) { + let c = Calendar::from_seconds(st.running(host)); + let mut image = st.cells.clone(); + image[CONTROL + 1] = bcd(c.second) | if st.stopped.is_some() { 0x80 } else { 0 }; + image[CONTROL + 2] = bcd(c.minute); + image[CONTROL + 3] = bcd(c.hour); + image[CONTROL + 5] = bcd(c.day); + image[CONTROL + 6] = bcd(c.month); + image[CONTROL + 7] = bcd(c.year); + let mut reference = [0; 16]; + reference[..8].copy_from_slice(MAGIC); + reference[8..].copy_from_slice(&saved_at.to_le_bytes()); + (image, reference) + } + /// Export the exact raw image and companion clock reference, without changing RAM. + pub fn persistence_image_at(&self, saved_at: i64) -> (Vec, [u8; 16]) { + let st = self.st.lock(); + Self::image(&st, self.host(&st), saved_at) + } + /// Restore validated storage. Invalid/unwritten clock cells keep the initial clock. + /// A missing/invalid reference adds no elapsed time, matching the model's policy. + pub fn restore_image_at( + &self, + image: &[u8], + reference: Option<&[u8]>, + now: i64, + ) -> Result<(), String> { + if image.len() != SIZE { + return Err(format!( + "Timekeeper: backing image has {} bytes, expected {SIZE}", + image.len() + )); + } + if reference.is_some_and(|r| r.len() < 16) { + return Err("Timekeeper: clock reference is shorter than 16 bytes".into()); + } + let mut st = self.st.lock(); + st.cells.copy_from_slice(image); + st.frozen = None; + st.staged = None; + st.stopped = None; + if let Some(c) = saved_calendar(image) { + let stopped = image[0x7ff9] & 0x80 != 0; + let elapsed = reference.filter(|r| &r[..8] == MAGIC).map_or(0, |r| { + now.saturating_sub(i64::from_le_bytes(r[8..16].try_into().unwrap())) + .max(0) + }); + let epoch = (c.seconds() + + if stopped { + 0 + } else { + elapsed.rem_euclid(CENTURY) + }) + .rem_euclid(CENTURY); + st.offset = epoch - self.host(&st); + if stopped { + st.stopped = Some(epoch); + } + } else { + st.offset = self.initial_epoch; + crate::dlog!(crate::devlog::LogModule::Rtc, "Timekeeper: unwritten or invalid clock cells; following initial clock"); + } + Ok(()) + } + /// Bind a raw store and its `.clock` companion. Validate both before creating either. + pub fn set_nvram_file(&self, path: &Path) -> Result<(), String> { + let clock_path = PathBuf::from(format!("{}.clock", path.display())); + let image = match std::fs::read(path) { + Ok(v) => Some(v), + Err(e) if e.kind() == std::io::ErrorKind::NotFound => None, + Err(e) => return Err(e.to_string()), + }; + if image.as_ref().is_some_and(|v| v.len() != SIZE) { + return Err("Timekeeper: backing image must contain exactly 32768 bytes".into()); + } + let reference = match std::fs::read(&clock_path) { + Ok(v) => Some(v), + Err(e) if e.kind() == std::io::ErrorKind::NotFound => None, + Err(e) => return Err(e.to_string()), + }; + if reference.as_ref().is_some_and(|v| v.len() < 16) { + return Err("Timekeeper: clock reference is shorter than 16 bytes".into()); + } + if let Some(image) = image.as_ref() { + self.restore_image_at(image, reference.as_deref(), unix_now())?; + } + use std::io::Write; + if image.is_none() { + std::fs::OpenOptions::new() + .write(true) + .create_new(true) + .open(path) + .and_then(|mut f| f.write_all(&vec![0; SIZE])) + .map_err(|e| e.to_string())?; + } + if reference.is_none() { + std::fs::OpenOptions::new() + .write(true) + .create_new(true) + .open(clock_path) + .and_then(|mut f| f.write_all(&[0; 16])) + .map_err(|e| e.to_string())?; + } + self.st.lock().file = Some(path.to_path_buf()); + Ok(()) + } + pub fn flush_nvram(&self) -> Result<(), String> { + let st = self.st.lock(); + if let Some(path) = &st.file { + let (image, reference) = Self::image(&st, self.host(&st), unix_now()); + std::fs::write(path, image).map_err(|e| e.to_string())?; + use std::io::Write; + std::fs::OpenOptions::new() + .write(true) + .open(format!("{}.clock", path.display())) + .and_then(|mut f| f.write_all(&reference)) + .map_err(|e| e.to_string())?; + } + Ok(()) + } +} +fn unix_now() -> i64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .map_or(0, |d| d.as_secs() as i64) +} +impl BusDevice for Timekeeper { + fn read8(&self, addr: u64) -> BusRead8 { + let Ok(a) = usize::try_from(addr) else { + return BusRead8::err(); + }; + if a >= SIZE { + return BusRead8::err(); + } + let st = self.st.lock(); + let c = st.calendar(self.host(&st)); + let data = match a { + reg::SECONDS => bcd(c.second) | if st.stopped.is_some() { 0x80 } else { 0 }, + reg::MINUTES => bcd(c.minute), + reg::HOURS => bcd(c.hour), + reg::DAY => bcd(c.day), + reg::MONTH => bcd(c.month), + reg::YEAR => bcd(c.year), + _ => st.cells[a], + }; + BusRead8 { + status: BUS_OK, + data, + } + } + fn write8(&self, addr: u64, val: u8) -> u32 { + let Ok(a) = usize::try_from(addr) else { + return BUS_ERR; + }; + if a >= SIZE { + return BUS_ERR; + } + let mut st = self.st.lock(); + let host = self.host(&st); + let old_control = st.cells[CONTROL]; + let transfer = a == CONTROL && st.cells[CONTROL] & 0x80 != 0 && val & 0x80 == 0; + if a == CONTROL { + let old = st.cells[a]; + if val & 0x40 != 0 && old & 0x40 == 0 { + st.frozen = Some(st.running(host)); + } else if val & 0x40 == 0 { + st.frozen = None; + } + if val & 0x80 != 0 && old & 0x80 == 0 { + st.staged = Some((st.calendar(host), st.stopped.is_some())); + } else if val & 0x80 == 0 && old & 0x80 != 0 { + if let Some((c, stop)) = st.staged.take() { + let t = c.seconds(); + st.offset = t - host; + st.stopped = if stop { Some(t) } else { None }; + } + } + st.cells[a] = val; + } else if a > CONTROL { + if a == reg::CLOCK_WRITE && st.staged.is_some() { + st.cells[a] = val; + } else if let Some((c, stop)) = &mut st.staged { + match a { + reg::SECONDS => { + c.second = unbcd(val & 0x7f); + *stop = val & 0x80 != 0; + } + reg::MINUTES => c.minute = unbcd(val), + reg::HOURS => c.hour = unbcd(val), + reg::DAY => c.day = unbcd(val), + reg::MONTH => c.month = unbcd(val), + reg::YEAR => c.year = unbcd(val), + _ => {} + } + } else { + crate::dlog!(crate::devlog::LogModule::Rtc, "Timekeeper: clock write at {a:#x} without write enable"); + } + } else { + st.cells[a] = val; + } + if !transfer + && (a < CONTROL + || a == CONTROL && (old_control ^ val) & !0xc0 != 0 + || a == reg::CLOCK_WRITE && st.staged.is_some()) + { + // Ordinary storage writes must not save or replace the clock cells. + if let Some(path) = &st.file { + use std::io::{Seek, SeekFrom, Write}; + let result = + std::fs::OpenOptions::new() + .write(true) + .open(path) + .and_then(|mut f| { + f.seek(SeekFrom::Start(a as u64))?; + f.write_all(&[st.cells[a]]) + }); + if let Err(e) = result { + log::error!("Timekeeper: backing write failed: {e}"); + } + } + } + drop(st); + if transfer { + if let Err(e) = self.flush_nvram() { + log::error!("Timekeeper: clock save failed: {e}"); + } + } + BUS_OK + } +} +impl Resettable for Timekeeper { + fn power_on(&self) { + let mut st = self.st.lock(); + st.frozen = None; + st.staged = None; + st.cells[CONTROL] &= !0xc0; + } +} +impl Device for Timekeeper { + fn start(&self) {} + fn step(&self, _cycles: u64) {} + fn is_running(&self) -> bool { + false + } + fn get_clock(&self) -> u64 { + 0 + } + fn stop(&self) { + if let Err(e) = self.flush_nvram() { + log::error!("Timekeeper: saving failed: {e}"); + } + } +} +impl Drop for Timekeeper { + fn drop(&mut self) { + Device::stop(self); + } +} +#[cfg(test)] +mod tests { + // Cases ported from tests/qtest/sn1-fuel-test.c in jamesbraid/qemu-sgi-origami. + use super::*; + use crate::traits::{BusDevice, Resettable}; + + /// The guest can freeze, stop, stage, and resume the M48T35 calendar. + #[test] + fn clock_control_freezes_stops_and_resets_the_calendar() { + let t = Timekeeper::fuel_at(2067, 12, 31, 23, 59, 59).unwrap(); + t.write8(0x7f8, 0x42); + t.write8(CONTROL as u64, 0x40); + t.advance_for_test(2); + assert_eq!(t.read8(reg::YEAR as u64).data, 0x99); + t.power_on(); + assert_eq!(t.read8(CONTROL as u64).data, 0); + assert_eq!(t.read8(reg::YEAR as u64).data, 0); + assert_eq!(t.read8(reg::MONTH as u64).data, 1); + assert_eq!(t.read8(reg::DAY as u64).data, 1); + assert_eq!(t.read8(0x7f8).data, 0x42); + t.write8(CONTROL as u64, 0x80); + t.write8(reg::SECONDS as u64, 0x89); + t.write8(CONTROL as u64, 0); + t.advance_for_test(60); + assert_eq!(t.read8(reg::SECONDS as u64).data, 0x89); + } + + /// A restored guest clock advances by host elapsed time unless its stop bit was saved. + #[test] + fn restored_clock_accounts_for_elapsed_time_unless_stopped() { + for stopped in [false, true] { + let t = Timekeeper::fuel_at(2024, 2, 28, 23, 59, 59).unwrap(); + t.write8(0x7ff0, 0xa5); + if stopped { + t.write8(CONTROL as u64, 0x80); + t.write8(reg::SECONDS as u64, 0x85); + t.write8(CONTROL as u64, 0); + } + let (image, reference) = t.persistence_image_at(100); + let restored = Timekeeper::fuel_at(2026, 1, 1, 0, 0, 0).unwrap(); + restored.restore_image_at(&image, Some(&reference), 102).unwrap(); + assert_eq!(restored.read8(reg::DAY as u64).data, if stopped { 0x28 } else { 0x29 }); + assert_eq!(restored.read8(reg::SECONDS as u64).data, if stopped { 0x85 } else { 1 }); + assert_eq!(restored.read8(0x7ff0).data, 0xa5); + assert!(restored.restore_image_at(&image[..2048], None, 102).is_err()); + } + } + + /// NVRAM persists across emulator restarts, and malformed backing images are rejected. + #[test] + fn backed_nvram_preserves_saved_state_and_rejects_invalid_images() { + let dir = std::env::current_exe().unwrap().parent().unwrap() + .join(format!("timekeeper-test-{}", std::process::id())); + std::fs::create_dir(&dir).unwrap(); + let path = dir.join("nvram"); + let t = Timekeeper::fuel_at(2026, 1, 1, 0, 0, 0).unwrap(); + t.set_nvram_file(&path).unwrap(); + assert_eq!(std::fs::read(&path).unwrap(), vec![0; SIZE]); + t.write8(0x7f8, 0x42); + assert_eq!(std::fs::read(&path).unwrap()[0x7f8], 0x42); + t.flush_nvram().unwrap(); + drop(t); + // Saved clock state is authoritative even if the host date is out of range. + let restored = Timekeeper::backed(&path, Timekeeper::fuel_at(2068, 1, 1, 0, 0, 0)).unwrap(); + restored.power_on(); + assert_eq!(restored.read8(0x7f8).data, 0x42); + assert_eq!(restored.read8(reg::YEAR as u64).data, 0x58); + drop(restored); + std::fs::remove_file(dir.join("nvram.clock")).unwrap(); + std::fs::write(&path, [0xa5; 2048]).unwrap(); + assert!(Timekeeper::backed(&path, Timekeeper::fuel_at(2026, 1, 1, 0, 0, 0)).is_err()); + assert_eq!(std::fs::read(&path).unwrap(), vec![0xa5; 2048]); + assert!(!dir.join("nvram.clock").exists()); + std::fs::remove_file(&path).unwrap(); + assert!(Timekeeper::backed(&path, Timekeeper::fuel_at(2068, 1, 1, 0, 0, 0)).is_err()); + assert!(!path.exists()); + std::fs::remove_dir(dir).unwrap(); + } + + const WEEKDAY: u64 = reg::CLOCK_WRITE as u64; + + fn clock_cells(t: &Timekeeper) -> [u8; 7] { + std::array::from_fn(|i| t.read8(reg::SECONDS as u64 + i as u64).data) + } + + fn set_clock(t: &Timekeeper, cells: [u8; 7]) { + t.write8(CONTROL as u64, 0x80); + for (i, v) in cells.into_iter().enumerate() { + t.write8(reg::SECONDS as u64 + i as u64, v); + } + t.write8(CONTROL as u64, 0); + } + + // Cells are seconds, minutes, hours, weekday, day, month, year, all BCD. + + /// The PROM and kernel derive file timestamps from this calendar, so + /// 2024 must have a 29 February and 2026 must not. + #[test] + fn the_calendar_rolls_over_february_in_leap_years_only() { + let t = Timekeeper::fuel_at(2026, 1, 1, 0, 0, 0).unwrap(); + set_clock(&t, [0x59, 0x59, 0x23, 0x05, 0x28, 0x02, 0x56]); + t.advance_for_test(1); + assert_eq!(clock_cells(&t), [0, 0, 0, 0x05, 0x29, 0x02, 0x56]); + set_clock(&t, [0x59, 0x59, 0x23, 0x05, 0x28, 0x02, 0x58]); + t.advance_for_test(1); + assert_eq!(clock_cells(&t), [0, 0, 0, 0x05, 0x01, 0x03, 0x58]); + } + + /// The part counts a two-digit year from 1968, so the calendar wraps after + /// 2067; the guest must see year 00 again, not garbage. + #[test] + fn the_year_wraps_from_2067_to_1968() { + let t = Timekeeper::fuel_at(2026, 1, 1, 0, 0, 0).unwrap(); + set_clock(&t, [0x59, 0x59, 0x23, 0x05, 0x31, 0x12, 0x99]); + assert_eq!(t.read8(reg::YEAR as u64).data, 0x99); + t.advance_for_test(1); + assert_eq!(clock_cells(&t), [0, 0, 0, 0x05, 0x01, 0x01, 0]); + } + + /// A guest reading the clock one byte at a time needs the read bit to + /// freeze every counter at once, or a rollover tears the timestamp. + #[test] + fn the_read_bit_freezes_all_counters_until_it_is_cleared() { + let t = Timekeeper::fuel_at(2026, 1, 1, 0, 0, 0).unwrap(); + let end_of_century = [0x59, 0x59, 0x23, 0x05, 0x31, 0x12, 0x99]; + set_clock(&t, end_of_century); + t.write8(CONTROL as u64, 0x40); + assert_eq!(t.read8(CONTROL as u64).data, 0x40); + t.advance_for_test(2); + assert_eq!(clock_cells(&t), end_of_century); + t.write8(CONTROL as u64, 0); + assert_eq!(clock_cells(&t), [0x01, 0, 0, 0x05, 0x01, 0x01, 0]); + } + + /// Writes in the write-enable window are staged: the guest sets six + /// counters and the weekday, and nothing but the weekday is visible until + /// the window closes. + #[test] + fn staged_clock_writes_apply_only_when_the_write_bit_clears() { + let t = Timekeeper::fuel_at(2026, 1, 2, 3, 4, 5).unwrap(); + let before = clock_cells(&t); + t.write8(CONTROL as u64, 0x80); + for (i, v) in [0x59u8, 0x58, 0x17, 0x03, 0x15, 0x06, 0x30].into_iter().enumerate() { + t.write8(reg::SECONDS as u64 + i as u64, v); + } + let during = clock_cells(&t); + for i in (0..7).filter(|&i| i != 3) { + assert_eq!(during[i], before[i], "cell {i}"); + } + assert_eq!(t.read8(WEEKDAY).data, 0x03); + t.write8(CONTROL as u64, 0); + assert_eq!(clock_cells(&t), [0x59, 0x58, 0x17, 0x03, 0x15, 0x06, 0x30]); + } + + /// The stop bit is how a guest parks the oscillator while it sets the time; + /// the clock must hold until the guest clears it, then continue from there. + #[test] + fn the_stop_bit_holds_the_seconds_until_it_is_cleared() { + let t = Timekeeper::fuel_at(2026, 1, 1, 0, 0, 0).unwrap(); + t.write8(CONTROL as u64, 0x80); + t.write8(reg::SECONDS as u64, 0x82); + t.write8(CONTROL as u64, 0); + t.advance_for_test(3); + assert_eq!(t.read8(reg::SECONDS as u64).data, 0x82); + t.write8(CONTROL as u64, 0x80); + t.write8(reg::SECONDS as u64, 0x02); + t.write8(CONTROL as u64, 0); + t.advance_for_test(1); + assert_eq!(t.read8(reg::SECONDS as u64).data, 0x03); + } + + /// The 32 KiB window ends at the year register; an access past it must + /// fault, and ordinary storage must keep its contents across clock writes. + #[test] + fn accesses_past_the_window_fault_and_ordinary_storage_is_untouched() { + let t = Timekeeper::fuel_at(2026, 1, 1, 0, 0, 0).unwrap(); + t.write8(0, 0x5a); + t.write8(0x7ff7, 0xa5); + assert_eq!(t.write8(SIZE as u64, 0x69), BUS_ERR); + assert_ne!(t.read8(SIZE as u64).status, BUS_OK); + set_clock(&t, [0, 0, 0, 0x05, 0x01, 0x03, 0x58]); + assert_eq!(t.read8(0).data, 0x5a); + assert_eq!(t.read8(0x7ff7).data, 0xa5); + } +} From 4b81837ab2048f33bde646e7beba487c6a95d916 Mon Sep 17 00:00:00 2001 From: James Braid Date: Fri, 30 Oct 2026 08:13:36 +0000 Subject: [PATCH 16/31] ioc3: add the Fuel peripheral population Fuel attaches an M48T35 and has different PCI header policy from the Octane. Carry those choices in the population configuration and wire the timekeeper once so byte-bus accesses need no mutable attachment slot. The Fuel's onboard IOC3 also carries the PS/2 keyboard and mouse controller: OpenBSD configures a keyboard controller through the onboard IOC3 (pci/ioc.c, dev/iockbc.c), and the controller core is the one the Octane already has. Turn the controller on in the Fuel's population, as the Octane's is, and test it by sending a host key through the IOC3 registers: it latches the keyboard interrupt and arrives in the K_RX register, and a mouse command answers from its own port, both before and after a chip reset. --- src/dev/ioc3.rs | 97 ++++++++++++++++++++++++++++++++++++------- src/dev/ioc3_tests.rs | 79 +++++++++++++++++++++++++++++++++++ 2 files changed, 162 insertions(+), 14 deletions(-) diff --git a/src/dev/ioc3.rs b/src/dev/ioc3.rs index f82bbbde5..a17803a9b 100644 --- a/src/dev/ioc3.rs +++ b/src/dev/ioc3.rs @@ -1,10 +1,11 @@ -//! SGI IOC3 base I/O controller, as the Octane (IP30) fits it. +//! SGI IOC3 base I/O controller, with Octane and Fuel board populations. //! //! One PCI function with one 1 MiB memory BAR. Behind it: two 16550 serial //! ports with a ring-DMA engine each, a PS/2 keyboard and mouse controller, //! a parallel port, a MicroLAN (one-wire) master, the GPIO pins, a DS1685 -//! clock on the byte bus, the Ethernet buffer RAM and, in the register range -//! at 0xf0, the Ethernet MAC (which lives in `ioc3_eth.rs` behind +//! clock on the Octane byte bus or Fuel's flat system Timekeeper, the +//! Ethernet buffer RAM and, in the register range at 0xf0, +//! the Ethernet MAC (which lives in `ioc3_eth.rs` behind //! [`Ioc3EthPort`]). //! //! Behaviour follows the public Linux and OpenBSD drivers and measured guest @@ -60,7 +61,7 @@ use crate::dev::w1::sgi::{self, BoardRecord}; use crate::dev::ns16550::Ns16550; use crate::dev::serial_host::SerialBackend; use crate::irq::{InterruptController, IrqLine}; -use crate::pci::{Bar, PciBusDevice, PciConfig, PciDma, PciDmaStatus, PCI_INTA}; +use crate::pci::{Bar, PciBusDevice, PciConfig, PciConfigPolicy, PciDma, PciDmaStatus, PCI_INTA}; use crate::hptimer::{TimerId, TimerManager, TimerReturn}; use crate::traits::{BusDevice, Device, MachineEvent, Resettable, Saveable}; use crate::snapshot::{ @@ -72,6 +73,8 @@ pub const DEVICE_ID: u16 = 0x0003; pub const REVISION: u8 = 0x01; /// PCI subsystem id of the Octane's system board. pub const SUBSYS_IP30_SYSBOARD: u16 = 0xc304; +/// Fuel system-board population. +pub const SUBSYS_IP34_SYSBOARD: u16 = 0xc30a; pub const BAR_SIZE: u32 = 1 << 20; /// The configuration dwords the chip decodes; the rest of the 16 repeat them. const CONFIG_DECODED_DWORDS: usize = 8; @@ -113,6 +116,7 @@ pub mod reg { pub const PAR_ECP_BASE: u32 = 0x20150; pub const SSRAM: u32 = 0x40000; pub const SSRAM_SIZE: u32 = 0x40000; + pub const BYTEBUS_DEV0: u32 = 0x80000; pub const BYTEBUS_DEV1: u32 = 0xa0000; pub const BYTEBUS_DEV2: u32 = 0xc0000; pub const BYTEBUS_SIZE: u32 = 0x20000; @@ -734,6 +738,10 @@ fn int_out_valid(v: u32) -> bool { #[derive(Clone, Debug)] pub struct Ioc3Config { pub subsystem_id: u16, + /// Function-specific PCI header write behavior. + pub pci_policy: PciConfigPolicy<'static>, + /// Reset the PCI header along with the chip on power-on. + pub reset_pci_on_power_on: bool, /// The board's station address, which the one-wire part holds. pub mac: [u8; 6], pub serial: [bool; 2], @@ -763,6 +771,30 @@ impl BoardRecordText { } impl Ioc3Config { + /// Fuel's IOC3 fits the keyboard and mouse controller but no indexed Dallas clock. + pub fn fuel(mac: [u8; 6]) -> Self { + Self { + subsystem_id: SUBSYS_IP34_SYSBOARD, + pci_policy: PciConfigPolicy { + status: 0x0280, + status_w1c_mask: 0, + command_mask: 0xffff, + latency_mask: 0xff, + cache_line_mask: 0, + ..Default::default() + }, + reset_pci_on_power_on: true, + mac, + serial: [true, true], + parallel: true, + keyboard_mouse: true, + rtc: false, + gpio_input_mask: 0, + gpio_input_values: 0, + records: Vec::new(), + } + } + /// The Octane's IOC3 (`octane2` selects the Octane2's board records). /// The fictitious barcodes and ROM serials are the /// marked-synthetic ones (barcode `EMU...`). @@ -774,6 +806,8 @@ impl Ioc3Config { }; Self { subsystem_id: SUBSYS_IP30_SYSBOARD, + pci_policy: PciConfigPolicy::default(), + reset_pci_on_power_on: false, mac, serial: [true, true], parallel: true, @@ -2382,6 +2416,7 @@ impl Shared { /// The IOC3 PCI function. pub struct Ioc3 { sh: Arc, + timekeeper: OnceLock>, /// The serial receive threads and the NVRAM writer, joined in `stop`. threads: Mutex>>, } @@ -2389,22 +2424,26 @@ pub struct Ioc3 { impl Ioc3 { pub fn new(cfg: Ioc3Config) -> Ioc3 { let subsystem = cfg.subsystem_id; + let pci_policy = cfg.pci_policy; let sh = Arc::new_cyclic(|weak: &Weak| { let line = IrqLine::new(Arc::new(KbcSink(weak.clone())), 0); let ps2 = Arc::new(Ps2Controller::new(Some(line))); // Host key and mouse events are accepted from the start (this // only sets a flag); `stop` and `start` follow the chip's. Device::start(&*ps2); - let pci = PciConfig::new( - VENDOR_ID, - DEVICE_ID, - 0xff_0000, - REVISION, - [Bar::Mem(BAR_SIZE), Bar::Unused, Bar::Unused, Bar::Unused, Bar::Unused, Bar::Unused], - PCI_INTA, + let bars = [ + Bar::Mem(BAR_SIZE), + Bar::Unused, + Bar::Unused, + Bar::Unused, + Bar::Unused, + Bar::Unused, + ]; + let pci = PciConfig::new_with_policy( + VENDOR_ID, DEVICE_ID, 0xff_0000, REVISION, bars, PCI_INTA, pci_policy, ); // The subsystem identity selects what a guest instantiates. - pci.write(11, ((subsystem as u32) << 16) | VENDOR_ID as u32); + pci.set_subsystem_identity(((subsystem as u32) << 16) | VENDOR_ID as u32); Shared { pci, st: Mutex::new(St::new(cfg)), @@ -2429,7 +2468,7 @@ impl Ioc3 { nvram_ready: Condvar::new(), } }); - Ioc3 { sh, threads: Mutex::new(Vec::new()) } + Ioc3 { sh, timekeeper: OnceLock::new(), threads: Mutex::new(Vec::new()) } } /// The Octane's (or Octane2's) onboard IOC3: both serial ports, PS/2, @@ -2438,7 +2477,13 @@ impl Ioc3 { Ioc3::new(Ioc3Config::octane(mac, octane2)) } - /// The timer manager the deadlines are armed on. Call before `start`. + /// Attach Fuel's independently owned flat select-0 system clock. + pub fn attach_timekeeper(&self, clock: Arc) { + assert!(!self.sh.st.lock().cfg.rtc, "indexed Dallas and flat Timekeeper populations conflict"); + assert!(self.timekeeper.set(clock).is_ok(), "IOC3 timekeeper attached more than once"); + } + + /// The timer manager for device deadlines. Set it before start. pub fn set_timer_manager(&self, tm: Arc) { let _ = self.sh.timers.set(tm); } @@ -2545,6 +2590,9 @@ impl Ioc3 { /// Ethernet block is a separate device here and keeps its state: the /// machine resets it on its own. pub fn reset(&self) { + if self.sh.st.lock().cfg.reset_pci_on_power_on { + self.sh.pci.reset(); + } self.sh.with_st(false, |st, _| st.reset()); } @@ -2703,6 +2751,16 @@ impl PciBusDevice for Ioc3 { if width == 4 && a <= reg::HEADER_LAST && a & 3 == 0 { return self.header_dword((a / 4) as usize); } + if (reg::BYTEBUS_DEV0..reg::BYTEBUS_DEV0 + crate::dev::timekeeper::SIZE as u32).contains(&a) { + if let Some(clock) = self.timekeeper.get() { + if width != 1 { + crate::dlog!(crate::devlog::LogModule::Ioc, "IOC3 Timekeeper {width}-byte read refused"); + return 0; + } + return u32::from(clock.read8(u64::from(a - reg::BYTEBUS_DEV0)).data); + } + } + if (reg::ENET_FIRST..=reg::ENET_LAST).contains(&a) { // Only 32-bit accesses reach the Ethernet block. if width != 4 { @@ -2719,6 +2777,17 @@ impl PciBusDevice for Ioc3 { return; } let a = off ^ (4 - width.min(4) as u32); + if (reg::BYTEBUS_DEV0..reg::BYTEBUS_DEV0 + crate::dev::timekeeper::SIZE as u32).contains(&a) { + if let Some(clock) = self.timekeeper.get() { + if width != 1 { + crate::dlog!(crate::devlog::LogModule::Ioc, "IOC3 Timekeeper {width}-byte write refused"); + return; + } + clock.write8(u64::from(a - reg::BYTEBUS_DEV0), val as u8); + return; + } + } + if (reg::ENET_FIRST..=reg::ENET_LAST).contains(&a) { if width == 4 && a & 3 == 0 { if let Some(e) = self.sh.eth() { diff --git a/src/dev/ioc3_tests.rs b/src/dev/ioc3_tests.rs index ec7f1bfe5..825907170 100644 --- a/src/dev/ioc3_tests.rs +++ b/src/dev/ioc3_tests.rs @@ -1636,3 +1636,82 @@ fn a_short_buffer_ram_is_refused() { chip.insert("ssram".into(), toml::Value::Array(Vec::new())); assert!(c.load_state(&v).unwrap_err().contains("ssram")); } + +/// The Octane keeps the IOC3's PCI header across a chip reset; only the +/// Fuel population resets it along with the chip. +#[test] +fn an_octane_chip_reset_keeps_the_pci_header() { + let r = Rig::new(); + let cfg = r.c.config(); + cfg.write(1, 0xffff_0146); + cfg.write(3, 0xffff_ffff); + r.c.reset(); + assert_eq!(cfg.read(1), 0x0200_0146); + assert_eq!(cfg.read(3), 0xffff); +} + +/// Fuel's IOC3 has its own header write policy and resets the header with +/// the chip, reaches the flat Timekeeper on byte-bus device 0, leaves the +/// Dallas clock ports unfitted, and has no PHY reset on GPIO 5. +#[test] +fn the_fuel_population_keeps_its_clock_and_header_policy_across_reset() { + let r = Rig::with(Ioc3::new(Ioc3Config::fuel(MAC))); + let clock = Arc::new(crate::dev::timekeeper::Timekeeper::fuel_at(2026, 1, 1, 0, 0, 0).unwrap()); + r.c.attach_timekeeper(clock.clone()); + let cfg = r.c.config(); + cfg.write(1, 0xffff_0146); + cfg.write(3, 0xffff_ffff); + assert_eq!(cfg.read(1), 0x0280_0146); + assert_eq!(cfg.read(3), 0x0000_ff00); + let base = reg::BYTEBUS_DEV0; + r.w8(base + 0x7f8, 0xa5); + r.c.reset(); + clock.power_on(); + assert_eq!(r.r8(base + 0x7fff), 0x58); + assert_eq!(r.r8(base + 0x7f8), 0xa5); + assert_eq!(cfg.read(1), 0x0280_0000); + assert_eq!(cfg.read(3), 0); + // Fuel selects the flat clock, leaving indexed Dallas ports unfitted. + r.w8(reg::BYTEBUS_DEV1, 0x40); + r.w8(reg::BYTEBUS_DEV2, 0x55); + assert_eq!(r.r8(reg::BYTEBUS_DEV2), 0); + r.c.attach_eth(crate::dev::ioc3_eth::Ioc3Eth::new_fuel(MAC)); + use crate::dev::ioc3_eth::{bits::MII_READ, reg::{MII_CONTROL, MII_DATA_R, MII_DATA_W}}; + r.w32(MII_DATA_W, 7); + r.w32(MII_CONTROL, (1 << 5) | 0x19); + r.w32(reg::PIN_CONTROL_SET, 1 << 5); + r.w32(reg::PIN_REG_FIRST + 5 * 4, 0); + r.w32(reg::PIN_DATA, 1 << 5); + r.w32(MII_CONTROL, MII_READ | (7 << 5) | 3); + assert_eq!(r.r32(MII_DATA_R), 0x5c10, "Fuel GPIO5 has no PHY reset connection"); +} + +/// Fuel's IOC3 fits the keyboard and mouse controller, as the Octane's does +/// (OpenBSD configures it through the onboard IOC3's keyboard controller). A +/// key pushed from the host arrives in K_RX through the IOC3 registers and +/// latches the keyboard interrupt, and a mouse report arrives in M_RX; both +/// survive a chip reset. +#[test] +fn the_fuel_ioc3_delivers_keyboard_and_mouse_traffic_across_reset() { + let r = Rig::with(Ioc3::new(Ioc3Config::fuel(MAC))); + for _ in 0..2 { + r.c.reset(); + r.c.ps2().power_on(); + r.w32(K_TX, 0xff); + let v = r.r32(K_RX); + assert_ne!(v & VALID[2], 0, "the keyboard answers a reset"); + assert_eq!((v >> 16) & 0xff, 0xfa, "with an ACK"); + r.w32(M_TX, 0xf2); + let v = r.r32(M_RX); + assert_ne!(v & (1 << 27), 0, "the mouse answers from its own port"); + // Enable scanning, then inject a key from the host. + r.w32(K_TX, 0xf4); + while r.r32(K_RX) & VALID[2] != 0 {} + r.w32(reg::INT_STATUS, int::KEYBOARD); + r.c.ps2().push_kb(winit::keyboard::KeyCode::KeyA, true); + assert_ne!(r.r32(reg::INT_STATUS) & int::KEYBOARD, 0, "a key latches the interrupt"); + let v = r.r32(K_RX); + assert_ne!(v & VALID[2], 0, "and arrives in K_RX"); + assert_eq!(v & (1 << 27), 0, "from the keyboard"); + } +} From fe1ce5870a40f97ad62d46c6c96476b38566402e Mon Sep 17 00:00:00 2001 From: James Braid Date: Fri, 30 Oct 2026 08:13:37 +0000 Subject: [PATCH 17/31] sflash: add the AMD command set Fuel boot flash uses AMD unlock, program and erase commands. Keep that state machine separate from array storage and bus decode so its command handling can serve each fitted flash array. The Intel and AMD command sets need the same rules for seeding and writing flash images, so share the file storage while each command engine keeps its own command state. --- HACKING.md | 3 + debug.md | 3 +- src/dev/sflash.rs | 39 ++--- src/dev/sflash/amd.rs | 313 ++++++++++++++++++++++++++++++++++++++ src/dev/sflash/storage.rs | 62 ++++++++ src/devlog.rs | 8 +- 6 files changed, 400 insertions(+), 28 deletions(-) create mode 100644 src/dev/sflash/amd.rs create mode 100644 src/dev/sflash/storage.rs diff --git a/HACKING.md b/HACKING.md index b271ee97d..9028bb6f4 100644 --- a/HACKING.md +++ b/HACKING.md @@ -104,6 +104,9 @@ their earlier behaviour of not reporting resets to targets. The ISP12160 releases its controller state lock before walking either bus. Target callbacks run under that bus lock and must not reacquire it. +Flash command engines share `FlashBacking` file storage while retaining +their own command state. + ## 2. Data path and endianness **Word-transparent architecture.** Host `u32`/`u64` values are bit diff --git a/debug.md b/debug.md index 3f2dfb5fd..5d795820d 100644 --- a/debug.md +++ b/debug.md @@ -541,9 +541,10 @@ Per-module logging; all of it needs a developer build to produce output. | `log file ` | Send a module's log to a file | Modules: `net hpc3 seeq hal2 mc rex3 mips ioc scsi pdma vino dcb vc2 cmap -xmap bt445 scc ps2 rtc eeprom l1i l1d l2c`. Mask categories: `pdma` — +xmap bt445 scc ps2 rtc eeprom l1i l1d l2c bedrock`. Mask categories: `pdma` — `hal enet scsi`; `mips` — `insn tlb mem fpu cp0`; `l1i`/`l1d`/`l2c` — `hit miss op`; every module also takes `on`/`all`, `off`/`none` or a hex mask. +`bedrock` covers the Fuel's devices on its Bedrock hub. `IRIS_DEBUG_LOG` or `[debug] debug_log` set the same thing at startup. ### Performance and locks diff --git a/src/dev/sflash.rs b/src/dev/sflash.rs index 6d0e75ea6..5d35bfe91 100644 --- a/src/dev/sflash.rs +++ b/src/dev/sflash.rs @@ -30,8 +30,7 @@ //! [`Bridge::attach_flash`]: crate::dev::bridge::Bridge::attach_flash use std::cell::Cell; -use std::fs::{File, OpenOptions}; -use std::io::{self, Read, Seek, SeekFrom, Write}; +use std::io; use std::path::Path; use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::Arc; @@ -40,6 +39,10 @@ use parking_lot::Mutex; use crate::snapshot::{field_u8, StateTable}; use crate::traits::{BusDevice, BusRead16, Saveable, BUS_ERR, BUS_OK}; +pub(crate) mod storage; +pub(crate) mod amd; +use self::storage::FlashBacking; + /// The Octane's part: 1 MiB. pub const FLASH_SIZE: usize = 1 << 20; /// Erase block size. @@ -78,7 +81,7 @@ struct State { data: Vec, mode: Mode, status: u8, - file: Option, + backing: Option, } pub struct CfiFlash { @@ -114,7 +117,7 @@ impl CfiFlash { assert!(image.len() <= size, "image must fit the part"); let mut data = vec![0xff; size]; data[..image.len()].copy_from_slice(image); - Self { st: Mutex::new(State { data, mode: Mode::Array, status: SR_READY, file: None }), gen: AtomicU64::new(0) } + Self { st: Mutex::new(State { data, mode: Mode::Array, status: SR_READY, backing: None }), gen: AtomicU64::new(0) } } /// A reset: back to read-array mode with the status clear. The array @@ -132,23 +135,10 @@ impl CfiFlash { /// untouched, since it is probably some other file. pub fn with_file(image: &[u8], size: usize, path: impl AsRef) -> io::Result { let flash = Self::new(image, size); - let path = path.as_ref(); - let mut file = OpenOptions::new().read(true).write(true).create(true).truncate(false).open(path)?; - { - let mut st = flash.st.lock(); - let len = file.metadata()?.len(); - if len == size as u64 { - file.read_exact(&mut st.data)?; - } else if len == 0 { - file.write_all(&st.data)?; - } else { - return Err(io::Error::new( - io::ErrorKind::InvalidData, - format!("flash file {} is {len} bytes, expected {size} (or empty); not overwriting it", path.display()), - )); - } - st.file = Some(file); - } + let mut st = flash.st.lock(); + let backing = FlashBacking::open(path, &mut st.data)?; + st.backing = Some(backing); + drop(st); Ok(flash) } @@ -168,10 +158,9 @@ impl CfiFlash { } impl State { - fn sync(&self, off: usize, len: usize) { - if let Some(mut f) = self.file.as_ref() { - // `&File` seeks and writes without needing `&mut self`. - let res = f.seek(SeekFrom::Start(off as u64)).and_then(|_| f.write_all(&self.data[off..off + len])); + fn sync(&mut self, off: usize, len: usize) { + if let Some(backing) = self.backing.as_mut() { + let res = backing.write_range(off, &self.data[off..off + len]); if let Err(e) = res { // The guest sees a completed cycle; the file is only the backing store. log::warn!("flash: backing file write at {off:#x} failed: {e}"); diff --git a/src/dev/sflash/amd.rs b/src/dev/sflash/amd.rs new file mode 100644 index 000000000..41c4cdff9 --- /dev/null +++ b/src/dev/sflash/amd.rs @@ -0,0 +1,313 @@ +//! The AMD flash command set: unlock, word program, sector erase and CFI +//! query state, over storage supplied by the caller. +//! +//! Array bytes and bus translation belong to the enclosing device. Fuel uses +//! this 8 MiB command engine for each boot-flash array. The CFI data and timing +//! are compatibility values; they do not identify the fitted flash chip. +//! Buffered writes, protection, bypass and erase suspend are absent. +//! +//! Ported from MAME's intelfsh.cpp AMD command path, revision +//! 2ee9b1e30299b122b131cab0bd7912abc2b095ba (BSD-3-Clause). + +// Copyright (c) Aaron Giles + +pub const SECTOR_SIZE: usize = 64 * 1024; +const SECTORS: usize = 128; +pub const PART_SIZE: usize = SECTOR_SIZE * SECTORS; +pub const ERASE_SETUP_NS: u64 = 50_000; +pub const ERASE_NS: u64 = 512_000_000; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum Mode { + Array, + Unlock1, + Unlock2, + Program, + Erase1, + Erase2, + Erase3, + Query, + Identify, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Effect { + None, + Program { offset: usize, value: u16 }, +} + +pub struct AmdCore { + mode: Mode, + selected: [bool; SECTORS], + /// Remaining time includes the sector-selection interval. + remaining_ns: u64, + setup_ns: u64, + toggle: bool, + failed: bool, +} + +impl Default for AmdCore { + fn default() -> Self { + Self { + mode: Mode::Array, + selected: [false; SECTORS], + remaining_ns: 0, + setup_ns: 0, + toggle: false, + failed: false, + } + } +} + +impl AmdCore { + pub fn reset(&mut self) { + *self = Self::default(); + } + pub fn busy(&self) -> bool { + self.remaining_ns != 0 || self.failed + } + pub fn selecting(&self) -> bool { + self.setup_ns != 0 + } + pub fn array_reads(&self) -> bool { + !self.busy() && self.mode != Mode::Query && self.mode != Mode::Identify + } + /// None selects ordinary backing bytes. Polling never advances time. + pub fn read16(&mut self, offset: usize) -> Option { + if self.busy() { + let selected = self.selected[offset / SECTOR_SIZE]; + self.toggle = !self.toggle; + return Some( + (if self.toggle { + 0x40 | if selected { 4 } else { 0 } + } else { + 0 + }) | if self.setup_ns == 0 { 8 } else { 0 } + | if self.failed { 0x20 } else { 0 }, + ); + } + match self.mode { + Mode::Query => Some(query(offset / 2) as u16), + // IDs remain unspecified until a part/profile is selected. + Mode::Identify => Some(0), + _ => None, + } + } + pub fn write16(&mut self, offset: usize, value: u16) -> Result { + if offset & 1 != 0 || offset >= PART_SIZE { + return Err(()); + } + let cmd = value as u8; + let word = offset / 2; + if self.busy() { + if (self.failed || self.setup_ns != 0) && cmd == 0xf0 { + self.reset(); + } else if self.setup_ns != 0 && cmd == 0x30 { + self.selected[offset / SECTOR_SIZE] = true; + self.setup_ns = ERASE_SETUP_NS; + self.remaining_ns = ERASE_SETUP_NS + ERASE_NS; + } else if self.setup_ns != 0 { + self.reset(); + crate::dlog!( + crate::devlog::LogModule::Bedrock, + "amd-flash: sector selection cancelled by command {cmd:#x} at word offset {word:#x}" + ); + return Err(()); + } else if cmd != 0xf0 { + crate::dlog!( + crate::devlog::LogModule::Bedrock, + "amd-flash: unsupported command {cmd:#x} while erasing at word offset {word:#x}" + ); + return Err(()); + } + return Ok(Effect::None); + } + // The pending program data cycle consumes every bit, including F0. + if self.mode == Mode::Program { + self.mode = Mode::Array; + return Ok(Effect::Program { offset, value }); + } + if cmd == 0xf0 { + self.reset(); + return Ok(Effect::None); + } + let next = match (self.mode, word, cmd) { + (Mode::Array, 0x55, 0x98) => Mode::Query, + (Mode::Array, 0x555, 0xaa) => Mode::Unlock1, + (Mode::Unlock1, 0x2aa, 0x55) => Mode::Unlock2, + (Mode::Unlock2, 0x555, 0xa0) => Mode::Program, + (Mode::Unlock2, 0x555, 0x80) => Mode::Erase1, + (Mode::Unlock2, 0x555, 0x90) => Mode::Identify, + (Mode::Erase1, 0x555, 0xaa) => Mode::Erase2, + (Mode::Erase2, 0x2aa, 0x55) => Mode::Erase3, + (Mode::Erase3, _, 0x30) => { + self.selected[offset / SECTOR_SIZE] = true; + self.setup_ns = ERASE_SETUP_NS; + self.remaining_ns = ERASE_SETUP_NS + ERASE_NS; + Mode::Array + } + _ => { + crate::dlog!( + crate::devlog::LogModule::Bedrock, + "amd-flash: unsupported cycle {value:#06x} at byte offset {offset:#x} in {:?}", + self.mode + ); + self.mode = Mode::Array; + return Err(()); + } + }; + self.mode = next; + Ok(Effect::None) + } + /// The owner persists and applies selected sectors before calling finish. + pub fn advance_ns(&mut self, ns: u64) -> bool { + if self.remaining_ns == 0 || self.failed { + return false; + } + self.setup_ns = self.setup_ns.saturating_sub(ns); + self.remaining_ns = self.remaining_ns.saturating_sub(ns); + self.remaining_ns == 0 + } + pub fn selected_sectors(&self) -> impl Iterator + '_ { + self.selected + .iter() + .enumerate() + .filter_map(|(i, selected)| selected.then_some(i)) + } + pub fn finish_erase(&mut self, success: bool) { + if success { + self.reset(); + } else { + self.failed = true; + } + } +} + +// Generic model response; voltage/timing do not identify Fuel's fitted part. +// Field positions/units: AMD publication 21444 rev E+2, pp14-15. +// Region encoding also follows Renode AMDCFIFlash.cs (MIT), revision +// 5448105bc7875aa6c5305761d97a371dc640dd7f. No Renode mutation code is used. +fn query(index: usize) -> u8 { + const PROFILE: [u8; 33] = [ + b'Q', b'R', b'Y', 2, 0, 0, 0, 0, 0, 0, 0, 0x27, 0x36, 0, 0, 3, 0, 9, 0, 4, 0, 4, 0, 23, 1, + 0, 0, 0, 1, 0x7f, 0, 0, 1, + ]; + index + .checked_sub(0x10) + .and_then(|i| PROFILE.get(i)) + .copied() + .unwrap_or(0) +} + +#[cfg(test)] +mod tests { + // Cases ported from upstream QEMU's tests/qtest/pflash-cfi02-test.c. + use super::*; + + // Word addresses from the command sequences, as byte offsets on the 16-bit bus. + const UNLOCK0: usize = 0x555 * 2; + const UNLOCK1: usize = 0x2aa * 2; + const CFI: usize = 0x55 * 2; + + fn cmd(f: &mut AmdCore, offset: usize, value: u16) -> Effect { + f.write16(offset, value).unwrap() + } + + fn unlock(f: &mut AmdCore) { + cmd(f, UNLOCK0, 0xaa); + cmd(f, UNLOCK1, 0x55); + } + + fn start_sector_erase(f: &mut AmdCore, offset: usize) { + unlock(f); + cmd(f, UNLOCK0, 0x80); + unlock(f); + cmd(f, offset, 0x30); + } + + /// The guest's erase loop polls DQ3, DQ6 and DQ7; get the phases wrong and + /// firmware either times out or reads the sector before it is blank. + #[test] + fn a_sector_erase_reports_toggle_status_until_it_completes() { + let mut f = AmdCore::default(); + let sector = 5 * SECTOR_SIZE; + start_sector_erase(&mut f, sector); + assert!(f.busy() && f.selecting()); + assert!(!f.array_reads()); + // DQ3 clear: still inside the window where more sectors may be added. + let first = f.read16(sector).unwrap(); + assert_eq!(first & 0x08, 0); + assert_eq!(first & 0x80, 0); + f.advance_ns(ERASE_SETUP_NS); + assert!(!f.selecting()); + let a = f.read16(sector).unwrap(); + let b = f.read16(sector).unwrap(); + assert_eq!(a & 0x08, 0x08); + assert_eq!(a & 0x80, 0); + assert_eq!(a & 0x40, !b & 0x40, "DQ6 toggles on successive reads"); + assert!(f.advance_ns(ERASE_NS)); + assert_eq!(f.selected_sectors().collect::>(), [5]); + f.finish_erase(true); + assert!(f.array_reads()); + assert_eq!(f.read16(sector), None); + } + + /// Firmware erases several sectors with one command; each further 0x30 + /// inside the selection window must join the set, not start a second erase. + #[test] + fn extra_sectors_join_a_pending_erase_inside_the_selection_window() { + let mut f = AmdCore::default(); + start_sector_erase(&mut f, 0); + cmd(&mut f, 3 * SECTOR_SIZE, 0x30); + assert_eq!(f.selected_sectors().collect::>(), [0, 3]); + // The window restarts on each selection. + f.advance_ns(ERASE_SETUP_NS - 1); + assert!(f.selecting()); + // A non-erase command in the window cancels the whole operation. + assert!(f.write16(0, 0x55).is_err()); + assert!(!f.busy()); + assert_eq!(f.selected_sectors().count(), 0); + } + + /// Reset (F0) is how firmware backs out of a half-finished command or + /// leaves CFI mode, so it must always return the part to array reads. + #[test] + fn a_reset_command_abandons_a_partial_unlock_and_leaves_cfi_mode() { + let mut f = AmdCore::default(); + cmd(&mut f, UNLOCK0, 0xaa); + cmd(&mut f, 0, 0xf0); + // The unlock is forgotten: a program setup now is not a valid cycle. + assert!(f.write16(UNLOCK1, 0x55).is_err()); + + cmd(&mut f, CFI, 0x98); + assert!(!f.array_reads()); + cmd(&mut f, 0, 0xf0); + assert!(f.array_reads()); + assert_eq!(f.read16(0x10 * 2), None); + } + + /// Firmware that probes the part reads the CFI "QRY" signature and the + /// device-size exponent to learn how much flash it has. + #[test] + fn the_cfi_query_returns_the_qry_signature_and_the_size_exponent() { + let mut f = AmdCore::default(); + cmd(&mut f, CFI, 0x98); + let q = |f: &mut AmdCore, word: usize| f.read16(word * 2).unwrap(); + assert_eq!([q(&mut f, 0x10), q(&mut f, 0x11), q(&mut f, 0x12)], [b'Q' as u16, b'R' as u16, b'Y' as u16]); + assert_eq!(1usize << q(&mut f, 0x27), PART_SIZE); + } + + /// Word program is the unlock sequence, A0, then one data cycle; the data + /// cycle goes to the owner as an effect and is not decoded as a command. + #[test] + fn a_program_sequence_hands_the_data_word_to_the_owner() { + let mut f = AmdCore::default(); + unlock(&mut f); + cmd(&mut f, UNLOCK0, 0xa0); + // 0xf0 as data must not be taken for a reset. + assert_eq!(cmd(&mut f, 0x1234 * 2, 0xa5f0), Effect::Program { offset: 0x1234 * 2, value: 0xa5f0 }); + assert!(f.array_reads()); + // Odd byte offsets are not word cycles. + assert!(f.write16(UNLOCK0 + 1, 0xaa).is_err()); + } +} diff --git a/src/dev/sflash/storage.rs b/src/dev/sflash/storage.rs new file mode 100644 index 000000000..815e1aa6d --- /dev/null +++ b/src/dev/sflash/storage.rs @@ -0,0 +1,62 @@ +//! Exact-size file backing shared by flash command-set models. + +use std::fs::{File, OpenOptions}; +use std::io::{self, Read, Seek, SeekFrom, Write}; +use std::path::Path; + +pub(crate) struct FlashBacking { + file: File, + size: usize, +} + +impl FlashBacking { + #[cfg(test)] + pub(crate) fn from_file(file: File, size: usize) -> Self { + Self { file, size } + } + + /// Open a flash store, loading its bytes or seeding a new empty file. + /// A non-empty file must match the supplied array size and is left intact + /// when its size is refused. + pub(crate) fn open(path: impl AsRef, data: &mut [u8]) -> io::Result { + let path = path.as_ref(); + let mut file = OpenOptions::new() + .read(true) + .write(true) + .create(true) + .truncate(false) + .open(path)?; + let len = file.metadata()?.len(); + if len == data.len() as u64 { + file.read_exact(data)?; + } else if len == 0 { + file.write_all(data)?; + } else { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + format!( + "flash file {} is {len} bytes, expected {} (or empty); not overwriting it", + path.display(), + data.len() + ), + )); + } + Ok(Self { + file, + size: data.len(), + }) + } + + /// Write one byte range through to the backing file. + pub(crate) fn write_range(&mut self, offset: usize, bytes: &[u8]) -> io::Result<()> { + if offset > self.size || bytes.len() > self.size - offset { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "flash backing range is outside the array", + )); + } + self.file.seek(SeekFrom::Start(offset as u64))?; + self.file.write_all(bytes)?; + self.file.flush() + } +} diff --git a/src/devlog.rs b/src/devlog.rs index d082e7ba1..b9ee4f6fa 100644 --- a/src/devlog.rs +++ b/src/devlog.rs @@ -55,6 +55,7 @@ pub enum LogModule { /// The Octane's VPro (Odyssey) graphics board: guest accesses the /// model does not support. Odyssey = 28, + Bedrock = 29, } // Mask bits shared by L1i / L1d / L2c @@ -63,7 +64,7 @@ pub const CACHE_LOG_MISS: u32 = 0x2; // log misses and fills pub const CACHE_LOG_OP: u32 = 0x4; // log CACHE instruction ops impl LogModule { - pub const COUNT: usize = 29; + pub const COUNT: usize = 30; pub fn from_str(s: &str) -> Option { match s { @@ -96,6 +97,7 @@ impl LogModule { "gr2" => Some(Self::Gr2), "rad1" => Some(Self::Rad1), "odyssey" => Some(Self::Odyssey), + "bedrock" => Some(Self::Bedrock), _ => None, } } @@ -131,6 +133,7 @@ impl LogModule { Self::Gr2 => "gr2", Self::Rad1 => "rad1", Self::Odyssey => "odyssey", + Self::Bedrock => "bedrock", } } @@ -141,7 +144,7 @@ impl LogModule { Self::Vino, Self::Dcb, Self::Vc2, Self::Cmap, Self::Xmap, Self::Bt445, Self::Scc, Self::Ps2, Self::Rtc, Self::Eeprom, Self::L1i, Self::L1d, Self::L2c, Self::Ultra, Self::Scsi1, - Self::Nveeprom, Self::Gr2, Self::Rad1, Self::Odyssey, + Self::Nveeprom, Self::Gr2, Self::Rad1, Self::Odyssey, Self::Bedrock, ] } @@ -283,6 +286,7 @@ impl DevLog { ModuleLog::new(), // Gr2 ModuleLog::new(), // Rad1 ModuleLog::new(), // Odyssey + ModuleLog::new(), // Bedrock ], } } From d603450d834396bea13afbc25a54541729df9b66 Mon Sep 17 00:00:00 2001 From: James Braid Date: Fri, 30 Oct 2026 08:03:25 +0000 Subject: [PATCH 18/31] cpu: send uncached byte and halfword stores with the whole register Fuel PROM 6.210's memory test fails when uncached SB and SH reach the bus carrying only their low bits. Its L1 console shows: mtest_bank_main failed: Walking Address RSLT mtest_bank_mai FAIL diag_rc = 43 Walking Address In both memory banks the expected halfwords 0xa55a, 0x5aa5, 0x9999 and 0x6666 read back from the ECC check-bit view as 0x0000. The PROM stores each check word with SH from a register whose low 16 bits are zero; the word sits higher in the register, at a position that depends on the tested address. With the whole register visible to the ECC view and the same PROM configuration it reports ALL DIAGS PASSED and enables all 512 MiB. Add BusDevice::write_narrow_from, whose default stores the architectural byte or halfword exactly as write8/write16 do, and call it for every uncached SB and SH. Ordinary devices therefore still see the low bits of the register, and only a device that models a path observing the other lanes overrides it; Bedrock's ECC view does, in its own commit. Which lanes real hardware drives for a narrow store is undefined. Debug writes keep the ordinary-width path, and the JIT callouts pass the same source register. The default adds one virtual call to a path that already makes one. --- HACKING.md | 4 ++ src/cpu/mips_exec.rs | 23 +++++-- src/cpu/mips_exec_test.rs | 140 ++++++++++++++++++++++++++++++++++++++ src/traits.rs | 14 ++++ 4 files changed, 174 insertions(+), 7 deletions(-) diff --git a/HACKING.md b/HACKING.md index 9028bb6f4..6b89acbf0 100644 --- a/HACKING.md +++ b/HACKING.md @@ -384,6 +384,10 @@ consumer thread never takes `rec`. monomorphised over it, so every CPU is in every binary with no per-model branch on the hot path. `Machine::new` picks one from `[machine] cpu`. +- Every uncached SB/SH goes to the bus through `BusDevice::write_narrow_from` + with the whole source register: the byte or halfword is the low bits, the + other lanes are undefined by hardware, and only Bedrock's ECC view reads + them. - With `opcodefusion` (implied by `lightning`), the interpreter executes common pairs (branch + NOP, LUI + ORI/ADDIU, address + load/store) as one dispatch. diff --git a/src/cpu/mips_exec.rs b/src/cpu/mips_exec.rs index 0e7e64ff4..0bb4133b9 100644 --- a/src/cpu/mips_exec.rs +++ b/src/cpu/mips_exec.rs @@ -1364,20 +1364,21 @@ unsafe extern "C" fn jit_read64(ctx: *mut core::ffi::c_void #[cfg(feature = "jitv2")] unsafe extern "C" fn jit_write8(ctx: *mut core::ffi::c_void, va: u64, val: u64) -> u32 { let exec = unsafe { &mut *exec_from_core::(ctx) }; - let val = val as u8; // mask to the real width ourselves — see write8_fn's doc comment on why the FFI parameter is u64 + // The whole register, as the interpreter's SB passes it: `write_data` + // narrows it where it stores, and an uncached store hands it to the bus + // (`BusDevice::write_narrow_from`). #[cfg(feature = "jitv2_lockstep")] - { return exec.lockstep_jit_write::<1>(va, val as u64); } + { return exec.lockstep_jit_write::<1>(va, val); } #[cfg(not(feature = "jitv2_lockstep"))] - exec.write_data::<1>(va, val as u64) + exec.write_data::<1>(va, val) } #[cfg(feature = "jitv2")] unsafe extern "C" fn jit_write16(ctx: *mut core::ffi::c_void, va: u64, val: u64) -> u32 { let exec = unsafe { &mut *exec_from_core::(ctx) }; - let val = val as u16; // mask to the real width ourselves — see write16_fn's doc comment #[cfg(feature = "jitv2_lockstep")] - { return exec.lockstep_jit_write::<2>(va, val as u64); } + { return exec.lockstep_jit_write::<2>(va, val); } #[cfg(not(feature = "jitv2_lockstep"))] - exec.write_data::<2>(va, val as u64) + exec.write_data::<2>(va, val) } #[cfg(feature = "jitv2")] unsafe extern "C" fn jit_write32(ctx: *mut core::ffi::c_void, va: u64, val: u64) -> u32 { @@ -5132,7 +5133,15 @@ va={:#018x} phys={:#010x} (code pfn {:#x}, page {:#010x}, word {}/{})", if !DEBUG && mips_log(MIPS_LOG_MEM) { dlog_dev!(LogModule::Mips, "Uncached Write{}: PC={:016x} VA={:016x} PA={:016x} Val={:016x}", SIZE*8, self.core.pc, virt_addr, phys_addr, val); } - let ws = if SIZE == 1 { + let ws = if !DEBUG && SIZE <= 2 { + // The bus sees the whole source register: Fuel PROM 6.210's + // Walking Address test reads its check words back from + // Bedrock's ECC view, and the word sits elsewhere in the + // register than the addressed lanes. Ordinary devices take + // the architectural low bits. Debug writes carry just the + // value they name, so they keep the ordinary-width path. + self.sysad.write_narrow_from(phys_addr, SIZE, val) + } else if SIZE == 1 { self.sysad.write8(phys_addr, val as u8) } else if SIZE == 2 { self.sysad.write16(phys_addr, val as u16) diff --git a/src/cpu/mips_exec_test.rs b/src/cpu/mips_exec_test.rs index b7734c0c7..e0df5ff92 100644 --- a/src/cpu/mips_exec_test.rs +++ b/src/cpu/mips_exec_test.rs @@ -13,12 +13,18 @@ mod tests { // Mock Memory Interface pub struct MockMemory { pub data: Mutex>, + /// `write8`/`write16` calls as (width, address, value). + pub narrow_writes: Mutex>, + /// `write_narrow_from` calls as (width, address, source register). + pub narrow_from_writes: Mutex>, } impl MockMemory { pub fn new() -> Self { Self { data: Mutex::new(HashMap::new()), + narrow_writes: Mutex::new(Vec::new()), + narrow_from_writes: Mutex::new(Vec::new()), } } @@ -67,6 +73,7 @@ mod tests { } fn write8(&self, addr: u64, val: u8) -> u32 { + self.narrow_writes.lock().unwrap().push((1, addr, val as u64)); self.set_byte(addr as u64, val); BUS_OK } @@ -82,6 +89,7 @@ mod tests { fn write16(&self, addr: u64, val: u16) -> u32 { let aligned_addr = (addr & !1) as u64; + self.narrow_writes.lock().unwrap().push((2, aligned_addr, val as u64)); let bytes = val.to_be_bytes(); for i in 0..2 { self.set_byte(aligned_addr + i as u64, bytes[i]); @@ -111,6 +119,12 @@ mod tests { BUS_OK } + fn write_narrow_from(&self, addr: u64, size: usize, gpr: u64) -> u32 { + self.narrow_from_writes.lock().unwrap().push((size, addr, gpr)); + // Same architectural bytes as the default. + if size == 1 { self.write8(addr, gpr as u8) } else { self.write16(addr, gpr as u16) } + } + // This whole file predates jitv2 and no test here calls // `install_jit_hooks` — every `create_executor*` helper below // explicitly disables both the real dispatch gate @@ -575,6 +589,132 @@ mod tests { assert_eq!(mem.get_byte(0x100B), 0xEF); } + /// Fuel PROM memory diagnostics need the whole source register on the ECC + /// view, so every CPU hands an uncached SB or SH to + /// `BusDevice::write_narrow_from` with it. The default of that call stores + /// the architectural byte or halfword, which is all an ordinary device + /// sees, and the addressed lanes hold it either way. + #[test] + fn uncached_sb_sh_carry_the_whole_source_register() { + fn check>>() { + let mem = Arc::new(MockMemory::new()); + let bus: Arc = mem.clone(); + let cfg = MipsCpuConfig::for_model::(Arc::new(crate::ip22_bus::memory_map(false))); + let mut exec = MipsExecutor::::new( + bus, + PassthroughTlb::default(), + &cfg, + ); + disable_jitv2(&mut exec); + + let base = 0x2000u64; + let source = 0xFEDC_BA98_7654_3210u64; + let original = 0xAABB_CCDD_EEFF_0011u64; + exec.core.write_gpr(2, base); + exec.core.write_gpr(1, source); + + for (size, op, offsets) in [(1usize, OP_SB, vec![0u64, 1, 2, 3, 4, 5, 6, 7]), + (2usize, OP_SH, vec![0u64, 2, 4, 6])] { + for offset in offsets { + mem.set_double(base, original); + mem.narrow_writes.lock().unwrap().clear(); + mem.narrow_from_writes.lock().unwrap().clear(); + let instr = make_i(op, 2, 1, offset as u16); + assert_eq!(exec.exec(instr), EXEC_COMPLETE); + + let addr = base + offset; + assert_eq!( + *mem.narrow_from_writes.lock().unwrap(), + vec![(size, addr, source)], + "SB/SH source register at byte offset {offset}", + ); + + let first = if size == 1 { source as u8 } else { (source as u16 >> 8) as u8 }; + let last = if size == 1 { first } else { source as u8 }; + assert_eq!(mem.get_byte(addr), first, "stored leading byte at offset {offset}"); + if size == 2 { + assert_eq!(mem.get_byte(addr + 1), last, "stored trailing byte at offset {offset}"); + } + let original_bytes = original.to_be_bytes(); + for lane in 0..8u64 { + if lane < offset || lane >= offset + size as u64 { + assert_eq!(mem.get_byte(base + lane), original_bytes[lane as usize], + "inactive byte lane {lane} at subword offset {offset}"); + } + } + } + } + + // A zero source register still takes the same path and clears + // only the addressed lanes. + exec.core.write_gpr(0, u64::MAX); + mem.set_double(base, original); + mem.narrow_from_writes.lock().unwrap().clear(); + exec.core.write_gpr(2, base + 7); + assert_eq!(exec.exec(make_i(OP_SB, 2, 0, 0)), EXEC_COMPLETE); + assert_eq!(mem.get_byte(base + 7), 0); + assert_eq!(mem.narrow_from_writes.lock().unwrap().len(), 1); + + mem.set_double(base, original); + exec.core.write_gpr(2, base + 6); + assert_eq!(exec.exec(make_i(OP_SH, 2, 0, 0)), EXEC_COMPLETE); + assert_eq!(mem.get_byte(base + 6), 0); + assert_eq!(mem.get_byte(base + 7), 0); + assert_eq!(mem.narrow_from_writes.lock().unwrap().len(), 2); + + mem.narrow_writes.lock().unwrap().clear(); + mem.narrow_from_writes.lock().unwrap().clear(); + assert_eq!(exec.debug_write(base + 1, source, 1, u64::MAX), EXEC_COMPLETE); + assert_eq!(*mem.narrow_writes.lock().unwrap(), vec![(1, base + 1, source as u8 as u64)]); + assert!(mem.narrow_from_writes.lock().unwrap().is_empty(), "debug writes keep the ordinary-width path"); + } + + check::(); + check::(); + check::(); + check::(); + } + + /// JIT store callouts carry the same source register as the interpreter. + #[test] + #[cfg(feature = "jitv2")] + fn jit_subword_hooks_carry_the_source_register() { + fn check>>() { + let mem = Arc::new(MockMemory::new()); + let bus: Arc = mem.clone(); + let cfg = MipsCpuConfig::for_model::(Arc::new(crate::ip22_bus::memory_map(false))); + let mut exec = MipsExecutor::::new( + bus, + PassthroughTlb::default(), + &cfg, + ); + disable_jitv2(&mut exec); + exec.install_jit_hooks(); + + let base = 0x2000u64; + let source = 0xFEDC_BA98_7654_3210u64; + let write8 = exec.core.write8_fn; + let write16 = exec.core.write16_fn; + let core = &mut exec.core as *mut crate::cpu::mips_core::MipsCore; + // JIT callouts receive the core pointer biased by the shared + // ABI constant; core_from_arg removes this before recovery. + let ctx = core.cast::() + .wrapping_add(crate::cpu::jitv2::codegen::CALLOUT_CORE_BIAS as usize) + .cast(); + unsafe { + write8(ctx, base, source); + write16(ctx, base + 2, source); + } + assert_eq!( + *mem.narrow_from_writes.lock().unwrap(), + vec![(1, base, source), (2, base + 2, source)], + ); + } + + check::(); + check::(); + } + #[test] fn test_load_store_pc_advances() { // Regression test: every load/store dispatch-target handler must diff --git a/src/traits.rs b/src/traits.rs index 604cb65c2..a0f322836 100644 --- a/src/traits.rs +++ b/src/traits.rs @@ -223,6 +223,20 @@ pub trait BusDevice: Send + Sync { BUS_OK } + /// An uncached SB (`size` 1) or SH (`size` 2) to `addr`, with the whole + /// source register `gpr` for a bus that can see the lanes the store does + /// not select. The CPU calls this instead of `write8`/`write16` for an + /// uncached SB or SH. + /// + /// The architectural byte or halfword is the low bits of `gpr`, which is + /// all the default delivers. Which other lanes a real processor drives for + /// a narrow store is undefined, so only a device that models a path + /// observing them (Bedrock's memory ECC view) overrides this, and a bus + /// that forwards accesses must forward this call too. + fn write_narrow_from(&self, addr: u64, size: usize, gpr: u64) -> u32 { + if size == 1 { self.write8(addr, gpr as u8) } else { self.write16(addr, gpr as u16) } + } + /// JIT v2 (clanker/jitv2/jit-v2-design.md §2.4): pointer to the generation counter /// for the 4KB physical page containing `addr`, or null if this device doesn't /// back JIT-compilable memory (MMIO, etc). RAM devices return one counter per From 72eac996051b2bcab71a436680c894e8c0e04d7c Mon Sep 17 00:00:00 2001 From: James Braid Date: Fri, 30 Oct 2026 05:16:10 -0700 Subject: [PATCH 19/31] cpu: carry the R10000 family's uncached attribute through mapped accesses The R10000 drives a two-bit uncached attribute on the system bus. For an unmapped xkphys access it is address bits 58:57; for a mapped reference it comes from the TLB's uncached-attribute field, which EntryLo0/1 bits 63:62 load (R10000 User's Manual 007-2490-001). EntryLo already keeps those bits on the family, and a TLB entry holds them, but the entry's physical base dropped them, so a mapped uncached page always reached attribute zero. Carry them to physical bits 58:57 only for pages the model's C table decodes as uncached, so cached mappings are unchanged. The TLB already holds that table for its shadow entries; reading it there, and recomputing each entry's base when a table is installed, avoids a second decode of C. The attribute reaches the bus only on a board that decodes physical bits 58:57. The Octane, Indigo2 and O2 mask the physical address to 40, 32 and 32 bits before the bus, which folds it away, and a test shows that at the executor. The R4400 and R5000 never store the bits. Tests cover the attribute and the board masks, and the C table (a value the model decodes as cached carries nothing; a table installed after the entry was written decides). The uncached-attribute comments cite NEC U10278EJ4V0UM, the fourth edition of the VR10000/VR12000 user's manual, section 11.3, by document number; the manual itself is not added to the repository. --- src/cpu/mips_exec_test.rs | 56 +++++++++++++++++++++++++++++++++++++++ src/cpu/mips_tlb.rs | 38 ++++++++++++++++++++++---- src/cpu/mips_tlb_test.rs | 49 ++++++++++++++++++++++++++++++++++ 3 files changed, 138 insertions(+), 5 deletions(-) diff --git a/src/cpu/mips_exec_test.rs b/src/cpu/mips_exec_test.rs index e0df5ff92..c68df6285 100644 --- a/src/cpu/mips_exec_test.rs +++ b/src/cpu/mips_exec_test.rs @@ -4086,6 +4086,62 @@ mod tests { }); } + /// A mapped uncached page's attribute (EntryLo 63:62, carried to physical + /// bits 58:57) reaches the bus on a board that decodes them, and is folded + /// away on the Octane (40 bits), Indigo2 and O2 (32), which is what keeps + /// those machines unchanged. + #[test] + fn mapped_uncached_attribute_is_folded_by_the_boards_physical_mask() { + std::thread::Builder::new() + .stack_size(16 * 1024 * 1024) + .spawn(|| { + use crate::cpu::mips_tlb::MipsTlb; + const VA: u64 = 0x4000_0000; + + fn translate>>( + phys_mask: u64, alias: Option<(u64, u64, u64)>, pfn: u64, + ) -> u64 { + let bus: Arc = Arc::new(MockMemory::new()); + let cfg = MipsCpuConfig::for_model::(Arc::new(crate::ip22_bus::memory_map(false))); + let mut exec = Box::new(MipsExecutor::::new( + bus, MipsTlb::default(), &cfg)); + disable_jitv2(&mut exec); + exec.install_status_cb(); + let dmtc0 = |exec: &mut Box>, rd: u32, value: u64| { + exec.core.write_gpr(1, value); + let op = (OP_COP0 << 26) | (RS_DMTC0 << 21) | (1 << 16) | (rd << 11); + assert_eq!(exec.exec(op), EXEC_COMPLETE); + }; + let status = (exec.core.cp0_status & !crate::cpu::mips_core::STATUS_ERL) | STATUS_KX; + dmtc0(&mut exec, 12, status as u64); + dmtc0(&mut exec, 10, VA); + dmtc0(&mut exec, 0, 0); + dmtc0(&mut exec, 5, 0); + // C=2 (uncached), valid, dirty, attribute 1 in bits 63:62. + dmtc0(&mut exec, 2, (1 << 62) | (pfn << 6) | (2 << 3) | (1 << 2) | (1 << 1)); + dmtc0(&mut exec, 3, 0); + assert_eq!(exec.exec((OP_COP0 << 26) | (RS_TLB << 21) | FUNCT_TLBWI), EXEC_COMPLETE); + exec.phys_mask = phys_mask; + exec.phys_alias = alias; + exec.debug_translate(VA).phys + } + + type R10K = crate::cpu::mips_cache_shadow::R10000ShadowCache; + type R14K = crate::cpu::mips_cache_shadow::R12000ShadowCache; + const PA: u64 = 0x0d10_0000; + assert_eq!(translate::(u64::MAX, None, PA >> 12), (1 << 57) | PA, + "no mask: the attribute reaches the bus"); + for (name, mask) in [("Octane", (1u64 << 40) - 1), ("Indigo2/O2", u32::MAX as u64)] { + assert_eq!(translate::(mask, None, PA >> 12), PA, "{name} folds it away"); + } + assert_eq!(translate::(u64::MAX, None, PA >> 12), PA, + "a model without the attribute never stores it"); + }) + .unwrap() + .join() + .unwrap(); + } + /// The Indy, Indigo2 and O2 decode 32 physical bits, and IRIX relies on /// it: on the IP28 the kernel stack's wired entry carries PTE software /// bits as PA[34:33]. `phys_mask` must fold them away before the bus. diff --git a/src/cpu/mips_tlb.rs b/src/cpu/mips_tlb.rs index 7315c28ba..34a111dbf 100644 --- a/src/cpu/mips_tlb.rs +++ b/src/cpu/mips_tlb.rs @@ -147,6 +147,23 @@ pub enum AccessType { Debug = 3, } +/// The physical base a TLB entry's EntryLo maps. A model that keeps the +/// R10000 family's uncached attribute (EntryLo bits 63:62, see +/// `CpuModel::R10K_FAMILY`) has it carried to physical bits 58:57 +/// for an uncached page, where an unmapped xkphys access carries it; other +/// models never store those bits. For a mapped reference the R10000 takes the +/// two-bit uncached attribute from the TLB's uncached-attribute field, loaded +/// from EntryLo0/1 bits 63:62 (NEC U10278EJ4V0UM, VR10000/VR12000 user's +/// manual, 4th edition, section 11.3; R10000 User's Manual 007-2490-001). +/// +/// Whether the page is uncached is the caller's decode of C through the model's +/// table (`MipsTlb::c_attr`), not another decode here. +#[inline] +fn pfn_base(lo: u64, pfn_mask: u64, uncached: bool) -> u64 { + let attribute = if uncached { ((lo >> 62) & 3) << 57 } else { 0 }; + ((lo << 6) & pfn_mask) | attribute +} + /// TLB Entry structure matching R4000 TLB format #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct TlbEntry { @@ -996,6 +1013,16 @@ impl Default for MipsTlb { } } +impl MipsTlb { + /// `pfn_base` for both halves of an entry, deciding "uncached" with the + /// model's C table. + #[inline] + fn pfn_bases(&self, lo: &[u64; 2], pfn_mask: u64) -> [u64; 2] { + let uncached = |lo: u64| self.c_attr[((lo >> 3) & 7) as usize] == CacheAttr::Uncached; + [pfn_base(lo[0], pfn_mask, uncached(lo[0])), pfn_base(lo[1], pfn_mask, uncached(lo[1]))] + } +} + impl Tlb for MipsTlb { #[inline] fn translate(&mut self, virt_addr: u64, asid: u8, access_type: AccessType) -> TlbResult { @@ -1165,10 +1192,7 @@ impl Tlb for MipsTlb { entry.vpn_hi64 = entry.entry_hi & entry.vcmp64; entry.offset_mask = (1u64 << entry.selector_bit_shift) - 1; let pfn_mask = 0xFF_FFFF_FFFF_000 & !entry.offset_mask; - entry.pfn_base = [ - (entry.entry_lo[0] << 6) & pfn_mask, - (entry.entry_lo[1] << 6) & pfn_mask, - ]; + entry.pfn_base = self.pfn_bases(&entry.entry_lo, pfn_mask); self.irregular_mask |= mask & (mask + 1) != 0; if self.invalidate_on_write { self.invalidate_overlapped(index, &entry); @@ -1310,6 +1334,10 @@ impl Tlb for MipsTlb { fn set_cache_attrs(&mut self, table: [CacheAttr; 8]) { self.c_attr = table; for i in 0..self.num_entries { + // pfn_base carries the attribute for pages the new table decodes + // as uncached, so it follows the table. + let pfn_mask = 0xFF_FFFF_FFFF_000 & !self.entries[i].offset_mask; + self.entries[i].pfn_base = self.pfn_bases(&self.entries[i].entry_lo, pfn_mask); self.shadow[i] = ShadowEntry::from_entry(&self.entries[i], &self.c_attr); } } @@ -1368,7 +1396,7 @@ impl Tlb for MipsTlb { vcmp64, vpn_hi64: entry_hi & vcmp64, offset_mask, - pfn_base: [(lo[0] << 6) & pfn_mask, (lo[1] << 6) & pfn_mask], + pfn_base: self.pfn_bases(&lo, pfn_mask), }; } } diff --git a/src/cpu/mips_tlb_test.rs b/src/cpu/mips_tlb_test.rs index d594d6e74..98f25dd20 100644 --- a/src/cpu/mips_tlb_test.rs +++ b/src/cpu/mips_tlb_test.rs @@ -682,3 +682,52 @@ fn tlb_write_inside_a_larger_entry_count_is_kept() { assert_eq!(r10000.read(63).entry_hi, entry.entry_hi, "...and its last slot"); } } + +/// A mapped uncached page carries EntryLo's uncached attribute (bits 63:62) +/// to the bus as physical bits 58:57; a cached page never does, whatever those bits +/// hold, and no attribute leaves the PFN alone. "Uncached" is the model's C +/// table, so a model that decodes a C value differently, or a table installed +/// after the entry was written, changes the answer. +#[test] +fn only_uncached_pages_carry_the_uncached_attribute() { + use super::{MipsTlb, Tlb, TlbEntry}; + use crate::cpu::mips_exec::CacheAttr; + // MipsTlb is large; give the one instance room, as its neighbours do. + std::thread::Builder::new() + .stack_size(16 * 1024 * 1024) + .spawn(|| { + let pfn = 0x0d100u64 << 6; + let base = |tlb: &MipsTlb| tlb.read(0).pfn_base; + let write = |tlb: &mut MipsTlb, lo0: u64, lo1: u64| { + let mut entry = TlbEntry::new(); + entry.entry_hi = 0xC000_0000_0004_0000; + entry.entry_lo = [lo0, lo1]; + tlb.write(0, entry); + }; + let mut tlb = Box::new(MipsTlb::new(64)); + // The default table: C=2 and the reserved values are uncached. + for (cca, expect) in [(2u64, 1u64 << 57), (6, 1 << 57), (3, 0), (5, 0)] { + let lo = (1 << 62) | pfn | (cca << 3) | 0x6; + write(&mut tlb, lo, lo); + assert_eq!(base(&tlb), [0x0d10_0000 | expect; 2], "cca {cca}"); + } + write(&mut tlb, pfn | (2 << 3), 0); + assert_eq!(base(&tlb)[0], 0x0d10_0000, "no attribute leaves the PFN alone"); + + // The R10000 table reads C=7 (uncached accelerated) as uncached and + // 5 as cacheable; a model that read 7 as cacheable carries nothing. + let mut table = [CacheAttr::Uncached; 8]; + table[3] = CacheAttr::Cacheable; + table[5] = CacheAttr::CacheableCoherent; + tlb.set_cache_attrs(table); + let lo = (1 << 62) | pfn | (7 << 3) | 0x6; + write(&mut tlb, lo, lo); + assert_eq!(base(&tlb), [0x0d10_0000 | 1 << 57; 2], "C=7 uncached under this table"); + table[7] = CacheAttr::Cacheable; + tlb.set_cache_attrs(table); + assert_eq!(base(&tlb), [0x0d10_0000; 2], "the table, installed later, decides"); + }) + .unwrap() + .join() + .unwrap(); +} From d3c9d1c4c6dbba2ab51a7dbe0591aecb9fe328be Mon Sep 17 00:00:00 2001 From: James Braid Date: Fri, 30 Oct 2026 08:13:37 +0000 Subject: [PATCH 20/31] bedrock_md: add the Fuel memory controller Bedrock memory configuration changes the RAM decode. Keep ordinary CPU and DMA memory on Physical directly, divert diagnostic pages through MD, and update their generation state when memory configuration or ECC changes. The ECC view overrides write_narrow_from to take an uncached SH's check word from the whole source register, at the position Fuel PROM 6.210's memory test uses. Every other view stores the architectural low bits. --- src/dev/bedrock_md.rs | 1713 ++++++++++++++++++++++++++++++++++++++++ src/dev/bedrock_mem.rs | 311 ++++++++ src/dev/mod.rs | 2 + 3 files changed, 2026 insertions(+) create mode 100644 src/dev/bedrock_md.rs create mode 100644 src/dev/bedrock_mem.rs diff --git a/src/dev/bedrock_md.rs b/src/dev/bedrock_md.rs new file mode 100644 index 000000000..b5ef5936b --- /dev/null +++ b/src/dev/bedrock_md.rs @@ -0,0 +1,1713 @@ +//! Bedrock's memory and directory controller: bank configuration, memory +//! self-test, ECC, scrubbing and the directory and fetch-operation views. +//! +//! Memory is held by `bedrock_mem::BedrockMemory`. CPU coherence requests +//! and hardware protection enforcement are absent; protection registers +//! retain their words. Known unsupported writes are counted and ignored, +//! and unmapped accesses return a bus error. +//! +//! Plain register words, directory entries and reference counters, +//! protection entries and injected ECC check bytes live in sparse maps. The +//! guest touches a few of the millions of possible entries, and an entry it +//! never wrote reads as its default, so a dense table would only cost memory. +//! +//! Ported from hw/misc/sgi-bedrock-md.c in jamesbraid/qemu-sgi-origami. + +use super::bedrock_mem::{BedrockMemory, BANK_SPAN, FITTED_BANK}; +use crate::traits::*; +use parking_lot::Mutex; +use std::collections::{BTreeSet, HashMap, HashSet}; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::{Arc, Weak}; + + +pub mod reg { + pub const CELL: u64 = 0x0; + pub const CONFIG: u64 = 0x8; + pub const ARBITRATION: u64 = 0x10; + pub const MIGRATION_CONFIG: u64 = 0x18; + /// Outgoing reply queue sizes: read-only, so a write changes nothing. + pub const REPLY_QUEUE_SIZE: u64 = 0x60; + pub const PERF_SELECT0: u64 = 0x10000; + pub const PERF_SELECT1: u64 = 0x10008; + pub const PERF_COUNTER0: u64 = 0x10010; + pub const PERF_COUNTER1: u64 = 0x10018; + pub const REFRESH: u64 = 0x20000; + /// Flash, UART and Synergy timing on the local "junk" bus. + pub const JUNK_BUS_TIMING: u64 = 0x20008; + pub const SELFTEST_COMMAND: u64 = 0x20030; + pub const SELFTEST_DATA: u64 = 0x20038; + pub const SELFTEST_ERROR: u64 = 0x20040; + pub const SELFTEST_STATUS: u64 = 0x20048; + pub const ISSUE_DEBUG: u64 = 0x20060; + pub const DIRECTORY_CONFIG: u64 = 0x40000; + pub const DIRECTORY_ERROR: u64 = 0x40010; + /// Reads the directory error record and clears it. + pub const DIRECTORY_ERROR_ALIAS: u64 = 0x40018; + pub const MIGRATION_DIFFERENCE: u64 = 0x40040; + pub const MIGRATION_VALUE: u64 = 0x40048; + /// Outgoing request queue sizes; the model keeps them fixed. + pub const REQUEST_QUEUE_SIZE: u64 = 0x40050; + pub const DIRECTORY_DEBUG: u64 = 0x40060; + pub const ECC_CONFIG: u64 = 0x60000; + pub const MEMORY_ERROR: u64 = 0x60010; + /// Reads the memory error record and clears it. + pub const MEMORY_ERROR_ALIAS: u64 = 0x60018; + pub const MEMORY_DEBUG: u64 = 0x60040; + + /// Reset sizes: 12 entries in each header field and 24 in each data field. + pub const REPLY_QUEUE_SIZES: u64 = 0x0018_1818_0c0c_0c00; + /// Reset sizes: four requests for each of the three queues. + pub const REQUEST_QUEUE_SIZES: u64 = 0x0404_0400; +} + +const CONFIG_MASK: u64 = 0xfbfbfbfb | (3 << 32) | (15 << 44) | (63 << 48) | (7 << 56) | (3 << 60); +const ECC_COLUMNS: [u8; 72] = [ + 0x13, 0x23, 0x43, 0x83, 0x2f, 0xf1, 0x0d, 0x07, 0xd0, 0x70, 0x4f, 0xf8, 0x61, 0x62, 0x64, 0x68, + 0x1c, 0x2c, 0x4c, 0x8c, 0x15, 0x25, 0x45, 0x85, 0x19, 0x29, 0x49, 0x89, 0x1a, 0x2a, 0x4a, 0x8a, + 0x51, 0x52, 0x54, 0x58, 0x91, 0x92, 0x94, 0x98, 0xa1, 0xa2, 0xa4, 0xa8, 0x31, 0x32, 0x34, 0x38, + 0x16, 0x26, 0x46, 0x86, 0x1f, 0xf2, 0x0b, 0x0e, 0xb0, 0xe0, 0x8f, 0xf4, 0xc1, 0xc2, 0xc4, 0xc8, + 1, 2, 4, 8, 16, 32, 64, 128, +]; +pub fn ecc_compute(mut value: u64) -> u8 { + let mut code = 0; + while value != 0 { + code ^= ECC_COLUMNS[value.trailing_zeros() as usize]; + value &= value - 1; + } + code +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum MemoryError { + Correctable, + Uncorrectable, +} +type ErrorCallback = Arc; + +struct State { + cell: u64, + config: u64, + refresh: u64, + directory_config: u64, + ecc_config: u64, + storage: HashMap, + directory: HashMap, + counters: HashMap, + protection: HashMap, + compat: HashMap, + ecc: HashMap, + admitted: [u16; 2], + memory_error: u64, + directory_error: u64, + marked_directory: HashSet, + clock_ns: u64, + clock_anchor: Option, + selftest_deadline: Option, + selftest_result: bool, + selftest_status: u64, + selftest_error: u64, +} +impl Default for State { + fn default() -> Self { + Self { + cell: u64::MAX, + config: 0, + refresh: 0, + directory_config: 0, + ecc_config: 0, + storage: HashMap::new(), + directory: HashMap::new(), + counters: HashMap::new(), + protection: HashMap::new(), + compat: HashMap::new(), + ecc: HashMap::new(), + admitted: [0; 2], + memory_error: 0, + directory_error: 0, + marked_directory: HashSet::new(), + clock_ns: 0, + clock_anchor: None, + selftest_deadline: None, + selftest_result: false, + selftest_status: 0, + selftest_error: 0, + } + } +} + +pub struct BedrockMd { + memory: BedrockMemory, + clock: Option, + // Ordinary RAM uses Physical's bank views. Diagnostic metadata, repair + // and fetch operations retain the state lock; direct bank traffic has + // the same concurrency contract as the other machines' RAM. + state: Mutex, + // CPU-side metadata writes and resets serialize snapshot/publication. + // RAM/DMA reads and writes may remove metadata without taking this lock. + remapping: Mutex<()>, + callback: Mutex>, + unsupported: AtomicU64, +} +#[derive(Clone, Copy)] +enum View { + Registers, + Memory, + Uncached, + Scrub, + Directory, + Fetchop, + Ecc, +} +struct MdBus { + md: Weak, + view: View, + // External endpoints may outlive MD. Keep their generation pointers valid. + // The endpoint owned by Physical itself needs no extra reference. + _memory: Option>, + #[cfg(feature = "jitv2")] + _generations: Arc, +} + +impl BedrockMd { + pub fn new() -> Arc { + Self::build(None) + } + pub fn with_clock(clock: crate::dev::heart::Clock) -> Arc { + Self::build(Some(clock)) + } + fn build(clock: Option) -> Arc { + let md = Arc::new(Self { + memory: BedrockMemory::new(), + clock, + state: Mutex::new(State::default()), + remapping: Mutex::new(()), + callback: Mutex::new(None), + unsupported: AtomicU64::new(0), + }); + md.memory.set_slow_view(md.view(View::Memory)); + md.rebuild_memory(); + md + } + pub fn register_locks(self: &Arc) { + let weak = Arc::downgrade(self); + crate::locks::register_lock_fn("bedrock-md::state", move || { + weak.upgrade().is_some_and(|md| md.state.is_locked()) + }); + let weak = Arc::downgrade(self); + crate::locks::register_lock_fn("bedrock-md::remapping", move || { + weak.upgrade().is_some_and(|md| md.remapping.is_locked()) + }); + let weak = Arc::downgrade(self); + crate::locks::register_lock_fn("bedrock-md::callback", move || { + weak.upgrade().is_some_and(|md| md.callback.is_locked()) + }); + } + pub fn physical(&self) -> &Arc { + self.memory.physical() + } + fn view(self: &Arc, view: View) -> Arc { + Arc::new(MdBus { + md: Arc::downgrade(self), + view, + #[cfg(feature = "jitv2")] + _generations: self.memory.generations(), + _memory: if matches!(view, View::Uncached) { + Some(self.physical().clone()) + } else { + None + }, + }) + } + pub fn md_bus(self: &Arc) -> Arc { + self.view(View::Registers) + } + pub fn directory_bus(self: &Arc) -> Arc { + self.view(View::Directory) + } + pub fn fetchop_bus(self: &Arc) -> Arc { + self.view(View::Fetchop) + } + pub fn ecc_bus(self: &Arc) -> Arc { + self.view(View::Ecc) + } + /// DMA uses the same Physical entries as the CPU, without accessing + /// ppmem's CPU-owned bitmap. Hardware protection remains unsupported. + pub fn dma_view(self: &Arc) -> Arc { + Arc::new(DmaView(self.clone())) + } + + fn mapping(s: &State) -> (u64, BTreeSet) { + let pages = s.ecc.keys().map(|key| key / (4096 / 8)) + .chain(s.marked_directory.iter().map(|key| key / (4096 / 128))) + .collect(); + (s.config, pages) + } + + /// Reinstall RAM after the board sets Physical's default. Call on the + /// CPU thread or with the CPU stopped, before installing board overlays. + /// Ordinary CPU config/diagnostic writes rebuild their changed mappings + /// automatically. This also releases pages whose metadata DMA consumed. + pub fn rebuild_memory(&self) { + let _remapping = self.remapping.lock(); + let (config, pages) = Self::mapping(&self.state.lock()); + self.memory.remap(config, pages, true); + } + /// The guest stored `tag` in a primary data cache tag: the PROM building + /// its stack in cache lines, or ending that. While they stand the window + /// answers from a page of its own on every path and stays out of the + /// direct window. See [`BedrockMemory::startup_tag`]. Called on the CPU + /// thread, outside the cache's locks. + pub fn startup_tag_store(&self, tag: u32) { + let Some(live) = self.memory.startup_tag(tag) else { return }; + let _remapping = self.remapping.lock(); + if self.memory.set_startup(live) { + let (config, pages) = Self::mapping(&self.state.lock()); + self.memory.remap(config, pages, true); + } + } + /// The stable logical-physical counters used by both compiled stores + /// and SysAd page claims. Their storage outlives the CPU via its bus. + #[cfg(feature = "jitv2")] + pub(crate) fn generation_base(&self) -> *mut AtomicU64 { + self.memory.generations().base() + } + pub fn on_error(&self, callback: ErrorCallback) { + *self.callback.lock() = Some(callback); + } + /// Explicit elapsed time for a manually clocked controller. + pub fn advance_ns(&self, elapsed: u64) { + let mut s = self.state.lock(); + Self::advance(&mut s, elapsed); + } + fn advance(s: &mut State, elapsed: u64) { + s.clock_ns = s.clock_ns.saturating_add(elapsed); + if s.selftest_deadline.is_some_and(|end| s.clock_ns >= end) { + s.selftest_deadline = None; + s.selftest_status = if s.selftest_result { 3 } else { 2 }; + s.selftest_error = if s.selftest_result { 0 } else { 1 << 48 }; + } + } + fn catch_up(s: &mut State, now: u64) { + if let Some(anchor) = s.clock_anchor { + s.clock_anchor = Some(anchor.max(now)); + Self::advance(s, now.saturating_sub(anchor)); + } + } + pub fn start_clock(&self) { + if let Some(clock) = &self.clock { + let now = clock(); + self.state.lock().clock_anchor.get_or_insert(now); + } + } + pub fn stop_clock(&self) { + if let Some(clock) = &self.clock { + let now = clock(); + let mut s = self.state.lock(); + Self::catch_up(&mut s, now); + s.clock_anchor = None; + } + } + /// Normal MD memory semantics without re-entering the cached Physical map. + pub fn memory_view(self: &Arc) -> Arc { + self.view(View::Memory) + } + pub fn uncached_view(self: &Arc) -> Arc { + self.view(View::Uncached) + } + pub fn ecc_scrub_bus(self: &Arc) -> Arc { + self.view(View::Scrub) + } + pub fn current_cell(&self) -> u64 { + self.state.lock().cell + } + pub fn unsupported(&self) -> u64 { + self.unsupported.load(Ordering::Relaxed) + } + /// Count and log an access the model does not implement; the caller + /// answers a bus error. + fn refuse(&self, what: &str, at: u64) -> Option { + self.unsupported.fetch_add(1, Ordering::Relaxed); + crate::dlog!(crate::devlog::LogModule::Bedrock, + "Bedrock MD refused {what} access at {at:#x}"); + None + } + /// Called on the CPU thread or with the CPU stopped. + pub fn soft_reset(&self) { + let _remapping = self.remapping.lock(); + let mut s = self.state.lock(); + s.cell = u64::MAX; + s.storage.clear(); + s.ecc_config = 0; + s.memory_error = 0; + s.directory_error = 0; + s.ecc.clear(); + s.marked_directory.clear(); + s.selftest_deadline = None; + s.selftest_status = 0; + s.selftest_error = 0; + s.refresh &= (1 << 63) | 0xfff; + let (config, pages) = Self::mapping(&s); + drop(s); + self.memory.remap(config, pages, false); + } + fn access(&self, view: View, addr: u64, bytes: u64, write: Option) -> Option { + if ![1, 2, 4, 8].contains(&bytes) || addr % bytes != 0 { + return self.refuse("access", addr); + } + // Only CPU register/configuration and diagnostic writes can add + // metadata. Keep their remap off DMA threads: ordinary RAM, scrub + // and fetch operations only remove entries, leaving safe slow pages + // behind until a CPU-side rebuild. These write views, like HEART's + // MEMCFG, are CPU-thread-only while the machine runs. + let publish = write.is_some() && match view { + View::Registers => matches!(addr, reg::CONFIG | reg::SELFTEST_COMMAND), + View::Directory | View::Ecc => true, + _ => false, + }; + let remapping = publish.then(|| self.remapping.lock()); + let mut event = Vec::new(); + // Sample outside state: even an injected clock must not call out under + // the memory lock. Only register observations need the self-test timer. + let now = if matches!(view, View::Registers) { + self.clock.as_ref().map(|clock| clock()) + } else { + None + }; + let (result, snapshot) = { + let mut s = self.state.lock(); + if let Some(now) = now { + Self::catch_up(&mut s, now); + } + let result = match view { + View::Registers => self.register(&mut s, addr, bytes, write), + View::Directory => self.directory(&mut s, addr, bytes, write), + View::Ecc => self.ecc(&mut s, addr, bytes, write), + View::Scrub => self.scrub(&mut s, addr, bytes, write, &mut event), + View::Uncached => { + let result = self.ram(&mut s, addr, bytes, write, &mut event); + if write.is_none() && event.contains(&MemoryError::Uncorrectable) { + None + } else { + result + } + } + View::Fetchop => self.fetchop(&mut s, addr, bytes, write, &mut event), + View::Memory => self.ram(&mut s, addr, bytes, write, &mut event), + }; + (result, publish.then(|| Self::mapping(&s))) + }; + if let Some((config, pages)) = snapshot { + self.memory.remap(config, pages, false); + } + drop(remapping); + if !event.is_empty() { + let callback = self.callback.lock().clone(); + if let Some(callback) = callback { + for kind in event { + callback(kind); + } + } + } + result + } + fn register(&self, s: &mut State, at: u64, bytes: u64, write: Option) -> Option { + if bytes != 8 { + return self.refuse("register", at); + } + let old = match at { + reg::CELL => s.cell, + reg::CONFIG => s.config, + reg::REPLY_QUEUE_SIZE => reg::REPLY_QUEUE_SIZES, + reg::REFRESH => s.refresh, + reg::SELFTEST_ERROR => s.selftest_error, + reg::SELFTEST_STATUS => s.selftest_status, + reg::DIRECTORY_CONFIG => s.directory_config, + reg::ECC_CONFIG => s.ecc_config, + reg::REQUEST_QUEUE_SIZE => reg::REQUEST_QUEUE_SIZES, + reg::MEMORY_ERROR | reg::MEMORY_ERROR_ALIAS => s.memory_error, + reg::DIRECTORY_ERROR | reg::DIRECTORY_ERROR_ALIAS => s.directory_error, + _ if storage_mask(at).is_some() => *s.storage.get(&at).unwrap_or(&0), + _ if idle(at) => 0, + _ => return self.refuse("register", at), + }; + if let Some(value) = write { + match at { + reg::SELFTEST_COMMAND => return self.selftest(s, value), + reg::CELL => s.cell = value, + reg::CONFIG => { + s.config = value & CONFIG_MASK; + } + reg::REFRESH => s.refresh = value & ((1 << 63) | 0xfff), + reg::DIRECTORY_CONFIG => s.directory_config = value & 3, + reg::ECC_CONFIG => s.ecc_config = value & 1, + _ if storage_mask(at).is_some() => { + s.storage.insert(at, value & storage_mask(at).unwrap()); + } + _ => { + self.unsupported.fetch_add(1, Ordering::Relaxed); + crate::dlog!(crate::devlog::LogModule::Bedrock, + "Bedrock MD ignored register write at {at:#x}: {value:#x}"); + } + } + } else if at == reg::MEMORY_ERROR_ALIAS { + s.memory_error = 0; + } else if at == reg::DIRECTORY_ERROR_ALIAS { + s.directory_error = 0; + } + Some(old) + } + fn directory(&self, s: &mut State, at: u64, bytes: u64, write: Option) -> Option { + if bytes != 8 || at >= 1 << 30 { + return self.refuse("directory", at); + } + let software = at / (BANK_SPAN / 8); + let bank = software ^ ((s.config >> 32) & 3); + if at & 0x7c8 == 0x408 { + if bank >= 2 { + return Some(0); + } + let within = + ((((at % (BANK_SPAN / 8)) >> 11) << 14) | (((at >> 4) & 3) << 7)) % FITTED_BANK; + let key = ((bank * FITTED_BANK + within) >> 14) * 4 + ((within >> 7) & 3); + let old = *s.protection.get(&key).unwrap_or(&0); + if let Some(v) = write { + s.protection.insert(key, v); + } + return Some(old); + } + let within = ((at % (BANK_SPAN / 8)) & !15) * 8; + let field = s.config >> (bank / 2 * 8); + let code = (field >> 4) & 15; + if field & (1 << (bank & 1)) == 0 + || !(1..=5).contains(&code) + || within >= (32 << 20) << code + { + return self.refuse("directory", at); + } + let mask = if at & 8 != 0 || s.directory_config & 1 != 0 { + u64::MAX + } else { + u32::MAX as u64 + }; + let table; + let key; + if bank >= 2 || within >= FITTED_BANK { + if !s.compat.contains_key(&at) && s.compat.len() >= 74 { + return self.refuse("directory", at); + } + s.compat.entry(at).or_insert(0); + table = &mut s.compat; + key = at; + } else { + key = (bank * FITTED_BANK + within) / 128; + table = if at & 8 != 0 { + &mut s.counters + } else { + &mut s.directory + }; + } + let old = *table.get(&key).unwrap_or(&0) & mask; + if let Some(value) = write { + let value = if at & 8 == 0 && s.directory_config & 2 == 0 { + directory_encode(value, s.directory_config & 1 != 0) + } else { + value + }; + table.insert(key, value & mask); + if bank < 2 && within < FITTED_BANK && at & 8 == 0 { + if directory_syndrome(value, s.directory_config & 1 != 0) != 0 { + s.marked_directory.insert(key); + } else { + s.marked_directory.remove(&key); + } + } + } + Some(old) + } + fn checked_word( + &self, + s: &mut State, + bank: usize, + at: u64, + address: u64, + event: &mut Vec, + ) -> u64 { + let mut data = self.memory.raw_read(bank, at); + if s.ecc_config & 1 != 0 || s.ecc.is_empty() { + return data; + } + let key = (bank as u64 * FITTED_BANK + at) / 8; + if let Some(&check) = s.ecc.get(&key) { + let syndrome = check ^ ecc_compute(data); + if syndrome != 0 { + let bit = ECC_COLUMNS.iter().position(|&v| v == syndrome); + let valid = if bit.is_some() { 1 << 52 } else { 1 << 60 }; + if bit.is_none() + && s.memory_error & (1 << 60) != 0 + && (s.memory_error ^ address) & (((1u64 << 30) - 1) << 3) & !127 != 0 + { + s.memory_error |= 1 << 63; + } + if s.memory_error & (valid | (1 << 60)) != 0 { + s.memory_error |= valid << 1; + } else { + s.memory_error = (s.memory_error + & !((0xff << 40) | (((1u64 << 30) - 1) << 3))) + | ((syndrome as u64) << 40) + | (address & (((1u64 << 30) - 1) << 3)); + } + s.memory_error |= valid; + if let Some(bit) = bit { + if bit < 64 { + data ^= 1 << bit; + self.memory.raw_write(bank, at, data); + } + s.ecc.remove(&key); + event.push(MemoryError::Correctable); + } else { + event.push(MemoryError::Uncorrectable); + } + } + } + data + } + fn ram( + &self, + s: &mut State, + address: u64, + bytes: u64, + write: Option, + event: &mut Vec, + ) -> Option { + if let Some(page) = self.memory.startup_device(address, bytes) { + return Some(startup_access(page, address, bytes, write)); + } + let Some((bank, offset)) = BedrockMemory::decode(s.config, address, bytes) else { + return self.refuse("ram", address); + }; + if !s.marked_directory.is_empty() { + let line = (bank as u64 * FITTED_BANK + offset) / 128; + if s.marked_directory.contains(&line) { + let premium = s.directory_config & 1 != 0; + let word = *s.directory.get(&line).unwrap_or(&0); + if write.is_some() { + s.directory.insert(line, directory_encode(word, premium)); + s.marked_directory.remove(&line); + } else if s.directory_config & 2 == 0 { + let syndrome = directory_syndrome(word, premium); + if syndrome != 0 { + let bit = directory_columns(premium) + .iter() + .position(|&v| v == syndrome); + let valid = if bit.is_some() { 1 << 61 } else { 1 << 63 }; + if s.directory_error & ((1 << 63) | valid) != 0 { + s.directory_error |= if bit.is_some() { 1 << 57 } else { 1 << 59 }; + } else { + let carrier = 0x1c0000000 + + (address / BANK_SPAN) * (BANK_SPAN / 8) + + ((offset & !127) / 8); + s.directory_error = (s.directory_error + & !((0x7f << 40) | (((1u64 << 30) - 1) << 3))) + | ((syndrome as u64) << 40) + | (carrier & (((1u64 << 30) - 1) << 3)); + } + s.directory_error |= valid; + if let Some(bit) = bit { + s.directory.insert(line, word ^ (1 << bit)); + s.marked_directory.remove(&line); + event.push(MemoryError::Correctable); + } else { + event.push(MemoryError::Uncorrectable); + return None; + } + } + } + } + } + let at = offset & !7; + let shift = (8 - (offset & 7) - bytes) * 8; + let mask = u64::MAX >> ((8 - bytes) * 8); + if let Some(value) = write { + let old = self.memory.raw_read(bank, at); + self.memory.raw_write( + bank, + at, + (old & !(mask << shift)) | ((value & mask) << shift), + ); + if !s.ecc.is_empty() { + s.ecc.remove(&((bank as u64 * FITTED_BANK + at) / 8)); + } + Some(0) + } else { + Some((self.checked_word(s, bank, at, address, event) >> shift) & mask) + } + } + fn fetchop( + &self, + s: &mut State, + at: u64, + bytes: u64, + write: Option, + event: &mut Vec, + ) -> Option { + if bytes != 4 && bytes != 8 { + return self.refuse("fetchop", at); + } + let op = at & 0x38; + if op > if write.is_some() { 32 } else { 24 } { + return self.refuse("fetchop", at); + } + let address = (at & !63) | (at & 4); + let old = self.ram(s, address, bytes, None, event)?; + let next = match (write, op) { + (None, 0) => return Some(old), + (Some(v), 0) => v, + (_, 8) => old.wrapping_add(1), + (_, 16) => old.wrapping_sub(1), + (None, 24) => 0, + (Some(v), 24) => old & v, + (Some(v), 32) => old | v, + _ => return self.refuse("fetchop", at), + }; + self.ram(s, address, bytes, Some(next), event)?; + Some(old) + } + // Approximation: report whether the selected fitting covers the + // programmed bank, after 1000 guest ns. Only the zero write phase mutates + // memory; other patterns have no published transformation here. + fn selftest(&self, s: &mut State, command: u64) -> Option { + if command == 2 || command == 4 { + s.selftest_deadline = None; + s.selftest_status = 0; + s.selftest_error = 0; + return Some(0); + } + if command & 7 != 1 || command & !0x71 != 0 { + self.unsupported.fetch_add(1, Ordering::Relaxed); + crate::dlog!(crate::devlog::LogModule::Bedrock, + "Bedrock MD ignored self-test command {command:#x}"); + return Some(0); + } + let selected = (command >> 4) & 7; + let configured = selected ^ ((s.config >> 32) & 3); + let field = s.config >> ((configured / 2) * 8); + let code = (field >> 4) & 15; + let size = if (1..=5).contains(&code) { + (32 << 20) << code + } else { + 0 + }; + let fits = selected < 2 + && field & (1 << (configured & 1)) != 0 + && size != 0 + && size <= FITTED_BANK; + s.selftest_result = fits; + s.selftest_status = 0; + s.selftest_error = 0; + s.selftest_deadline = Some(s.clock_ns.saturating_add(1000)); + if fits && s.storage.get(&0x20038).copied().unwrap_or(0) == 0x80 { + self.memory.zero(selected as usize, size); + let start = selected * FITTED_BANK; + s.ecc + .retain(|key, _| !(*key >= start / 8 && *key < (start + size) / 8)); + let lines: Vec = s + .marked_directory + .iter() + .copied() + .filter(|&line| line >= start / 128 && line < (start + size) / 128) + .collect(); + for line in lines { + let word = *s.directory.get(&line).unwrap_or(&0); + s.directory + .insert(line, directory_encode(word, s.directory_config & 1 != 0)); + s.marked_directory.remove(&line); + } + } + Some(0) + } + fn scrub( + &self, + s: &mut State, + at: u64, + bytes: u64, + write: Option, + event: &mut Vec, + ) -> Option { + if write.is_none() { + return self.ecc(s, at, bytes, None); + } + if bytes != 8 || at >= 1 << 30 { + return self.refuse("scrub", at); + } + let software = at / (BANK_SPAN / 8); + let within = at % (BANK_SPAN / 8); + let address = software * BANK_SPAN + within * 8; + let Some((bank, offset)) = BedrockMemory::decode(s.config, address, 8) else { + return self.refuse("scrub", at); + }; + for i in 0..8 { + if BedrockMemory::decode(s.config, address + i * 8, 8).is_some() { + self.checked_word(s, bank, offset + i * 8, address + i * 8, event); + } + } + Some(0) + } + fn ecc(&self, s: &mut State, at: u64, bytes: u64, write: Option) -> Option { + if at >= 1 << 30 || (bytes != 1 && bytes != 2) { + return self.refuse("ecc", at); + } + let software = at / (BANK_SPAN / 8); + let within = (at % (BANK_SPAN / 8)) ^ if bytes == 1 { 7 } else { 6 }; + let address = software * BANK_SPAN + within * 8; + let Some((bank, offset)) = BedrockMemory::decode(s.config, address, bytes * 8) else { + if bytes == 2 && [0x04000006, 0x08000006].contains(&at) { + let i = usize::from(at == 0x08000006); + let old = s.admitted[i]; + if let Some(value) = write { + s.admitted[i] = value as u16; + } + return Some(old as u64); + } + return self.refuse("ecc", at); + }; + let mut old = 0; + for i in 0..bytes { + let raw = offset + i * 8; + let key = (bank as u64 * FITTED_BANK + raw) / 8; + let computed = ecc_compute(self.memory.raw_read(bank, raw)); + old |= (s.ecc.get(&key).copied().unwrap_or(computed) as u64) << (i * 8); + if let Some(value) = write { + let code = (value >> (i * 8)) as u8; + if code == computed { + s.ecc.remove(&key); + } else { + s.ecc.insert(key, code); + } + } + } + Some(old) + } +} +impl Resettable for BedrockMd { + fn power_on(&self) { + // Reset is called on the CPU thread or with it stopped. + let _remapping = self.remapping.lock(); + *self.state.lock() = State::default(); + let was_live = self.memory.startup_live(); + self.memory.clear_startup(); + self.memory.remap(0, BTreeSet::new(), was_live); + } +} +/// Resolve each DMA access through the atomic Physical table, as HEART +/// does. Do not ask Physical for mem_ptr or gen_ptr: those read the inline +/// CPU bitmap. Default block methods split accesses and respect slow pages +/// even if a caller supplies a block crossing a 4 KiB page boundary. +struct DmaView(Arc); +impl DmaView { + fn at(&self, address: u64, bytes: u64) -> Option<&dyn BusDevice> { + if let Some(page) = self.0.memory.startup_device(address, bytes) { + return Some(page); + } + BedrockMemory::decode(self.0.memory.config(), address, bytes)?; + let phys = self.0.physical(); + let entry = phys.entry_at(address); + if crate::ppmem::devmap::entry_eq(entry, phys.default_entry()) { + return None; + } + // SAFETY: the owning MD retains Physical, which keeps every entry + // alive across remaps, including the weak MD slow endpoint. + Some(unsafe { &*entry }) + } +} +macro_rules! dma_rw { + ($read:ident, $write:ident, $result:ident, $ty:ty) => { + fn $read(&self, address: u64) -> $result { + self.at(address, std::mem::size_of::<$ty>() as u64) + .map_or($result::err(), |device| device.$read(address)) + } + fn $write(&self, address: u64, value: $ty) -> u32 { + self.at(address, std::mem::size_of::<$ty>() as u64) + .map_or(BUS_ERR, |device| { + let result = device.$write(address, value); + if result == BUS_OK { self.0.memory.written(address); } + result + }) + } + }; +} +impl BusDevice for DmaView { + dma_rw!(read8, write8, BusRead8, u8); + dma_rw!(read16, write16, BusRead16, u16); + dma_rw!(read32, write32, BusRead32, u32); + dma_rw!(read64, write64, BusRead64, u64); + #[cfg(feature = "jitv2")] + fn gen_ptr(&self, address: u64) -> *const AtomicU64 { + self.0.memory.gen_ptr(address) + } +} + +/// A read or write of `bytes` at `address` in the startup page. The page is +/// plain memory: no ECC, no directory, no fetch-and-op. +fn startup_access(page: &dyn BusDevice, address: u64, bytes: u64, write: Option) -> u64 { + match (bytes, write) { + (1, None) => page.read8(address).data as u64, + (2, None) => page.read16(address).data as u64, + (4, None) => page.read32(address).data as u64, + (_, None) => page.read64(address).data, + (1, Some(v)) => { page.write8(address, v as u8); 0 } + (2, Some(v)) => { page.write16(address, v as u16); 0 } + (4, Some(v)) => { page.write32(address, v as u32); 0 } + (_, Some(v)) => { page.write64(address, v); 0 } + } +} + +impl BusDevice for MdBus { + /// Bedrock's ECC view is the one device that sees the whole source register + /// of an uncached SH (`BusDevice::write_narrow_from`); every other view + /// takes the architectural low bits. + fn write_narrow_from(&self, addr: u64, size: usize, gpr: u64) -> u32 { + if size == 2 && matches!(self.view, View::Ecc) { + // PROM 6.210's memory test stores each check word with `sh` from a + // register whose low 16 bits are zero: the word sits elsewhere in + // the register, by the address of the memory it checks. Observed + // (memory address mod 0x40, carrier offset mod 8, register bits): + // 0x00 6 15:0 + // 0x10 4 63:48 + // 0x20 2 47:32 + // 0x30 0 31:16 + // The shift below gives exactly these four. Which lanes the R14000 + // drives for a narrow store, and how Bedrock wires them to the + // check bits, is unknown; this matches the PROM, not a hardware + // description. + let memory_address = (addr ^ 6).wrapping_mul(8); + return self.write16(addr, (gpr >> (memory_address.wrapping_neg() & 63)) as u16); + } + if size == 1 { self.write8(addr, gpr as u8) } else { self.write16(addr, gpr as u16) } + } + + #[cfg(feature = "jitv2")] + fn gen_ptr(&self, address: u64) -> *const std::sync::atomic::AtomicU64 { + let Some(md) = self.md.upgrade() else { + return std::ptr::null(); + }; + if !matches!(self.view, View::Memory | View::Uncached) { + return std::ptr::null(); + } + md.memory.gen_ptr(address) + } + + fn read8(&self, a: u64) -> BusRead8 { + self.md + .upgrade() + .and_then(|m| m.access(self.view, a, 1, None)) + .map_or(BusRead8::err(), |v| BusRead8::ok(v as u8)) + } + fn read16(&self, a: u64) -> BusRead16 { + self.md + .upgrade() + .and_then(|m| m.access(self.view, a, 2, None)) + .map_or(BusRead16::err(), |v| BusRead16::ok(v as u16)) + } + fn read32(&self, a: u64) -> BusRead32 { + self.md + .upgrade() + .and_then(|m| m.access(self.view, a, 4, None)) + .map_or(BusRead32::err(), |v| BusRead32::ok(v as u32)) + } + fn read64(&self, a: u64) -> BusRead64 { + self.md + .upgrade() + .and_then(|m| m.access(self.view, a, 8, None)) + .map_or(BusRead64::err(), BusRead64::ok) + } + fn write8(&self, a: u64, v: u8) -> u32 { + self.md + .upgrade() + .and_then(|m| m.access(self.view, a, 1, Some(v as u64))) + .map_or(BUS_ERR, |_| BUS_OK) + } + fn write16(&self, a: u64, v: u16) -> u32 { + self.md + .upgrade() + .and_then(|m| m.access(self.view, a, 2, Some(v as u64))) + .map_or(BUS_ERR, |_| BUS_OK) + } + fn write32(&self, a: u64, v: u32) -> u32 { + self.md + .upgrade() + .and_then(|m| m.access(self.view, a, 4, Some(v as u64))) + .map_or(BUS_ERR, |_| BUS_OK) + } + fn write64(&self, a: u64, v: u64) -> u32 { + self.md + .upgrade() + .and_then(|m| m.access(self.view, a, 8, Some(v))) + .map_or(BUS_ERR, |_| BUS_OK) + } +} +fn storage_mask(at: u64) -> Option { + Some(match at { + reg::ARBITRATION => 0xff, + reg::MIGRATION_CONFIG => 0x3ff | (0xff << 16) | (1 << 32), + reg::PERF_SELECT0 | reg::PERF_SELECT1 => { + 3 | (15 << 4) | (127 << 8) | (15 << 16) | (0xffffffffff << 20) | (1 << 63) + } + reg::PERF_COUNTER0 | reg::PERF_COUNTER1 => (1 << 41) - 1, + reg::JUNK_BUS_TIMING => 0xffffffffffff, + reg::SELFTEST_DATA => 0xff, + reg::ISSUE_DEBUG => 3, + reg::MIGRATION_DIFFERENCE => 0x7fff | (7 << 32) | (7 << 35) | (1 << 63), + reg::MIGRATION_VALUE => 0x7fff | (7 << 32) | (7 << 35) | (3 << 62), + reg::DIRECTORY_DEBUG => 3, + reg::MEMORY_DEBUG => 1, + _ => return None, + }) +} +fn idle(at: u64) -> bool { + matches!( + at, + 0x20 | 0x28 + | 0x40 + | 0x48 + | 0x58 + | 0x20010 + | 0x20018 + | 0x20020 + | 0x20028 + | 0x20030 + | 0x20040 + | 0x20048 + | 0x40020 + | 0x40028 + | 0x40030 + | 0x40038 + | 0x40058 + | 0x60020 + | 0x60028 + | 0x60030 + | 0x60038 + ) +} + +#[cfg(test)] +mod tests { + // Cases follow tests/qtest/sn1-bedrock-md-test.c in + // jamesbraid/qemu-sgi-origami: memory-banks, reset, queue-sizes, + // writable-masks, selftest-phases, fetchop-operations, ecc-carrier and + // directory-storage. + use super::*; + fn configured() -> Arc { + let md = BedrockMd::new(); + assert_eq!(md.md_bus().write64(8, 0x33), BUS_OK); + md + } + + /// What PROM 6.210 stores as primary data cache tags (TagLo), measured: + /// test patterns, then an invalidate, then a valid tag per page of the + /// window, then the invalidate sweep it leaves. + const WINDOW_TAGS: [u32; 8] = [0x100c0, 0x101c1, 0x102c0, 0x103c1, 0x104c0, 0x105c1, 0x106c0, 0x107c1]; + + /// A valid tag for a page of the window starts a private page that every + /// path reaches, whether or not DRAM is configured; an invalid tag ends it + /// and leaves memory as it was. + #[test] + fn a_valid_tag_for_the_window_starts_a_page_every_path_answers() { + let md = BedrockMd::new(); + let phys = md.physical(); + let (dma, uncached) = (md.dma_view(), md.uncached_view()); + let at = 0x10_0008 + 0x7000; + assert_eq!(phys.read64(at).status, BUS_ERR, "no memory behind the window yet"); + md.startup_tag_store(WINDOW_TAGS[0]); + assert_eq!(phys.write64(at, 0x1122_3344_5566_7788), BUS_OK); + assert_eq!(phys.read64(at).data, 0x1122_3344_5566_7788); + assert_eq!(dma.read64(at).data, 0x1122_3344_5566_7788, "the cached path and DMA"); + assert_eq!(uncached.read32(at + 4).data, 0x5566_7788, "the uncached path"); + assert_eq!(uncached.write8(at, 0xaa), BUS_OK); + assert_eq!(phys.read64(at).data, 0xaa22_3344_5566_7788); + assert_eq!(phys.read64(0x10_8000).status, BUS_ERR, "only the window's 32 KiB"); + // The page wraps in itself, and the bank behind it is untouched. + assert_eq!(md.md_bus().write64(reg::CONFIG, 0x33), BUS_OK); + assert_eq!(phys.read64(at).data, 0xaa22_3344_5566_7788, "configuring DRAM keeps the page"); + assert_eq!(phys.banks()[0].read64(at).data, 0, "while it stands, memory is not the page"); + md.startup_tag_store(0); + assert_eq!(phys.read64(at).data, 0, "the window is memory again"); + assert_eq!(phys.banks()[0].read64(at).data, 0); + } + + /// Test patterns, a tag outside the window, and a kernel's invalid tags + /// leave the decode alone; only the window's valid tags start the page. + #[test] + fn other_tags_leave_the_window_to_memory() { + let md = BedrockMd::new(); + for tag in [0xffff_ffcf, 0xaaaa_aa8a, 0x5555_5545, 0, 0x108c0, 0xff00c0, 0x1000c0 >> 4] { + md.startup_tag_store(tag); + assert!(!md.memory.startup_live(), "{tag:#x}"); + } + assert_eq!(md.physical().read64(0x10_0000).status, BUS_ERR); + // The tag's node bits above the node's 8 GiB do not move the window. + md.startup_tag_store(0x2001_00c0); + assert!(md.memory.startup_live()); + } + + /// The whole PROM pattern: the page starts at the first install, stays + /// through the rest and across a configuration write, and ends at the + /// first invalid tag of the final sweep. + #[test] + fn the_proms_tag_sequence_starts_and_ends_the_page_once() { + let md = BedrockMd::new(); + let phys = md.physical(); + for tag in [0xffff_ffcf, 0xaaaa_aa8a, 0x5555_5545, 0] { + md.startup_tag_store(tag); + } + assert!(!md.memory.startup_live()); + for tag in WINDOW_TAGS { + md.startup_tag_store(tag); + assert!(md.memory.startup_live()); + } + assert_eq!(phys.write64(0x10_0000, 7), BUS_OK); + md.md_bus().write64(reg::CONFIG, 0x33); + md.soft_reset(); + assert_eq!(phys.read64(0x10_0000).data, 7, "a controller soft reset is not a power-on"); + md.startup_tag_store(0); + assert!(!md.memory.startup_live()); + md.startup_tag_store(0); + assert!(!md.memory.startup_live()); + // A reboot through the PROM tags again, and the cache still holds + // what the page held. + md.startup_tag_store(WINDOW_TAGS[7]); + assert!(md.memory.startup_live()); + assert_eq!(phys.read64(0x10_0000).data, 7); + } + + /// The final sweep stores 2050 invalid tags; the page ends at the first, + /// and the PROM's later stores to the window reach memory, not the page. + #[test] + fn the_page_ends_at_the_first_invalid_tag_of_the_final_sweep() { + let md = configured(); + let phys = md.physical(); + for tag in WINDOW_TAGS { + md.startup_tag_store(tag); + } + assert_eq!(phys.write64(0x10_0000, 5), BUS_OK); + md.startup_tag_store(0); + assert!(!md.memory.startup_live(), "the first invalid store ends it"); + assert_eq!(phys.read64(0x10_0000).data, 0); + assert_eq!(phys.write64(0x10_0000, 6), BUS_OK); + for _ in 1..2050 { + md.startup_tag_store(0); + } + assert!(!md.memory.startup_live()); + assert_eq!(phys.read64(0x10_0000).data, 6, "memory, untouched by the rest of the sweep"); + } + + /// While the page stands its 64 MiB region is out of the direct window, + /// so cached loads and the JIT reach it through the decode; the rest of + /// the region comes back with the page's end. + #[test] + fn the_page_keeps_its_region_out_of_the_direct_window() { + use crate::ppmem::MappedMemory; + let md = configured(); + let space = md.physical().ppmem_space().unwrap(); + assert_ne!(space.mapped_bitmap() & 1, 0, "configured memory is in the window"); + md.startup_tag_store(WINDOW_TAGS[0]); + assert_eq!(space.mapped_bitmap() & 1, 0, "while the page stands"); + assert_ne!(space.mapped_bitmap() & 2, 0, "the next region is not affected"); + md.startup_tag_store(0); + assert_ne!(space.mapped_bitmap() & 1, 0, "and it returns"); + } + + /// Power-on drops the page, zeroed, and its decode. + #[test] + fn power_on_clears_the_page() { + use crate::traits::Resettable; + let md = configured(); + md.startup_tag_store(WINDOW_TAGS[0]); + assert_eq!(md.physical().write64(0x10_0000, 9), BUS_OK); + md.power_on(); + assert!(!md.memory.startup_live()); + assert_eq!(md.physical().read64(0x10_0000).status, BUS_ERR, "no memory until configured"); + md.startup_tag_store(WINDOW_TAGS[0]); + assert_eq!(md.physical().read64(0x10_0000).data, 0, "the page starts zero"); + } + + /// Ordinary CPU and DMA RAM accesses remain available while MD state is locked. + #[test] + fn ordinary_cpu_and_dma_memory_do_not_take_the_md_lock() { + let md = configured(); + let dma = md.dma_view(); + let _state = md.state.lock(); + assert_eq!(md.physical().write64(0x1000, 0x1122334455667788), BUS_OK); + assert_eq!(md.physical().read64(0x1000).data, 0x1122334455667788); + assert_eq!(dma.read64(0x1000).data, 0x1122334455667788); + assert_eq!(dma.write32(0x1004, 0xabcdef01), BUS_OK); + assert_eq!(md.physical().read64(0x1000).data, 0x11223344abcdef01); + assert_eq!(dma.read64(FITTED_BANK).status, BUS_ERR); + } + + /// An ECC injection removes its direct mapping; DMA repair keeps the page slow. + #[test] + fn injected_checks_disable_the_window_and_dma_repair_keeps_the_slow_page() { + use crate::ppmem::MappedMemory; + let md = configured(); + let phys = md.physical(); + let space = phys.ppmem_space().expect("a window"); + assert_eq!(space.mapped_bitmap() & 3, 3); + assert!(phys.mem_ptr(0).is_some()); + assert_eq!(md.ecc_bus().write8(7, 0x13), BUS_OK); + assert_eq!(space.mapped_bitmap() & 3, 2, "the full 64 MiB region loses its direct bit"); + assert!(phys.mem_ptr(0).is_none()); + let slow = phys.entry_at(0); + assert!(!crate::ppmem::devmap::entry_eq(slow, phys.entry_at(0x1000))); + let dma = md.dma_view(); + let repaired = std::thread::spawn(move || dma.read64(0)).join().unwrap(); + assert_eq!(repaired.status, BUS_OK); + assert_eq!(repaired.data, 1); + assert!(md.state.lock().ecc.is_empty()); + assert_eq!(space.mapped_bitmap() & 3, 2, "a DMA thread never republishes the CPU bitmap"); + assert!(crate::ppmem::devmap::entry_eq(slow, phys.entry_at(0))); + // Adjacent pages still use the ordinary bank entry, even though + // the region no longer has a direct window mapping. + { + let _state = md.state.lock(); + assert_eq!(phys.write64(0x1000, 9), BUS_OK); + assert_eq!(phys.read64(0x1000).data, 9); + } + md.soft_reset(); + assert_eq!(space.mapped_bitmap() & 3, 3); + assert!(phys.mem_ptr(0).is_some()); + assert_eq!(phys.read64(0).data, 1); + } + + /// Directory-marked pages follow their physical bank after a configuration swap. + #[test] + fn marked_directory_pages_follow_their_bank_when_swizzled() { + use crate::ppmem::MappedMemory; + let md = configured(); + let phys = md.physical(); + let space = phys.ppmem_space().expect("a window"); + md.md_bus().write64(reg::DIRECTORY_CONFIG, 2); + md.directory_bus().write64(0, 0x001e0001); + assert_eq!(space.mapped_bitmap() & 1, 0); + assert!(phys.mem_ptr(0).is_none()); + md.md_bus().write64(reg::CONFIG, 0x33 | (1 << 32)); + assert_ne!(space.mapped_bitmap() & 1, 0, "the other bank now answers at zero"); + assert_eq!(space.mapped_bitmap() & (1 << 16), 0); + assert!(phys.mem_ptr(BANK_SPAN).is_none()); + md.md_bus().write64(reg::DIRECTORY_CONFIG, 0); + assert_eq!(phys.read64(BANK_SPAN).status, BUS_OK); + assert_ne!(md.md_bus().read64(reg::DIRECTORY_ERROR).data & (1 << 61), 0); + md.rebuild_memory(); + assert!(phys.mem_ptr(BANK_SPAN).is_some()); + md.power_on(); + assert_eq!(space.mapped_bitmap(), 0); + assert_eq!(phys.read64(BANK_SPAN).status, BUS_ERR); + } + + /// Guest code must not keep a compiled page after its backing bank changes. + /// page_for deliberately retains its generation pointer. A remap must + /// invalidate that page and subsequent writes to its new bank must keep + /// invalidating the same pointer, including after swapping back. + #[cfg(feature = "jitv2")] + #[test] + fn a_claimed_jit_page_tracks_physical_ownership_across_bank_swaps() { + use crate::cpu::jitv2::jitv2::Jitv2; + let md = configured(); + let dma = md.dma_view(); + let mut jit = Jitv2::new(4); + let slot = jit.page_for(0, 0, dma.as_ref(), false).unwrap(); + let page = jit.page_ptr(slot); + let generation = || unsafe { (*page).current_gen() }; + let pointer = dma.gen_ptr(0); + let mut entries = [0u64; crate::cpu::jitv2::jitv2::BITMAP_WORDS]; + entries[0] = 1; + // Exercise the JIT publication/invalidation protocol without + // executing a synthetic function body. + let code = std::ptr::NonNull::<()>::dangling().as_ptr() as *const (); + assert!(unsafe { (*page).publish(&entries, code, generation(), 1, 0, false) }); + assert!(unsafe { (*page).is_runnable(0) }); + let first = generation(); + md.md_bus().write64(reg::CONFIG, 0x33 | (1 << 32)); + assert_eq!(jit.page_for(0, 0, dma.as_ref(), false), Some(slot)); + assert_eq!(dma.gen_ptr(0), pointer); + assert!(generation() > first); + assert!(!unsafe { (*page).is_runnable(0) }); + let swapped = generation(); + assert!(unsafe { (*page).publish(&entries, code, swapped, 1, 0, false) }); + assert!(unsafe { (*page).is_runnable(0) }); + assert_eq!(dma.write64(0, 0x11223344), BUS_OK); + assert!(generation() > swapped); + assert!(!unsafe { (*page).is_runnable(0) }, "code published after a swap sees new-bank writes"); + assert_eq!(md.physical().banks()[1].read64(0).data, 0x11223344); + let written = generation(); + md.md_bus().write64(reg::CONFIG, 0x33); + assert!(generation() > written, "swapping back cannot lower the logical generation"); + let restored = generation(); + md.uncached_view().write64(0, 0x55667788); + assert!(generation() > restored); + let before_fetchop = generation(); + md.fetchop_bus().write64(0, 0); + assert!(generation() > before_fetchop); + md.ecc_bus().write8(7, 0x13); + let before_repair = generation(); + assert_eq!(dma.read64(0).data, 1); + assert!(generation() > before_repair, "ECC repair invalidates logical code pages"); + md.md_bus().write64(reg::CONFIG, 0x13); + md.md_bus().write64(0x20038, 0x80); + let before_zero = generation(); + md.md_bus().write64(reg::SELFTEST_COMMAND, 1); + assert!(generation() > before_zero, "zero self-test invalidates logical code pages"); + let before_disable = generation(); + md.power_on(); + assert!(generation() > before_disable); + assert_eq!(dma.gen_ptr(0), pointer, "a disabled bank keeps its logical counter alive"); + } + + /// Guest density settings remap fitted RAM, and reset restores the default map. + #[test] + fn bank_configuration_remaps_memory_and_reset_restores_defaults() { + let md = configured(); + let bus = md.physical(); + assert_eq!(bus.write64(0, 0x1122334455667788), BUS_OK); + assert_eq!(bus.write64(BANK_SPAN, 0xabcdef), BUS_OK); + assert_eq!(bus.read8(0).data, 0x11); + assert_eq!(bus.read32(4).data, 0x55667788); + assert_eq!(bus.read64(FITTED_BANK).status, BUS_ERR); + assert_eq!(bus.read64(2 * BANK_SPAN).status, BUS_ERR); + assert_eq!(md.md_bus().write64(8, 0x33 | 1 << 32), BUS_OK); + assert_eq!(bus.read64(0).data, 0xabcdef); + assert_eq!(bus.read64(BANK_SPAN).data, 0x1122334455667788); + assert!(bus.mem_ptr(0).is_some()); + md.soft_reset(); + assert_eq!(bus.read64(0).data, 0xabcdef); + md.power_on(); + assert_eq!(bus.read64(0).status, BUS_ERR); + bus.map_unmapped(0x1fc00000, 0x1fc01000); + for (code, size) in [(1, 64 << 20), (2, 128 << 20), (3, FITTED_BANK), + (4, FITTED_BANK), (5, FITTED_BANK)] { + md.md_bus().write64(reg::CONFIG, 1 | (code << 4)); + assert_eq!(bus.read64(size - 8).status, BUS_OK); + assert_eq!(bus.read64(size).status, BUS_ERR); + assert_eq!(bus.read64(0x1fc00000).status, BUS_OK, "a non-RAM mapping survives"); + } + md.md_bus().write64(reg::CONFIG, 0x11); + assert_eq!(bus.read64(64 << 20).status, BUS_ERR, "the old prefix is removed on shrink"); + md.md_bus().write64(reg::CONFIG, 0x61); + assert_eq!(bus.read64(0).status, BUS_ERR, "unsupported density does not map RAM"); + #[cfg(feature = "jitv2")] + { + let md = configured(); + let ram = md.physical(); + let first = ram.gen_ptr(0); + let second = ram.gen_ptr(BANK_SPAN); + assert!(!first.is_null() && !second.is_null()); + assert_ne!(first, second); + let before = unsafe { (*first).load(Ordering::Relaxed) }; + ram.write64(0, 0x55); + assert!(unsafe { (*first).load(Ordering::Relaxed) } > before); + md.md_bus().write64(8, 0x33 | (1 << 32)); + // The window address stays fixed while its backing bank changes. + assert_eq!(ram.gen_ptr(0), first); + assert_eq!(ram.gen_ptr(BANK_SPAN), second); + let bank0 = ram.banks()[0].gen_ptr(0); + let bank1 = ram.banks()[1].gen_ptr(0); + let before0 = unsafe { (*bank0).load(Ordering::Relaxed) }; + let before1 = unsafe { (*bank1).load(Ordering::Relaxed) }; + assert_eq!(ram.write64(0, 0xaa), BUS_OK); + assert_eq!(unsafe { (*bank0).load(Ordering::Relaxed) }, before0); + assert_eq!(unsafe { (*bank1).load(Ordering::Relaxed) }, before1 + 1); + assert_eq!(unsafe { (*first).load(Ordering::Relaxed) }, before1 + 1); + let view = md.uncached_view(); + let logical = view.gen_ptr(0); + assert_eq!(logical, md.memory.gen_ptr(0)); + let logical_before = unsafe { (*logical).load(Ordering::Relaxed) }; + drop(md); + assert_eq!(unsafe { (*logical).load(Ordering::Relaxed) }, logical_before, + "an external endpoint keeps its logical generation pointers alive"); + // The endpoint keeps the backing memory alive after MD disappears. + assert!(unsafe { (*bank1).load(Ordering::Relaxed) } > 0); + assert_eq!(view.read64(0).status, BUS_ERR); + } + + } + /// The guest reads ECC configuration and memory or directory errors through status registers. + #[test] + fn guest_registers_expose_ecc_configuration_and_error_latches() { + { + let md = configured(); + let r = md.md_bus(); + // These carrier offsets are idle; they do not select storage. + for at in [0x20, 0x20010, 0x40020, 0x60020] { + let before = md.unsupported(); + assert_eq!(r.write64(at, u64::MAX), BUS_OK, "{at:#x}"); + assert_eq!(r.read64(at).data, 0, "{at:#x}"); + assert_eq!(md.unsupported(), before + 1); + } + for (at, value) in [(0x60, 0x001818180c0c0c00), (0x40050, 0x04040400)] { + let before = md.unsupported(); + assert_eq!(r.write64(at, 0), BUS_OK); + assert_eq!(r.read64(at).data, value); + assert_eq!(md.unsupported(), before + 1); + } + // Error-clear writes are ignored too: only a read clears the record. + md.ecc_bus().write8(7, 3); + md.uncached_view().read64(0); + r.write64(0x40000, 2); + md.directory_bus().write64(0, 3 << 16); + r.write64(0x40000, 0); + md.physical().read64(0); + let memory_error = r.read64(0x60010).data; + let directory_error = r.read64(0x40010).data; + assert_ne!(memory_error & (1 << 60), 0); + assert_ne!(directory_error & (1 << 63), 0); + for at in [0x40010, 0x40018, 0x60010, 0x60018] { + let before = md.unsupported(); + assert_eq!(r.write64(at, u64::MAX), BUS_OK); + assert_eq!(r.read64(0x60010).data, memory_error); + assert_eq!(r.read64(0x40010).data, directory_error); + assert_eq!(md.unsupported(), before + 1); + } + // An unsupported self-test command leaves an in-flight command alone. + r.write64(0x20030, 1); + let before = md.unsupported(); + assert_eq!(r.write64(0x20030, 3), BUS_OK); + assert_eq!(md.unsupported(), before + 1); + md.advance_ns(1000); + assert_eq!(r.read64(0x20048).data, 3); + assert_eq!(r.write64(0x70000, 1), BUS_ERR); + assert_eq!(r.read64(0x70000).status, BUS_ERR); + assert_eq!(r.write32(0x60, 0), BUS_ERR); + assert_eq!(r.write64(0x61, 0), BUS_ERR); + } + { + let md = configured(); + let regs = md.md_bus(); + assert_eq!(regs.read64(0x60000).data, 0); + assert_eq!(regs.write64(0x60000, u64::MAX), BUS_OK); + assert_eq!(regs.read64(0x60000).data, 1); + assert_eq!(regs.write64(0x60000, !1), BUS_OK); + assert_eq!(regs.read64(0x60000).data, 0); + assert_eq!(regs.write8(0x60000, 1), BUS_ERR); + assert_eq!(regs.write64(0x60001, 1), BUS_ERR); + regs.write64(0x60000, 1); + md.soft_reset(); + assert_eq!(regs.read64(0x60000).data, 0); + assert_eq!(regs.read64(8).data, 0x33); + regs.write64(0x60000, 1); + md.power_on(); + assert_eq!(regs.read64(0x60000).data, 0); + assert_eq!(regs.read64(8).data, 0); + } + { + for (code, kind, status, valid) in [ + (1, MemoryError::Correctable, BUS_OK, 1 << 52), + (3, MemoryError::Uncorrectable, BUS_ERR, 1 << 60), + ] { + let md = configured(); + let regs = md.md_bus(); + let ecc = md.ecc_bus(); + let ram = md.uncached_view(); + let events = Arc::new(Mutex::new(Vec::new())); + let seen = events.clone(); + md.on_error(Arc::new(move |event| seen.lock().push(event))); + assert_eq!(ram.read64(0).status, BUS_OK); + ecc.write8(7, code); + regs.write64(0x60000, u64::MAX); + assert_eq!(ram.read64(0).status, BUS_OK); + assert_eq!(ecc.read8(7).data, code); + assert_eq!(regs.read64(0x60010).data, 0); + assert!(events.lock().is_empty()); + regs.write64(0x60000, !1); + assert_eq!(regs.read64(0x60000).data, 0); + assert_eq!(ram.read64(0).status, status); + assert_ne!(regs.read64(0x60010).data & valid, 0); + assert_eq!(*events.lock(), vec![kind]); + if kind == MemoryError::Correctable { + assert_eq!(ecc.read8(7).data, 0); + assert_eq!(ram.read64(0).status, BUS_OK); + assert_eq!(*events.lock(), vec![kind]); + } + } + } + { + let md = configured(); + let regs = md.md_bus(); + let ecc = md.ecc_bus(); + let ram = md.uncached_view(); + ecc.write8(7, 3); + regs.write64(0x60000, 1); + assert_eq!(ram.write8(7, 0x5a), BUS_OK); + assert_eq!(ecc.read8(7).data, 0x82); + regs.write64(0x60000, 0); + assert_eq!(ram.read64(0).status, BUS_OK); + assert_eq!(ram.read64(0).data, 0x5a); + assert_eq!(regs.read64(0x60010).data, 0); + } + } + /// Self-test completion is visible only after its configured clock interval. + #[test] + fn selftest_waits_for_fitted_clock_cycles_and_zero_fills_ram() { + { + let now = Arc::new(AtomicU64::new(100)); + let time = now.clone(); + let md = BedrockMd::with_clock(Arc::new(move || time.load(Ordering::Relaxed))); + let regs = md.md_bus(); + regs.write64(8, 0x33); + md.start_clock(); + regs.write64(0x20030, 1); + now.store(1099, Ordering::Relaxed); + assert_eq!(regs.read64(0x20048).data, 0); + now.store(1100, Ordering::Relaxed); + assert_eq!(regs.read64(0x20048).data, 3); + now.store(5000, Ordering::Relaxed); + regs.write64(0x20030, 1); + now.store(5500, Ordering::Relaxed); + md.stop_clock(); + now.store(15000, Ordering::Relaxed); + assert_eq!(regs.read64(0x20048).data, 0); + md.start_clock(); + now.store(15499, Ordering::Relaxed); + assert_eq!(regs.read64(0x20048).data, 0); + now.store(15500, Ordering::Relaxed); + assert_eq!(regs.read64(0x20048).data, 3); + regs.write64(0x20030, 1); + now.store(15400, Ordering::Relaxed); + assert_eq!(regs.read64(0x20048).data, 0); + now.store(16499, Ordering::Relaxed); + assert_eq!(regs.read64(0x20048).data, 0); + now.store(16500, Ordering::Relaxed); + assert_eq!(regs.read64(0x20048).data, 3); + regs.write64(0x20030, 1); + regs.write64(0x20030, 2); + now.store(20000, Ordering::Relaxed); + assert_eq!(regs.read64(0x20048).data, 0); + md.power_on(); + regs.write64(8, 0x33); + md.start_clock(); + regs.write64(0x20030, 1); + now.store(20999, Ordering::Relaxed); + assert_eq!(regs.read64(0x20048).data, 0); + now.store(21000, Ordering::Relaxed); + assert_eq!(regs.read64(0x20048).data, 3); + } + { + let md = configured(); + let r = md.md_bus(); + r.write64(0x20030, 0x21); + md.advance_ns(1000); + assert_eq!(r.read64(0x20048).data, 2); + assert_eq!(r.read64(0x20040).data, 1 << 48); + r.write64(0x20030, 1); + r.write64(0x20030, 2); + md.advance_ns(1000); + assert_eq!(r.read64(0x20048).data, 0); + assert_eq!(r.write64(0x20030, 3), BUS_OK); + } + { + let md = configured(); + let r = md.md_bus(); + r.write64(8, 0x13); + md.physical().write64(0, 0x1234); + md.ecc_bus().write8(7, 1); + r.write64(0x20038, 0x80); + r.write64(0x20030, 1); + md.advance_ns(1000); + assert_eq!(r.read64(0x20048).data, 3); + assert_eq!(md.physical().read64(0).data, 0); + assert_eq!(md.ecc_bus().read8(7).data, 0); + assert_eq!(r.read64(0x60010).data, 0); + } + } + /// Fetch-and-op registers preserve the big-endian lanes used by the processor. + #[test] + fn fetchop_preserves_big_endian_word_lanes() { + let md = configured(); + let ram = md.physical(); + let f = md.fetchop_bus(); + ram.write64(0, 0xffffffff00000009); + assert_eq!(f.read32(8).data, u32::MAX); + assert_eq!(ram.read64(0).data, 9); + assert_eq!(f.read32(12).data, 9); + assert_eq!(ram.read64(0).data, 10); + assert_eq!(f.write32(28, 3), BUS_OK); + assert_eq!(ram.read64(0).data, 2); + assert_eq!(f.write64(32, 8), BUS_OK); + assert_eq!(ram.read64(0).data, 10); + assert_eq!(f.read64(24).data, 10); + assert_eq!(ram.read64(0).data, 0); + assert_eq!(f.read64(32).status, BUS_ERR); + assert_eq!(f.write64(40, 1), BUS_ERR); + assert_eq!(f.read16(0).status, BUS_ERR); + } + /// The guest reads ECC bytes and scrubbing repairs correctable memory. + #[test] + fn ecc_carriers_expose_check_bytes_and_scrub_corrections() { + { + let md = configured(); + let ecc = md.ecc_bus(); + for carrier in 0..8 { + let source = 0x11223344556677a5u64; + assert_eq!(ecc.write8(carrier, source as u8), BUS_OK); + assert_eq!(ecc.read8(carrier).data, source as u8, "carrier {carrier}"); + } + assert_eq!(ecc.write16(4, 0x1234), BUS_OK); + assert_eq!(ecc.read16(4).data, 0x1234); + assert_eq!(ecc.write16(6, 0x2313), BUS_OK); + assert_eq!(ecc.read16(6).data, 0x2313); + assert_eq!(ecc.write16(0x04000006, 0xface), BUS_OK); + assert_eq!(ecc.read16(0x04000006).data, 0xface); + + } + { + let md = configured(); + let events = Arc::new(Mutex::new(Vec::new())); + let seen = events.clone(); + let weak = Arc::downgrade(&md); + md.on_error(Arc::new(move |kind| { + assert_eq!(weak.upgrade().unwrap().current_cell(), u64::MAX); + seen.lock().push(kind); + })); + assert_eq!(md.ecc_bus().write8(7, 3), BUS_OK); + assert_eq!(md.ecc_bus().write8(6, 0x13), BUS_OK); + assert_eq!(md.ecc_bus().write8(5, 0x23), BUS_OK); + assert_eq!(md.ecc_scrub_bus().write64(0, 0), BUS_OK); + assert_eq!( + *events.lock(), + vec![ + MemoryError::Uncorrectable, + MemoryError::Correctable, + MemoryError::Correctable, + ] + ); + let record = md.md_bus().read64(0x60010).data; + assert_ne!(record & (1 << 60), 0); + assert_ne!(record & (1 << 52), 0); + assert_ne!(record & (1 << 53), 0); + assert_eq!((record >> 40) & 0xff, 3); + assert_eq!(record & (((1u64 << 30) - 1) << 3), 0); + assert_eq!(md.ecc_bus().read8(6).data, 0x13); + assert_eq!(md.ecc_bus().read8(5).data, 0x23); + } + } + /// The ECC view takes an uncached SH's check word from wherever the PROM + /// put it in the source register; other views, and bytes, take the low bits. + #[test] + fn ecc_view_reads_the_check_word_from_the_whole_source_register() { + let md = configured(); + let ecc = md.ecc_bus(); + // (carrier offset, register bits holding the check word) + for (carrier, shift) in [(6u64, 0u32), (4, 48), (2, 32), (0, 16)] { + let source = 0x1111_2222_3333_4444u64 ^ (0xa55a_u64 << shift); + let word = (source >> shift) as u16; + assert_eq!(ecc.write_narrow_from(carrier, 2, source), BUS_OK); + assert_eq!(ecc.read16(carrier).data, word, "carrier {carrier}"); + } + assert_eq!(ecc.write_narrow_from(5, 1, 0xfeed_0000_0000_0023), BUS_OK); + assert_eq!(ecc.read8(5).data, 0x23, "a byte keeps the architectural low bits"); + // The memory view is not the ECC view: its halfword is the low bits. + let view = md.uncached_view(); + assert_eq!(view.write_narrow_from(0x40, 2, 0xdead_beef_1234_5678), BUS_OK); + assert_eq!(md.dma_view().read64(0x40).data >> 48, 0x5678); + } + /// Uncached CPU and DMA views report uncorrectable errors with bus status. + #[test] + fn uncached_and_dma_views_report_errors_to_the_guest() { + let md = configured(); + let r = md.md_bus(); + let e = md.ecc_bus(); + e.write8(7, 3); + assert_eq!(md.uncached_view().read64(0).status, BUS_ERR); + assert_eq!(md.dma_view().read64(0).status, BUS_OK); + assert_ne!(r.read64(0x60018).data & (1 << 60), 0); + e.write8(7, 0x13); + assert_eq!(md.ecc_scrub_bus().write64(0, 0), BUS_OK); + assert_eq!(md.physical().read64(0).data, 1); + assert_eq!(e.read8(7).data, 0x13); + assert_eq!(md.ecc_scrub_bus().write32(0, 0), BUS_ERR); + } + /// Directory reads report invalid metadata and protection faults to the guest. + #[test] + fn directory_reads_check_metadata_and_report_protection_errors() { + { + let md = configured(); + let events = Arc::new(Mutex::new(Vec::new())); + let seen = events.clone(); + let weak = Arc::downgrade(&md); + md.on_error(Arc::new(move |kind| { + assert_eq!(weak.upgrade().unwrap().current_cell(), u64::MAX); + seen.lock().push(kind); + })); + let r = md.md_bus(); + md.directory_bus().write64(0, 1); + assert_eq!(md.directory_bus().read64(0).data, 0x001f0001); + r.write64(0x40000, 2); + md.directory_bus().write64(0, 0x001e0001); + r.write64(0x40000, 0); + md.ecc_bus().write8(7, 3); + assert_eq!(md.uncached_view().read64(0).status, BUS_ERR); + assert_eq!(md.directory_bus().read64(0).data, 0x001f0001); + assert_eq!( + *events.lock(), + vec![MemoryError::Correctable, MemoryError::Uncorrectable] + ); + assert_ne!(r.read64(0x40010).data & (1 << 61), 0); + assert_ne!(r.read64(0x60010).data & (1 << 60), 0); + } + { + let md = configured(); + let d = md.directory_bus(); + assert_eq!(d.write64(0x408, 0x1234), BUS_OK); + assert_eq!(d.read64(0x2000408).data, 0x1234); + assert_eq!(d.write64(0x418, 0x5678), BUS_OK); + assert_eq!(d.read64(0x408).data, 0x1234); + assert_eq!(d.read64(0x418).data, 0x5678); + assert_eq!(d.write64(8, u64::MAX), BUS_OK); + assert_eq!(d.read64(8).data, u64::MAX); + assert_eq!(d.read32(0).status, BUS_ERR); + assert_eq!(d.read64(1 << 30).status, BUS_ERR); + } + } +} + +const DIRECTORY_PREMIUM: [u8; 64] = [ + 0x1f, 0x2f, 0x37, 0x3b, 0x3d, 0x3e, 0x07, 0x0b, 0x0d, 0x0e, 0x13, 0x15, 0x16, 0x19, 0x1a, 0x1c, + 0x7f, 0x4f, 0x57, 0x5b, 0x5d, 0x5e, 0x67, 0x6b, 0x6d, 0x6e, 0x73, 0x75, 0x76, 0x79, 0x7a, 0x7c, + 1, 2, 4, 8, 16, 32, 0x23, 0x25, 0x26, 0x29, 0x2a, 0x2c, 0x31, 0x32, 0x34, 0x38, 0x40, 0x43, + 0x45, 0x46, 0x49, 0x4a, 0x4c, 0x51, 0x52, 0x54, 0x58, 0x61, 0x62, 0x64, 0x68, 0x70, +]; +const DIRECTORY_STANDARD: [u8; 32] = [ + 0x1f, 0x2f, 0x37, 0x3b, 0x3d, 0x3e, 0x07, 0x0b, 0x0d, 0x0e, 0x13, 0x15, 0x16, 0x19, 0x1a, 0x1c, + 1, 2, 4, 8, 16, 32, 0x23, 0x25, 0x26, 0x29, 0x2a, 0x2c, 0x31, 0x32, 0x34, 0x38, +]; +fn directory_columns(premium: bool) -> &'static [u8] { + if premium { + &DIRECTORY_PREMIUM + } else { + &DIRECTORY_STANDARD + } +} +fn directory_syndrome(mut word: u64, premium: bool) -> u8 { + if !premium { + word &= u32::MAX as u64; + } + let mut check = 0; + while word != 0 { + check ^= directory_columns(premium)[word.trailing_zeros() as usize]; + word &= word - 1; + } + check +} +pub fn directory_encode(word: u64, premium: bool) -> u64 { + let word = word + & !if premium { + (63 << 32) | (1 << 48) + } else { + 63 << 16 + }; + let check = directory_syndrome(word, premium) as u64; + word | if premium { + (check & 63) << 32 | (check >> 6) << 48 + } else { + check << 16 + } +} diff --git a/src/dev/bedrock_mem.rs b/src/dev/bedrock_mem.rs new file mode 100644 index 000000000..bec848941 --- /dev/null +++ b/src/dev/bedrock_mem.rs @@ -0,0 +1,311 @@ +//! The Fuel's fitted memory: two 256 MiB banks owned by `Physical`, with +//! the bank decode selected by Bedrock's memory configuration register. +//! +//! This population follows the QEMU model; it is not a measured Fuel +//! topology. Ordinary pages answer through Physical's bank views, without +//! the MD state lock. Pages with injected ECC or marked directory lines use +//! the MD endpoint. Their entire 64 MiB region is absent from the direct +//! window, so cache fills cannot bypass those checks. +//! +//! Remapping runs on the CPU thread, or with the CPU stopped, as HEART's +//! memory descriptors do. DMA may remove diagnostic metadata but retains +//! the slow mapping until the next CPU-side rebuild. Physical publishes +//! complete decode-table pages atomically; a concurrent bus master sees an +//! old or new entry. This does not promise a transaction across a remap. +//! +//! Ported from hw/misc/sgi-bedrock-md.c in jamesbraid/qemu-sgi-origami. + +use crate::physical::{Physical, RamBank}; +use crate::traits::*; +use crate::ppmem::devmap::SLOT_SHIFT; +use crate::ppmem::{MappedMemory, BITMAP_SHIFT}; +use parking_lot::Mutex; +use std::collections::BTreeSet; +use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; +use std::sync::Arc; + +pub const BANK_SPAN: u64 = 1 << 30; +pub const FITTED_BANK: u64 = 256 << 20; + +/// The PROM builds its stack and data in primary data cache lines it tags for +/// this node-local window, with no memory behind it: DRAM is not configured +/// yet. See [`BedrockMemory::startup_tag`]. +pub(crate) const STARTUP_BASE: u64 = 0x10_0000; +pub(crate) const STARTUP_SIZE: u64 = 0x8000; +/// A primary data cache tag: the state in bits 7:6, the physical address +/// bits 35:12 above bit 8. Bits above the node's 8 GiB (page bit 21) select +/// the node. +const TAG_STATE: u32 = 0xc0; +const TAG_PAGE_SHIFT: u32 = 8; +const TAG_NODE_PAGES: u32 = 1 << 21; + +#[derive(Clone, Copy, PartialEq, Eq)] +struct Run { + bank: usize, + base: u64, + size: u64, +} + +#[derive(Default)] +struct Mapping { + runs: Vec, + /// Whether the decode table's slots over the startup window hold the page. + startup_placed: bool, + // Bank-relative 4 KiB page numbers, bank 1 immediately after bank 0. + slow_pages: BTreeSet, + slow: Option>, +} + +struct NoMemory; +impl BusDevice for NoMemory {} + +/// JIT pages belong to physical addresses, not to the bank currently +/// answering there. Keep these counters separate from the banks' counters: +/// a compiled page pins its pointer across later bank reconfiguration. +#[cfg(feature = "jitv2")] +pub(crate) struct Generations(Box<[AtomicU64]>); +#[cfg(feature = "jitv2")] +impl Generations { + fn new() -> Self { + Self((0..1 << 20).map(|_| AtomicU64::new(0)).collect::>().into_boxed_slice()) + } + pub(crate) fn base(&self) -> *mut AtomicU64 { + self.0.as_ptr() as *mut AtomicU64 + } + fn bump(&self, address: u64) { + if let Some(counter) = self.0.get((address >> 12) as usize) { + counter.fetch_add(1, Ordering::Relaxed); + } + } + fn bump_all(&self) { + for slot in 0..4 { + let first = slot * (BANK_SPAN >> 12) as usize; + for counter in &self.0[first..first + (FITTED_BANK >> 12) as usize] { + counter.fetch_add(1, Ordering::Relaxed); + } + } + } +} + +pub struct BedrockMemory { + physical: Arc, + config: AtomicU64, + mapping: Mutex, + /// The window's bytes while the PROM's cache lines stand. Zero at power-on: + /// the PROM makes the lines valid without a fill. + startup_page: Box, + startup_live: AtomicBool, + #[cfg(feature = "jitv2")] + generations: Arc, +} +impl BedrockMemory { + pub fn new() -> Self { + let physical = Physical::new(vec![RamBank::new(256), RamBank::new(256)]); + physical.set_default(Arc::new(NoMemory), "Bedrock unconfigured memory"); + let startup_page = Box::new(RamBank::new(1)); + startup_page.set_addr_mask(STARTUP_SIZE as u32 - 1); + Self { physical, config: AtomicU64::new(0), mapping: Mutex::new(Mapping::default()), + startup_page, startup_live: AtomicBool::new(false), + #[cfg(feature = "jitv2")] + generations: Arc::new(Generations::new()), + } + } + pub fn physical(&self) -> &Arc { + &self.physical + } + pub(crate) fn config(&self) -> u64 { + self.config.load(Ordering::Acquire) + } + + /// Whether a primary data cache tag store starts or ends the startup + /// window, given its TagLo. PROM 6.210 stores test patterns (state bits + /// set, pages far from the window), an invalidate sweep, then one valid + /// tag for each of the window's eight pages, and a final invalidate sweep + /// once it has memory. So a valid tag for a page of the window starts it + /// and the next invalid one ends it; a kernel stores invalid tags only. + /// + /// The end is the sweep's first store, not its last. Counting accesses + /// that reach the page in an interpreted PROM 6.210 boot, none follow the + /// first of the final sweep's 2050 invalid stores (the count is the same + /// at the first and at the last), and the page is read and written + /// heavily up to that first store. Holding it to the sweep's end would + /// change nothing the PROM sees. + /// `Some(live)` is the state to move to, `None` leaves it alone. + pub(crate) fn startup_tag(&self, tag: u32) -> Option { + let live = self.startup_live.load(Ordering::Relaxed); + if tag & TAG_STATE == 0 { + return live.then_some(false); + } + let page = (tag >> TAG_PAGE_SHIFT) & (TAG_NODE_PAGES - 1); + let in_window = (STARTUP_BASE >> 12..(STARTUP_BASE + STARTUP_SIZE) >> 12).contains(&(page as u64)); + (in_window && !live).then_some(true) + } + /// Move to `live`; whether that changed anything, and so needs a remap. + pub(crate) fn set_startup(&self, live: bool) -> bool { + self.startup_live.swap(live, Ordering::Relaxed) != live + } + pub(crate) fn startup_live(&self) -> bool { + self.startup_live.load(Ordering::Relaxed) + } + /// Power-on: no cache lines stand, and the page holds zero. + pub(crate) fn clear_startup(&self) { + self.startup_live.store(false, Ordering::Relaxed); + for at in (0..STARTUP_SIZE).step_by(8) { + self.startup_page.write64(at, 0); + } + } + /// The page, for an access it answers: the window, while live. The bus + /// decode, the CPU's cached path and DMA all reach it through here or + /// through Physical's table, since nothing else is behind the window. + pub(crate) fn startup_device(&self, address: u64, bytes: u64) -> Option<&dyn BusDevice> { + let offset = address.checked_sub(STARTUP_BASE)?; + (self.startup_live() && offset + bytes <= STARTUP_SIZE) + .then(|| &*self.startup_page as &dyn BusDevice) + } + #[cfg(feature = "jitv2")] + pub(crate) fn generations(&self) -> Arc { self.generations.clone() } + #[cfg(feature = "jitv2")] + pub(crate) fn gen_ptr(&self, address: u64) -> *const AtomicU64 { + // Disabled RAM must still have a counter: a JIT page may already + // have claimed it before the guest enables or moves a bank. + if address / BANK_SPAN < 4 && address % BANK_SPAN < FITTED_BANK { + unsafe { self.generations.base().add((address >> 12) as usize) } + } else { std::ptr::null() } + } + pub(crate) fn written(&self, _address: u64) { + #[cfg(feature = "jitv2")] + self.generations.bump(_address); + } + + /// The endpoint is weak back to MD: Physical owns it without a cycle. + pub(crate) fn set_slow_view(&self, slow: Arc) { + self.mapping.lock().slow = Some(slow); + } + + fn runs(config: u64) -> Vec { + (0..8).filter_map(|slot| { + let base = slot * BANK_SPAN; + let (bank, _) = Self::decode(config, base, 1)?; + let code = (config >> (bank / 2 * 8 + 4)) & 15; + Some(Run { bank, base, size: FITTED_BANK.min((32 << 20) << code) }) + }).collect() + } + + /// Publish a state snapshot after releasing MD's lock. Only the CPU + /// thread, or a caller with the CPU stopped, may change ppmem's bitmap. + /// A forced rebuild follows the machine setting Physical's default. + pub(crate) fn remap(&self, config: u64, slow_pages: BTreeSet, force: bool) { + let runs = Self::runs(config); + let mut old = self.mapping.lock(); + if !force && old.runs == runs && old.slow_pages == slow_pages { + self.config.store(config, Ordering::Release); + return; + } + let phys = &self.physical; + let space = phys.ppmem_space(); + // Withdraw the direct window before publishing any checked page. + if let Some(space) = space { + space.clear_mappings(); + } + // Restore only ranges previously owned by RAM. In particular, the + // unused part of a 1 GiB slot may contain flash or other devices. + if old.startup_placed { + phys.fill_slots(STARTUP_BASE >> SLOT_SHIFT, STARTUP_SIZE >> SLOT_SHIFT, phys.default_entry()); + } + for run in &old.runs { + let retained = runs.iter().find(|next| next.base == run.base) + .map_or(0, |next| next.size.min(run.size)); + if retained < run.size { + phys.fill_slots((run.base + retained) >> SLOT_SHIFT, + (run.size - retained) >> SLOT_SHIFT, phys.default_entry()); + } + } + for run in &runs { + phys.fill_slots(run.base >> SLOT_SHIFT, run.size >> SLOT_SHIFT, phys.bank_view(run.bank, run.base)); + let bank_page = run.bank as u64 * (FITTED_BANK >> SLOT_SHIFT); + let pages = run.size >> SLOT_SHIFT; + let mut checked_regions = BTreeSet::new(); + for &page in slow_pages.range(bank_page..bank_page + pages) { + let offset = (page - bank_page) << SLOT_SHIFT; + let address = run.base + offset; + checked_regions.insert(offset >> BITMAP_SHIFT); + phys.map(address, address + (1 << SLOT_SHIFT), + old.slow.as_ref().expect("Bedrock MD slow endpoint").clone(), "Bedrock checked RAM"); + } + if let Some(space) = space { + let region_size = 1 << BITMAP_SHIFT; + for offset in (0..run.size).step_by(region_size as usize) { + // The startup page answers through the decode only, so the + // region holding it stays out of the direct window. + let startup = self.startup_live() && run.base + offset == 0; + if !startup && !checked_regions.contains(&(offset >> BITMAP_SHIFT)) { + if let Err(error) = space.map_range(run.bank, offset, run.base + offset, region_size) { + crate::dlog!(crate::devlog::LogModule::Bedrock, + "Bedrock RAM window at {:#x}: {error}", run.base + offset); + } + } + } + } + } + let live = self.startup_live(); + if live { + let page: *const RamBank = &*self.startup_page; + phys.fill_slots(STARTUP_BASE >> SLOT_SHIFT, STARTUP_SIZE >> SLOT_SHIFT, + page as crate::ppmem::devmap::Entry); + } + old.startup_placed = live; + self.config.store(config, Ordering::Release); + // Publish changed ownership before advancing the stable counters, + // so an in-flight compile cannot accept bytes from the old decode. + self.changed_decode(); + old.runs = runs; + old.slow_pages = slow_pages; + } + + pub(crate) fn decode(config: u64, addr: u64, bytes: u64) -> Option<(usize, u64)> { + let software = addr / BANK_SPAN; + if software >= 8 { + return None; + } + let bank = software ^ ((config >> 32) & 3); + let field = config >> (bank / 2 * 8); + let code = (field >> 4) & 15; + if bank >= 2 || field & (1 << (bank & 1)) == 0 || !(1..=5).contains(&code) { + return None; + } + let limit = FITTED_BANK.min((32 << 20) << code); + let offset = addr % BANK_SPAN; + (offset < limit && bytes <= limit - offset).then_some((bank as usize, offset)) + } + pub(crate) fn raw_read(&self, bank: usize, at: u64) -> u64 { + self.physical.banks()[bank].read64(at).data + } + pub(crate) fn raw_write(&self, bank: usize, at: u64, value: u64) -> u32 { + let result = self.physical.banks()[bank].write64(at, value); + if result == BUS_OK { + let slot = bank as u64 ^ ((self.config() >> 32) & 3); + self.written(slot * BANK_SPAN + at); + } + result + } + pub(crate) fn zero(&self, bank: usize, size: u64) { + assert!(size <= FITTED_BANK); + // The bank owns this contiguous mapping; zero has the same representation + // in host word order. Self-test does not synchronize concurrent bank accesses. + unsafe { + std::ptr::write_bytes(self.physical.banks()[bank].host_ptr(), 0, size as usize); + } + #[cfg(feature = "jitv2")] + self.physical.banks()[bank].bump_gen_all(); + #[cfg(feature = "jitv2")] + self.generations.bump_all(); + } + pub(crate) fn changed_decode(&self) { + #[cfg(feature = "jitv2")] + self.generations.bump_all(); + #[cfg(feature = "jitv2")] + for bank in self.physical.banks() { + bank.bump_gen_all(); + } + } +} diff --git a/src/dev/mod.rs b/src/dev/mod.rs index d8f7d6b72..013e5b626 100644 --- a/src/dev/mod.rs +++ b/src/dev/mod.rs @@ -60,6 +60,8 @@ pub mod vino; pub mod virtio; pub mod wd33c93a; pub mod z85c30; +pub mod bedrock_md; +pub mod bedrock_mem; pub mod dp83843; pub mod ics1890; pub mod isp12160; From f30271686f031da92b3a2937cb0dd2484a142901 Mon Sep 17 00:00:00 2001 From: James Braid Date: Fri, 30 Oct 2026 08:13:37 +0000 Subject: [PATCH 21/31] bedrock: add the node I/O, network, crossbar and local-bus blocks I/O (bedrock_io): Fuel needs Bedrock I/O to route programmed PIO windows to PCI widgets, check inbound DMA permissions and run its two block-transfer engines. Fault reporting and interrupt delivery stay with the block that owns the state, and unsupported accesses are reported. Register map and behaviour are ported from the QEMU model. Node routing (bedrock_ni): Fuel runs as one node with no NUMAlink peer, but the guest still reads and programs these local control words during machine setup. Retain the writes and keep the network fabric out of the model. Crossbar (bedrock_xb): Record the fabric error queues and let software service them, with the outputs available to the Bedrock processor interface. Unsupported queue reads return zero and writes are logged and ignored. Local bus (bedrock_lb): Add the node and region registers, atomic scratch aliases, error and vector status, and the MicroLAN master that board identity parts sit on. The tests drive it together with the I/O, node-interface and crossbar blocks it reports through, which land in this same commit. --- src/dev/bedrock_io.rs | 811 ++++++++++++++++++++++++++++++++++++++++++ src/dev/bedrock_lb.rs | 516 +++++++++++++++++++++++++++ src/dev/bedrock_ni.rs | 335 +++++++++++++++++ src/dev/bedrock_xb.rs | 278 +++++++++++++++ src/dev/mod.rs | 4 + 5 files changed, 1944 insertions(+) create mode 100644 src/dev/bedrock_io.rs create mode 100644 src/dev/bedrock_lb.rs create mode 100644 src/dev/bedrock_ni.rs create mode 100644 src/dev/bedrock_xb.rs diff --git a/src/dev/bedrock_io.rs b/src/dev/bedrock_io.rs new file mode 100644 index 000000000..4b3e659c3 --- /dev/null +++ b/src/dev/bedrock_io.rs @@ -0,0 +1,811 @@ +//! Bedrock IO routes programmed PIO windows to PCI widgets, applies inbound +//! permission checks to DMA requests and exposes two block-transfer engines. +//! Guests configure the widget masks before touching devices; the BTE moves +//! 128-byte lines and writes completion state through the configured target. +//! +//! # What is approximated here +//! +//! BTE transfers and their completion callbacks run synchronously when a +//! start is written. Timing and unpublished register behavior are not modeled. +//! +//! The plain register words sit in a sparse map keyed by offset, since the +//! 0x30010-byte block holds only a few dozen of them. +//! +//! Ported from hw/misc/sgi-bedrock-io.c in jamesbraid/qemu-sgi-origami. + +/// Length of the block-relative register aperture. +pub const SIZE: u64 = 0x30010; + +/// The public Linux Hub I/O protection-override key literal. +/// +/// Linux `arch/mips/include/asm/sn/sn0/hubio.h` at +/// `db2ddb87143519e20a95aa36c60b36107b736a58` defines +/// `IIO_PROTECT_OVRRD_KEY` as this value (lines 31, 47). This establishes +/// the literal's provenance only; it does not establish a register mapping. +pub const IIO_PROTECT_OVRRD_KEY: u64 = 0x53474972756c6573; + +pub mod reg { + pub const DEVICE_ID: u64 = 0x0; + pub const PROTECTION_KEY: u64 = 0x108; + pub const PIO_PERMISSION: u64 = 0x100; + pub const INBOUND_PERMISSION: u64 = 0x110; + pub const INBOUND_DEVICE_PERMISSION: u64 = 0x120; + pub const XIO_STATUS: u64 = 0x128; + pub const LINK_LOG: u64 = 0x130; + pub const LINK_STATUS: u64 = 0x138; + pub const BTE1_BASE: u64 = 0x10000; + pub const BTE_REGISTER_SIZE: u64 = 0x30; + pub const READ_ONLY: u64 = 0x8; + pub const BTE2_BASE: u64 = 0x20000; + pub const WIDGET_STATUS: u64 = 0x20; + pub const WIDGET_PERMISSION: u64 = 0x118; + pub const REGISTER_0150: u64 = 0x150; + pub const REGISTER_0158: u64 = 0x158; + pub const ITTE_BASE: u64 = 0x160; + pub const ITTE_LAST: u64 = 0x190; + pub const REGISTER_0198_BASE: u64 = 0x198; + pub const REGISTER_01D8_LAST: u64 = 0x1d8; + pub const READ_ONLY_ALIAS: u64 = 0x1f8; + pub const REGISTER_01E0: u64 = 0x1e0; + pub const REGISTER_01E8: u64 = 0x1e8; + pub const REGISTER_01F0: u64 = 0x1f0; + pub const REGISTER_0200: u64 = 0x200; + pub const REGISTER_03A8: u64 = 0x3a8; + pub const REGISTER_03B0: u64 = 0x3b0; + pub const REGISTER_03B8: u64 = 0x3b8; + pub const REGISTER_03C0: u64 = 0x3c0; + pub const RETAINED_WINDOW_BASE: u64 = 0x308; + pub const RETAINED_WINDOW_LAST: u64 = 0x348; + pub const RETAINED_CONFIG_BASE: u64 = 0x400; + pub const RETAINED_CONFIG_LAST: u64 = 0x5d8; + pub const GFX_MAP_BASE: u64 = 0x140; +} +pub mod bte_reg { + pub const STATUS: u64 = 0x0; + pub const SOURCE: u64 = 0x8; + pub const DESTINATION: u64 = 0x10; + pub const CONTROL: u64 = 0x18; + pub const NOTIFICATION: u64 = 0x20; + pub const INTERRUPT: u64 = 0x28; +} + +use crate::traits::{BusDevice, BusRead64, Resettable, BUS_ERR, BUS_OK}; +use parking_lot::Mutex; +use std::collections::{BTreeMap, HashSet}; +use std::sync::Arc; +use std::thread::ThreadId; +const ID: u64 = (2 << 28) | (0xc110 << 12) | (0x36 << 1); +const ADDR_MASK: u64 = 0xffffffff80; +const BUSY: u64 = 1 << 20; +const ERROR: u64 = 1 << 16; +#[derive(Clone, Copy, Default)] +struct Bte { + status: u64, + source: u64, + destination: u64, + control: u64, + notification: u64, + interrupt: u64, +} +struct Regs { + words: BTreeMap, + bte: [Bte; 2], + node: u8, + peer: bool, +} +impl Default for Regs { + fn default() -> Self { + let mut words = BTreeMap::new(); + for (a, v) in [ + (0x20, 0xa), + (0x100, !0), + (0x110, !0), + (0x118, !0), + (0x120, !0), + (0x128, 0x3fc0fff6403 & !(1 << 13)), + (0x1e8, 0x111), + ] { + words.insert(a, v); + } + Self { + words, + bte: [Bte::default(); 2], + node: 0, + peer: false, + } + } +} +// Only a synchronous callback on the same thread is recursive. DMA workers +// on other threads may call the thread-safe downstream bus independently. +struct InboundAccess<'a> { + active: &'a Mutex>, + thread: ThreadId, +} +impl Drop for InboundAccess<'_> { + fn drop(&mut self) { + self.active.lock().remove(&self.thread); + } +} + +pub struct BedrockIo { + inbound_threads: Mutex>, + regs: Arc>, + widgets: Mutex<[Option>; 16]>, + memory: Mutex>>, + hub: Mutex>>, +} +impl Default for BedrockIo { + fn default() -> Self { + Self::new() + } +} +impl BedrockIo { + pub fn new() -> Self { + let regs = Arc::new(Mutex::new(Regs::default())); + crate::locks::register_mutex("bedrock-io", ®s); + Self { + regs, + inbound_threads: Mutex::new(HashSet::new()), + widgets: Mutex::new(std::array::from_fn(|_| None)), + memory: Mutex::new(None), + hub: Mutex::new(None), + } + } + pub fn connect_widget(&self, widget: u8, dev: Arc) { + assert!(widget < 16); + self.widgets.lock()[widget as usize] = Some(dev); + self.set_xio_peer(true); + } + pub fn set_xio_peer(&self, peer: bool) { + let mut r = self.regs.lock(); + r.peer = peer; + if peer { + *r.words.entry(reg::XIO_STATUS).or_default() |= 1 << 13; + } else { + *r.words.entry(reg::XIO_STATUS).or_default() &= !(1 << 13); + } + } + pub fn connect_memory(&self, dev: Arc) { + *self.memory.lock() = Some(dev); + } + pub fn connect_hub(&self, dev: Arc) { + *self.hub.lock() = Some(dev); + } + pub fn set_node_number(&self, node: u8) { + self.regs.lock().node = node; + } + pub fn gfx_map(&self, entry: usize) -> u64 { + assert!(entry < 2); + self.word(reg::GFX_MAP_BASE + entry as u64 * 8) + } + fn word(&self, a: u64) -> u64 { + self.regs.lock().words.get(&a).copied().unwrap_or(0) + } + pub fn disable_device(&self, widget: u8, device: u8) { + if (8..16).contains(&widget) && device < 8 { + *self.regs.lock().words.entry(reg::INBOUND_DEVICE_PERMISSION).or_default() &= + !(1 << ((widget - 8) * 8 + device)); + } + } + fn permitted(r: &Regs, widget: u8, device: u8) -> bool { + widget < 16 + && device < 8 + && r.words.get(®::WIDGET_PERMISSION).copied().unwrap_or(0) & (1 << widget) != 0 + && (widget < 8 + || r.words.get(®::INBOUND_DEVICE_PERMISSION).copied().unwrap_or(0) & (1 << ((widget - 8) * 8 + device)) + != 0) + } + fn retained(a: u64) -> bool { + matches!( + a, + 0x208 + | 0x210 + | 0x218 + | 0x220 + | 0x228 + | 0x300 + | reg::GFX_MAP_BASE + | 0x148 + | 0x388 + | 0x390 + | 0x398 + | 0x3a0 + | 0x600 + | 0x608 + | 0x610 + | 0x30000 + | 0x30008 + ) || (reg::RETAINED_WINDOW_BASE..reg::RETAINED_WINDOW_LAST).contains(&a) + || (reg::RETAINED_CONFIG_BASE..=reg::RETAINED_CONFIG_LAST).contains(&a) + } + fn mask(a: u64) -> Option { + Some(match a { + reg::WIDGET_STATUS => 0x7fe01f, + reg::PIO_PERMISSION | reg::INBOUND_PERMISSION | reg::WIDGET_PERMISSION | reg::INBOUND_DEVICE_PERMISSION | reg::REGISTER_0150 | reg::REGISTER_0158 => !0, + reg::XIO_STATUS => 0x3ff0fff7d3f, + reg::LINK_STATUS => 0x101ff7f, + reg::ITTE_BASE..=reg::ITTE_LAST => 0x1f1f, + reg::REGISTER_0198_BASE..=reg::REGISTER_01D8_LAST => !0, + reg::REGISTER_01E0 => 0x3ffffff, + reg::REGISTER_01E8 => 0x1111, + reg::REGISTER_01F0 => 0x0fffffff03ffffff, + reg::REGISTER_0200 => 0x10f, + reg::REGISTER_03A8 => 0x13ffffffff30f3, + reg::REGISTER_03B0 => 0x1ff0f, + reg::REGISTER_03B8 => 0xff, + reg::REGISTER_03C0 => 0xffffff, + _ if Self::retained(a) => !0, + _ => return None, + }) + } + fn bte_at(a: u64) -> Option<(usize, u64)> { + if (reg::BTE1_BASE..reg::BTE1_BASE + reg::BTE_REGISTER_SIZE).contains(&a) { + Some((0, a - reg::BTE1_BASE)) + } else if (reg::BTE2_BASE..reg::BTE2_BASE + reg::BTE_REGISTER_SIZE).contains(&a) { + Some((1, a - reg::BTE2_BASE)) + } else { + None + } + } + fn target(&self, offset: u64) -> Option<(u8, u64)> { + if offset < 0x10000000 { + Some(((offset >> 24) as u8, offset & 0xffffff)) + } else if offset >= 0x20000000 { + let entry = (offset >> 29) - 1; + if entry >= 7 { + return None; + } + let itte = self.word(reg::ITTE_BASE + entry * 8); + Some(( + ((itte >> 8) & 15) as u8, + ((itte & 31) << 29) + | (offset & 0x1fffffff) + | if itte & (1 << 12) != 0 { 1 << 47 } else { 0 }, + )) + } else { + None + } + } + pub fn pio_read(&self, offset: u64, size: u8) -> BusRead64 { + self.pio(offset, size, None) + } + pub fn pio_write(&self, offset: u64, size: u8, value: u64) -> u32 { + self.pio(offset, size, Some(value)).status + } + fn pio(&self, offset: u64, size: u8, value: Option) -> BusRead64 { + let Some((widget, target)) = self.target(offset) else { + return BusRead64::err(); + }; + { + let r = self.regs.lock(); + if r.words.get(&0x100).copied().unwrap_or(0) & 1 == 0 + || r.words.get(&0x110).copied().unwrap_or(0) & (1 << widget) == 0 + { + return BusRead64::err(); + } + } + if !matches!(size, 1 | 2 | 4 | 8) { + return BusRead64::err(); + } + if widget == 1 && target == 0 { + return BusRead64::ok(ID); + } + let dev = self.widgets.lock()[widget as usize].clone(); + Self::access(dev, target, size, value) + } + /// Device None means every possible DMA master must be permitted. + pub fn inbound_read( + &self, + widget: u8, + device: Option, + address: u64, + size: u8, + ) -> BusRead64 { + self.inbound(widget, device, address, size, None) + } + pub fn inbound_write( + &self, + widget: u8, + device: Option, + address: u64, + size: u8, + value: u64, + ) -> u32 { + self.inbound(widget, device, address, size, Some(value)) + .status + } + fn inbound( + &self, + widget: u8, + device: Option, + address: u64, + size: u8, + value: Option, + ) -> BusRead64 { + if address >= (1 << 48) || size as u64 > (1 << 48) - address { + return BusRead64::err(); + } + { + let r = self.regs.lock(); + let allowed = if let Some(device) = device { + Self::permitted(&r, widget, device) + } else { + (0..8).all(|d| Self::permitted(&r, widget, d)) + }; + if !allowed { + crate::dlog!(crate::devlog::LogModule::Bedrock, + "Bedrock IO inbound access refused from widget {widget} device {device:?}" + ); + return BusRead64::err(); + } + } + let thread = std::thread::current().id(); + if !self.inbound_threads.lock().insert(thread) { + crate::dlog!(crate::devlog::LogModule::Bedrock, "Bedrock IO reentrant inbound packet refused"); + return BusRead64::err(); + } + let _access = InboundAccess { + active: &self.inbound_threads, + thread, + }; + if address & (1 << 47) != 0 { + let node = ((address >> 39) & 255) as u8; + let window = address & ((1 << 39) - 1); + if node != self.regs.lock().node || window & !0x7fffff != 0x1800000 { + return BusRead64::err(); + } + let dev = self.hub.lock().clone(); + Self::access(dev, window & 0x7fffff, size, value) + } else { + let dev = self.memory.lock().clone(); + Self::access(dev, address, size, value) + } + } + fn access(dev: Option>, a: u64, size: u8, value: Option) -> BusRead64 { + let Some(dev) = dev else { + return BusRead64::err(); + }; + if let Some(v) = value { + let status = match size { + 1 => dev.write8(a, v as u8), + 2 => dev.write16(a, v as u16), + 4 => dev.write32(a, v as u32), + 8 => dev.write64(a, v), + _ => BUS_ERR, + }; + BusRead64 { status, data: 0 } + } else { + match size { + 1 => { + let r = dev.read8(a); + BusRead64 { + status: r.status, + data: r.data as u64, + } + } + 2 => { + let r = dev.read16(a); + BusRead64 { + status: r.status, + data: r.data as u64, + } + } + 4 => { + let r = dev.read32(a); + BusRead64 { + status: r.status, + data: r.data as u64, + } + } + 8 => dev.read64(a), + _ => BusRead64::err(), + } + } + } + fn run_bte(&self, n: usize, mut b: Bte) { + let memory = self.memory.lock().clone(); + let count = b.status & 0xffff; + let bytes = count * 128; + let overflow = (b.control & 1 == 0 && b.source > (1 << 40) - bytes) + || b.destination > (1 << 40) - bytes; + let mut remaining = count; + if overflow { + b.status = remaining | ERROR; + } else { + while remaining != 0 { + let mut line = [0u64; 16]; + let mut status = BUS_OK; + if b.control & 1 == 0 { + for (i, v) in line.iter_mut().enumerate() { + let r = Self::access(memory.clone(), b.source + i as u64 * 8, 8, None); + status = r.status; + if status != BUS_OK { + break; + } + *v = r.data; + } + } + if status == BUS_OK { + for (i, v) in line.into_iter().enumerate() { + status = + Self::access(memory.clone(), b.destination + i as u64 * 8, 8, Some(v)) + .status; + if status != BUS_OK { + break; + } + } + } + if status != BUS_OK { + break; + } + remaining -= 1; + b.source = (b.source + 128) & ADDR_MASK; + b.destination = (b.destination + 128) & ADDR_MASK; + } + b.status = if remaining == 0 { 0 } else { remaining | ERROR }; + } + self.regs.lock().bte[n] = b; + if b.control & 0x10 != 0 { + if Self::access(memory, b.notification, 8, Some(b.status)).status != BUS_OK { + crate::dlog!(crate::devlog::LogModule::Bedrock, "Bedrock IO BTE notification write failed"); + } + } else { + let node = ((b.interrupt >> 1) & 255) as u8; + if node != self.regs.lock().node { + crate::dlog!(crate::devlog::LogModule::Bedrock, "Bedrock IO BTE interrupt reaches no node"); + return; + } + let hub = self.hub.lock().clone(); + let offset = if b.interrupt & 1 != 0 { 0x200090 } else { 0x90 }; + if Self::access(hub, offset, 8, Some(0x100 | ((b.interrupt >> 16) & 127))).status + != BUS_OK + { + crate::dlog!(crate::devlog::LogModule::Bedrock, "Bedrock IO BTE completion interrupt failed"); + } + } + } +} +impl Resettable for BedrockIo { + fn power_on(&self) { + let mut r = self.regs.lock(); + let peer = r.peer; + let node = r.node; + *r = Regs::default(); + r.node = node; + r.peer = peer; + if peer { + *r.words.entry(reg::XIO_STATUS).or_default() |= 1 << 13; + } + } +} +impl BusDevice for BedrockIo { + fn read64(&self, a: u64) -> BusRead64 { + if a & 7 != 0 { + return BusRead64::err(); + } + let mut r = self.regs.lock(); + if let Some((n, a)) = Self::bte_at(a) { + let b = &mut r.bte[n]; + return BusRead64::ok(match a { + bte_reg::STATUS => b.status, + bte_reg::SOURCE => b.source, + bte_reg::DESTINATION => b.destination, + bte_reg::CONTROL => { + b.status &= !BUSY; + b.control + } + bte_reg::NOTIFICATION => b.notification, + bte_reg::INTERRUPT => b.interrupt, + _ => 0, + }); + } + let v = match a { + reg::DEVICE_ID => ID, + reg::READ_ONLY | 0x1f8 => 0, + reg::LINK_STATUS => r.words.get(&a).copied().unwrap_or(0) & !(1 << 28), + reg::LINK_LOG => r.words.get(&a).copied().unwrap_or(0), + _ if Self::mask(a).is_some() => r.words.get(&a).copied().unwrap_or(0), + _ => { + crate::dlog!(crate::devlog::LogModule::Bedrock, "Bedrock IO unsupported read {a:#x}"); + 0 + } + }; + BusRead64::ok(v) + } + fn write64(&self, a: u64, v: u64) -> u32 { + if a & 7 != 0 { + return BUS_ERR; + } + let mut r = self.regs.lock(); + if let Some((n, offset)) = Self::bte_at(a) { + let b = &mut r.bte[n]; + let mut run = false; + match offset { + bte_reg::STATUS => b.status = v & (0xffff | BUSY), + bte_reg::SOURCE => b.source = v & ADDR_MASK, + bte_reg::DESTINATION => b.destination = v & ADDR_MASK, + bte_reg::CONTROL => { + b.control = v & 0x111; + if b.control & 0x100 != 0 { + crate::dlog!(crate::devlog::LogModule::Bedrock, "Bedrock IO BTE poison unsupported"); + } else if b.status & BUSY != 0 { + if b.status & 0xffff == 0 { + b.status = 0; + crate::dlog!(crate::devlog::LogModule::Bedrock, "Bedrock IO zero-count BTE start"); + } else { + run = true; + } + } + } + bte_reg::NOTIFICATION => b.notification = v & ADDR_MASK, + bte_reg::INTERRUPT => b.interrupt = v & 0x7f01ff, + _ => {} + } + let b = *b; + drop(r); + if run { + self.run_bte(n, b); + } + return BUS_OK; + } + match a { + reg::PROTECTION_KEY => { + if v == IIO_PROTECT_OVRRD_KEY { + *r.words.entry(0x100).or_default() |= 1; + } + } + reg::INBOUND_DEVICE_PERMISSION | reg::REGISTER_01E8 => { + *r.words.entry(a).or_default() |= v & Self::mask(a).unwrap(); + } + reg::XIO_STATUS => { + let v = v & Self::mask(a).unwrap() & if r.peer { !0 } else { !(1 << 13) }; + r.words.insert(a, v); + } + reg::LINK_LOG => { + if v == 0 { + r.words.insert(a, 0); + } else { + crate::dlog!(crate::devlog::LogModule::Bedrock, "Bedrock IO nonzero link-log write ignored"); + } + } + reg::READ_ONLY_ALIAS | reg::DEVICE_ID | reg::READ_ONLY => { + crate::dlog!(crate::devlog::LogModule::Bedrock, "Bedrock IO read-only or unsupported clear write {a:#x}"); + } + _ => { + if let Some(mask) = Self::mask(a) { + r.words.insert(a, v & mask); + } else { + crate::dlog!(crate::devlog::LogModule::Bedrock, "Bedrock IO unsupported write {a:#x} = {v:#x}"); + } + } + } + BUS_OK + } +} + +#[cfg(test)] +mod tests { + // Cases follow tests/qtest/sn1-bedrock-io-test.c in + // jamesbraid/qemu-sgi-origami: inbound-protection, bte and + // bte-error-termination. + use super::*; + #[derive(Default)] + struct Memory { + words: Mutex>, + fail_read: Mutex>, + fail_write: Mutex>, + } + impl BusDevice for Memory { + fn read64(&self, a: u64) -> BusRead64 { + if *self.fail_read.lock() == Some(a) { + return BusRead64::err(); + } + BusRead64::ok(self.words.lock().get(&a).copied().unwrap_or(0)) + } + fn write64(&self, a: u64, v: u64) -> u32 { + if *self.fail_write.lock() == Some(a) { + return BUS_ERR; + } + self.words.lock().insert(a, v); + BUS_OK + } + } + fn launch(d: &BedrockIo, base: u64, count: u64, src: u64, dst: u64, notify: u64, control: u64) { + d.write64(base, count | BUSY); + d.write64(base + 8, src); + d.write64(base + 0x10, dst); + d.write64(base + 0x20, notify); + d.write64(base + 0x18, control); + } + struct Reentrant(std::sync::Weak); + impl BusDevice for Reentrant { + fn read64(&self, a: u64) -> BusRead64 { + self.0.upgrade().unwrap().inbound_read(15, Some(0), a, 8) + } + } + struct Busy; + impl BusDevice for Busy { + fn read64(&self, _: u64) -> BusRead64 { + BusRead64::busy() + } + fn write64(&self, _: u64, _: u64) -> u32 { + crate::traits::BUS_BUSY + } + } + struct BlockingMemory { + entered: std::sync::mpsc::Sender<()>, + release: Mutex>, + words: Mutex>, + } + impl BusDevice for BlockingMemory { + fn write64(&self, address: u64, value: u64) -> u32 { + if address == 0x1000 { + self.entered.send(()).unwrap(); + self.release + .lock() + .recv_timeout(std::time::Duration::from_secs(5)) + .unwrap(); + } + self.words.lock().insert(address, value); + BUS_OK + } + } + + /// The guest must enable widget and requester masks before PIO or inbound DMA succeeds. + #[test] + fn the_io_block_checks_pio_and_inbound_requester_permissions() { + { + let d = BedrockIo::new(); + let m = Arc::new(Memory::default()); + d.connect_widget(8, m.clone()); + d.write64(reg::ITTE_BASE, 0x802); + assert_eq!(d.pio_write(0x20000080, 8, 0xabcd), BUS_OK); + assert_eq!(m.read64(0x40000080).data, 0xabcd); + assert_eq!(d.pio_read(0x20000080, 8).data, 0xabcd); + d.write64(reg::INBOUND_PERMISSION, 0); + assert_eq!(d.pio_read(0x20000080, 8).status, BUS_ERR); + d.write64(reg::INBOUND_PERMISSION, !0); + d.write64(reg::PIO_PERMISSION, 0); + assert_eq!(d.pio_read(0x20000080, 8).status, BUS_ERR); + d.write64(reg::PROTECTION_KEY, IIO_PROTECT_OVRRD_KEY); + assert_eq!(d.pio_read(0x20000080, 8).status, BUS_OK); + assert_eq!(d.pio_read(0x10000000, 8).status, BUS_ERR); + d.power_on(); + assert_eq!(d.read64(reg::PIO_PERMISSION).data, !0); + assert_eq!(d.read64(reg::REGISTER_01E8).data, 0x111); + + } + { + let d = BedrockIo::new(); + let m = Arc::new(Memory::default()); + let h = Arc::new(Memory::default()); + d.connect_memory(m.clone()); + d.connect_hub(h.clone()); + assert_eq!(d.inbound_write(14, Some(2), 0x80, 8, 42), BUS_OK); + assert_eq!(m.read64(0x80).data, 42); + d.disable_device(14, 2); + assert_eq!(d.inbound_read(14, Some(2), 0x80, 8).status, BUS_ERR); + assert_eq!(d.inbound_read(14, None, 0x80, 8).status, BUS_ERR); + assert_eq!(d.inbound_read(14, Some(1), 0x80, 8).status, BUS_OK); + let a = (1 << 47) | 0x1800090; + assert_eq!(d.inbound_write(15, Some(0), a, 8, 0x123), BUS_OK); + assert_eq!(h.read64(0x90).data, 0x123); + assert_eq!( + d.inbound_read(15, Some(0), a | (1 << 39), 8).status, + BUS_ERR + ); + assert_eq!(d.inbound_read(15, Some(0), 1 << 48, 8).status, BUS_ERR); + assert_eq!(d.inbound_read(15, Some(0), a + 0x800000, 8).status, BUS_ERR); + } + } + /// The guest observes BTE line copies and zero-fill through its completion registers. + #[test] + fn the_bte_copies_and_zeroes_lines_before_completion() { + let d = BedrockIo::new(); + let m = Arc::new(Memory::default()); + let h = Arc::new(Memory::default()); + d.connect_memory(m.clone()); + d.connect_hub(h.clone()); + for i in 0..32 { + m.write64(0x1000 + i * 8, 0xfeed0000 | i); + } + launch(&d, reg::BTE1_BASE, 2, 0x1000, 0x2000, 0x3000, 0x10); + for i in 0..32 { + assert_eq!(m.read64(0x2000 + i * 8).data, 0xfeed0000 | i); + } + assert_eq!(d.read64(reg::BTE1_BASE + bte_reg::STATUS).data, 0); + assert_eq!(d.read64(reg::BTE1_BASE + bte_reg::SOURCE).data, 0x1100); + assert_eq!(m.read64(0x3000).data, 0); + d.write64(reg::BTE2_BASE + bte_reg::INTERRUPT, 53 << 16); + launch(&d, reg::BTE2_BASE, 1, 0, 0x2000, 0, 1); + for i in 0..16 { + assert_eq!(m.read64(0x2000 + i * 8).data, 0); + } + assert_eq!(h.read64(0x90).data, 0x135); + } + /// A failed BTE line stops the transfer and reports the failing source and destination. + #[test] + fn the_bte_reports_transfer_errors_and_stops_after_a_failed_line() { + { + for (read, fail) in [(true, 0x1080), (false, 0x2080)] { + let d = BedrockIo::new(); + let m = Arc::new(Memory::default()); + d.connect_memory(m.clone()); + if read { + *m.fail_read.lock() = Some(fail); + } else { + *m.fail_write.lock() = Some(fail); + } + launch(&d, reg::BTE1_BASE, 3, 0x1000, 0x2000, 0x3000, 0x10); + assert_eq!(d.read64(reg::BTE1_BASE + bte_reg::STATUS).data, 2 | ERROR); + assert_eq!(d.read64(reg::BTE1_BASE + bte_reg::SOURCE).data, 0x1080); + assert_eq!(d.read64(reg::BTE1_BASE + bte_reg::DESTINATION).data, 0x2080); + assert_eq!(m.read64(0x3000).data, 2 | ERROR); + } + let d = BedrockIo::new(); + let m = Arc::new(Memory::default()); + d.connect_memory(m.clone()); + *m.fail_write.lock() = Some(0x3000); + launch(&d, reg::BTE1_BASE, 1, 0x1000, 0x2000, 0x3000, 0x10); + assert_eq!(d.read64(reg::BTE1_BASE + bte_reg::STATUS).data, 0); + launch(&d, reg::BTE1_BASE, 2, 0xffffffff80, 0x2000, 0x3000, 0x10); + assert_eq!(d.read64(reg::BTE1_BASE + bte_reg::STATUS).data, 2 | ERROR); + } + { + let d = BedrockIo::new(); + launch(&d, reg::BTE1_BASE, 1, 0, 0, 0, 0x100); + assert_eq!(d.read64(reg::BTE1_BASE + bte_reg::STATUS).data, 1 | BUSY); + d.read64(reg::BTE1_BASE + bte_reg::CONTROL); + assert_eq!(d.read64(reg::BTE1_BASE + bte_reg::STATUS).data, 1); + launch(&d, reg::BTE1_BASE, 0, 0, 0, 0, 0x10); + assert_eq!(d.read64(reg::BTE1_BASE + bte_reg::STATUS).data, 0); + } + } + /// A downstream callback may reenter the IO block while an inbound transaction is active. + #[test] + fn reentrant_inbound_forwards_are_refused_and_can_be_retried() { + let d = Arc::new(BedrockIo::new()); + d.connect_memory(Arc::new(Reentrant(Arc::downgrade(&d)))); + assert_eq!(d.inbound_read(15, Some(0), 0, 8).status, BUS_ERR); + d.connect_memory(Arc::new(Busy)); + assert_eq!( + d.inbound_read(15, Some(0), 0, 8).status, + crate::traits::BUS_BUSY + ); + d.connect_widget(8, Arc::new(Busy)); + assert_eq!(d.pio_read(0x8000000, 8).status, crate::traits::BUS_BUSY); + assert_eq!(d.pio_write(0x8000000, 8, 42), crate::traits::BUS_BUSY); + + } + /// The two guest BTEs can reach node memory without one worker holding the other off. + #[test] + fn independent_bte_workers_can_reach_memory_concurrently() { + let d = Arc::new(BedrockIo::new()); + let (entered_tx, entered_rx) = std::sync::mpsc::channel(); + let (release_tx, release_rx) = std::sync::mpsc::channel(); + let memory = Arc::new(BlockingMemory { + entered: entered_tx, + release: Mutex::new(release_rx), + words: Mutex::new(BTreeMap::new()), + }); + d.connect_memory(memory.clone()); + let first_io = d.clone(); + let first = + std::thread::spawn(move || first_io.inbound_write(14, Some(0), 0x1000, 8, 0x1234)); + entered_rx + .recv_timeout(std::time::Duration::from_secs(5)) + .unwrap(); + let second_io = d.clone(); + let second = + std::thread::spawn(move || second_io.inbound_write(15, Some(1), 0x2000, 8, 0x5678)); + let second_status = second.join().unwrap(); + release_tx.send(()).unwrap(); + let first_status = first.join().unwrap(); + assert_eq!(second_status, BUS_OK); + assert_eq!(first_status, BUS_OK); + assert_eq!( + *memory.words.lock(), + BTreeMap::from([(0x1000, 0x1234), (0x2000, 0x5678)]) + ); + } +} diff --git a/src/dev/bedrock_lb.rs b/src/dev/bedrock_lb.rs new file mode 100644 index 000000000..7f3d86b3d --- /dev/null +++ b/src/dev/bedrock_lb.rs @@ -0,0 +1,516 @@ +//! Bedrock LB provides the node and region registers, atomic scratch aliases, +//! error and vector status, and the MicroLAN master used by board identity +//! parts. The guest polls the MicroLAN completion bit after issuing a slot. +//! +//! # What is approximated here +//! +//! The slot is decoded immediately and completes after one microsecond of +//! machine time. Permission registers retain guest writes, but this block's +//! access interface does not carry requester identity, so it cannot enforce +//! those permissions. +//! +//! The plain register words sit in a sparse map keyed by offset, since the +//! 0xc040-byte block holds only a few dozen of them. +//! +//! Ported from hw/misc/sgi-bedrock-lb.c in jamesbraid/qemu-sgi-origami. + +/// Length of the block-relative register aperture. +pub const SIZE: u64 = 0xc040; + +pub mod reg { + pub const NODE_REGION: u64 = 0x0; + pub const PIO_PERMISSION: u64 = 0x4000; + pub const PIO_PERMISSION_KEY: u64 = 0x4008; + pub const INBOUND_PERMISSION: u64 = 0x4010; + pub const SOFT_RESET: u64 = 0x4018; + pub const MICRO_LAN: u64 = 0x4030; + pub const ERROR_STATUS: u64 = 0x4040; + pub const ERROR_CLEAR: u64 = 0x4048; + pub const PIO_REGION_MASK: u64 = 0x4028; + pub const NODE_REGION_CONFIG: u64 = 0x4050; + pub const NODE_REGION_MASK: u64 = 0x4058; + pub const PIO_CONFIG: u64 = 0x4100; + pub const INBOUND_CONFIG: u64 = 0x4200; + pub const INBOUND_MASK: u64 = 0x4208; + pub const VECTOR_REQUEST: u64 = 0xc000; + pub const VECTOR_STATUS: u64 = 0xc020; + pub const VECTOR_STATUS_ALIAS: u64 = 0xc038; + pub const PIO_WINDOW: u64 = 0x8000; + pub const PIO_WINDOW_END: u64 = 0x8020; + pub const ATOMIC_SCRATCH: u64 = 0x8040; + pub const ATOMIC_SCRATCH_END: u64 = 0x8050; + pub const ATOMIC_READ0: u64 = 0x8058; + pub const ATOMIC_READ1: u64 = 0x8060; + pub const REGISTER_4020: u64 = 0x4020; + pub const REGISTER_4060: u64 = 0x4060; + pub const REGISTER_C008: u64 = 0xc008; + pub const REGISTER_C010: u64 = 0xc010; + pub const REGISTER_C028: u64 = 0xc028; + pub const REGISTER_C030: u64 = 0xc030; +} + +use crate::dev::bedrock_io::IIO_PROTECT_OVRRD_KEY; +use crate::dev::w1; +use crate::traits::{BusDevice, BusRead64, Resettable, BUS_ERR, BUS_OK}; +use parking_lot::{Mutex, ReentrantMutex}; +use std::collections::BTreeMap; +use std::sync::Arc; +pub type Clock = Arc u64 + Send + Sync>; +type Level = Arc; +const ID: u64 = (0x36 << 1) | (0x3012 << 12) | (2 << 28); +const VECTOR_MASK: u64 = 3 | (0x1fffff << 3) | (0xff << 32) | (0x7ff << 40); +struct Regs { + words: BTreeMap, + scratch: [u64; 5], + node: u8, + region: u8, + deadline: Option, + sample: bool, +} +impl Default for Regs { + fn default() -> Self { + let mut words = BTreeMap::new(); + words.insert(0x4000, !0); + words.insert(0x4010, !0); + Self { + words, + scratch: [0; 5], + node: 0, + region: 0, + deadline: None, + sample: true, + } + } +} +pub struct BedrockLb { + regs: Arc>, + clock: Clock, + error: Mutex>, + delivery: ReentrantMutex<()>, + driven: Mutex>, + reset: Mutex>>, + microlan: Mutex>, + node_changed: Mutex>>, +} +impl BedrockLb { + pub fn new(clock: Clock) -> Self { + let regs = Arc::new(Mutex::new(Regs::default())); + crate::locks::register_mutex("bedrock-lb", ®s); + Self { + regs, + clock, + error: Mutex::new(None), + delivery: ReentrantMutex::new(()), + driven: Mutex::new(None), + reset: Mutex::new(None), + microlan: Mutex::new(None), + node_changed: Mutex::new(None), + } + } + pub fn connect_error(&self, cb: Level) { + let _delivery = self.delivery.lock(); + *self.error.lock() = Some(cb); + *self.driven.lock() = None; + self.update_irq(); + } + /// Attach the node's 1-wire bus to the local-block MicroLAN master. + pub fn set_microlan(&self, bus: w1::Bus) { + *self.microlan.lock() = Some(bus); + } + pub fn connect_soft_reset(&self, cb: Arc) { + *self.reset.lock() = Some(cb); + } + pub fn connect_node_changed(&self, cb: Arc) { + let _delivery = self.delivery.lock(); + *self.node_changed.lock() = Some(cb); + } + pub fn node_number(&self) -> u8 { + self.regs.lock().node + } + pub fn error_level(&self) -> bool { + self.regs.lock().words.get(&0x4040).copied().unwrap_or(0) != 0 + } + fn update_irq(&self) { + // Serialize the sample and callback, while leaving registers available + // to the callback. Suppress unchanged levels so callbacks can reenter + // register reads that recompute the output. + let _delivery = self.delivery.lock(); + let cb = self.error.lock().clone(); + if let Some(cb) = cb { + let level = self.error_level(); + let changed = { + let mut driven = self.driven.lock(); + let changed = *driven != Some(level); + *driven = Some(level); + changed + }; + if changed { + cb(level); + } + } + } + fn notify_node(&self) { + let _delivery = self.delivery.lock(); + let cb = self.node_changed.lock().clone(); + if let Some(cb) = cb { + cb(self.node_number()); + } + } + fn complete(r: &mut Regs, now: u64) { + if r.deadline.is_some_and(|d| now >= d) { + *r.words.entry(0x4030).or_default() |= 2 | u64::from(r.sample); + r.deadline = None; + } + } + fn mask(a: u64) -> Option { + Some(match a { + reg::PIO_PERMISSION | reg::INBOUND_PERMISSION | reg::REGISTER_4020 | reg::REGISTER_4060 | reg::REGISTER_C008 | reg::REGISTER_C010 | reg::REGISTER_C028 | reg::REGISTER_C030 => !0, + reg::PIO_REGION_MASK => 0x80ff80ff80ff80ff, + reg::ERROR_STATUS => 0x7ff, + reg::NODE_REGION_CONFIG => 0x7f | (0x7ff << 8) | (0x7ff << 20) | (0xf << 32) | (1 << 40) | (1 << 44), + reg::NODE_REGION_MASK => (1 << 38) - 1, + reg::PIO_CONFIG => 0x3f, + reg::INBOUND_CONFIG => 1 | (0x3ff << 4) | (0x3ff << 16) | (1 << 28) | (1 << 32), + reg::INBOUND_MASK => { + 0x1f | (0xfff << 8) | (1 << 20) | (0x3ff << 24) | (0x3ff << 36) | (0x3ff << 48) + } + reg::VECTOR_REQUEST => VECTOR_MASK, + _ => return None, + }) + } +} +impl Resettable for BedrockLb { + fn power_on(&self) { + let mut r = self.regs.lock(); + let survivors = [ + r.scratch[1], + r.scratch[2], + r.words.get(®::ERROR_STATUS).copied().unwrap_or(0), + ]; + *r = Regs::default(); + r.scratch[1] = survivors[0]; + r.scratch[2] = survivors[1]; + r.words.insert(reg::ERROR_STATUS, survivors[2]); + drop(r); + self.update_irq(); + self.notify_node(); + } +} +impl BusDevice for BedrockLb { + fn read64(&self, a: u64) -> BusRead64 { + if a & 7 != 0 { + return BusRead64::err(); + } + let now = (self.clock)(); + let mut r = self.regs.lock(); + Self::complete(&mut r, now); + let v = match a { + reg::NODE_REGION => ID | ((r.node as u64) << 32) | ((r.region as u64) << 46), + reg::PIO_WINDOW..=reg::PIO_WINDOW_END => r.scratch[(a as usize - reg::PIO_WINDOW as usize) / 8], + reg::ATOMIC_READ0 | reg::ATOMIC_READ1 => { + let n = 3 + (a as usize - reg::ATOMIC_READ0 as usize) / 8; + let v = r.scratch[n]; + r.scratch[n] = 1; + v + } + reg::VECTOR_STATUS_ALIAS => r.words.remove(&0xc020).unwrap_or(0), + reg::VECTOR_STATUS | reg::MICRO_LAN => r.words.get(&a).copied().unwrap_or(0), + _ if Self::mask(a).is_some() => r.words.get(&a).copied().unwrap_or(0), + _ => { + crate::dlog!(crate::devlog::LogModule::Bedrock, "Bedrock LB unsupported read {a:#x}"); + 0 + } + }; + BusRead64::ok(v) + } + fn write64(&self, a: u64, v: u64) -> u32 { + if a & 7 != 0 { + return BUS_ERR; + } + let now = (self.clock)(); + let mut r = self.regs.lock(); + Self::complete(&mut r, now); + match a { + reg::NODE_REGION => { + let region = ((v >> 46) & 3) as u8; + if region == 3 { + crate::dlog!(crate::devlog::LogModule::Bedrock, "Bedrock LB invalid region size"); + } else { + r.node = (v >> 32) as u8; + r.region = region; + } + } + reg::PIO_WINDOW..=reg::PIO_WINDOW_END => { + let n = (a as usize - 0x8000) / 8; + r.scratch[n] = if n < 3 { v } else { v & 1 }; + } + reg::ATOMIC_SCRATCH..=reg::ATOMIC_SCRATCH_END => { + let n = (a as usize - reg::ATOMIC_SCRATCH as usize) / 8; + if r.scratch[n] == 0 { + r.scratch[n] = v; + } + } + reg::PIO_PERMISSION_KEY => { + if v == IIO_PROTECT_OVRRD_KEY { + *r.words.entry(reg::PIO_PERMISSION).or_default() |= 1; + } + } + reg::SOFT_RESET => {} + reg::MICRO_LAN => { + if r.deadline.is_some() { + crate::dlog!(crate::devlog::LogModule::Bedrock, "Bedrock LB MicroLAN write while slot pending discarded"); + } else { + r.words.insert(a, v & ((1 << 34) - 4)); + let (pulse, sample) = w1::slot::fields(v); + r.sample = match self.microlan.lock().as_mut() { + Some(bus) => w1::slot::slot(bus, pulse, sample), + None => pulse >= w1::slot::RESET_PULSE + || pulse < w1::slot::ONE_PULSE, + }; + r.deadline = Some(now.saturating_add(1000)); + } + } + reg::ERROR_STATUS => { + *r.words.entry(a).or_default() |= v & 0x7ff; + } + reg::ERROR_CLEAR => { + *r.words.entry(0x4040).or_default() &= !(v & 0x7ff); + } + reg::VECTOR_REQUEST => { + r.words.insert(a, v & VECTOR_MASK); + if r.words.get(®::VECTOR_STATUS).copied().unwrap_or(0) & (1 << 63) != 0 { + *r.words.entry(reg::VECTOR_STATUS).or_default() |= 1 << 62; + } else { + crate::dlog!(crate::devlog::LogModule::Bedrock, "Bedrock LB vector launch reaches no peer; no reply latched"); + } + } + reg::VECTOR_STATUS | reg::VECTOR_STATUS_ALIAS => { + if v == 0 { + r.words.insert(reg::VECTOR_STATUS, 0); + } else { + crate::dlog!(crate::devlog::LogModule::Bedrock, "Bedrock LB nonzero vector status write ignored"); + } + } + _ => { + if let Some(mask) = Self::mask(a) { + r.words.insert(a, v & mask); + if (a == reg::PIO_PERMISSION || a == reg::INBOUND_PERMISSION) && v != !0 { + crate::dlog!(crate::devlog::LogModule::Bedrock, "Bedrock LB permission enforcement requires requester context"); + } + } else { + crate::dlog!(crate::devlog::LogModule::Bedrock, "Bedrock LB unsupported write {a:#x} = {v:#x}"); + } + } + } + drop(r); + self.update_irq(); + if a == reg::NODE_REGION { + self.notify_node(); + } + if a == reg::SOFT_RESET && v & 1 != 0 { + self.power_on(); + let cb = self.reset.lock().clone(); + if let Some(cb) = cb { + cb(); + } + } + BUS_OK + } +} + +#[cfg(test)] +mod tests { + // Cases follow tests/qtest/sn1-bedrock-lb-test.c in + // jamesbraid/qemu-sgi-origami: identity-permissions, scratch-reset and + // microlan-slot. + use super::*; + use std::sync::atomic::{AtomicBool, AtomicU8, AtomicU64, Ordering}; + /// Hold an IRQ packet's assertion callback while a CPU clears the register. + pub(super) fn check_order( + device: Arc, + install: impl FnOnce(Level), + raise: u64, + value: u64, + clear: impl FnOnce() + Send + 'static, + ) { + let (entered_tx, entered_rx) = std::sync::mpsc::channel(); + let (release_tx, release_rx) = std::sync::mpsc::channel(); + let release_rx = Mutex::new(release_rx); + let delivered = Arc::new(AtomicBool::new(false)); + let sink = delivered.clone(); + install(Arc::new(move |level| { + if level { + entered_tx.send(()).unwrap(); + release_rx + .lock() + .recv_timeout(std::time::Duration::from_secs(5)) + .unwrap(); + } + sink.store(level, Ordering::SeqCst); + })); + let io = crate::dev::bedrock_io::BedrockIo::new(); + io.connect_hub(device); + let first = std::thread::spawn(move || { + io.inbound_write(14, Some(0), (1 << 47) | 0x1800000 | raise, 8, value) + }); + entered_rx + .recv_timeout(std::time::Duration::from_secs(5)) + .unwrap(); + let (done_tx, done_rx) = std::sync::mpsc::channel(); + let second = std::thread::spawn(move || { + clear(); + done_tx.send(()).unwrap(); + }); + // An ordered callback waits for the blocked assertion. A stale publisher + // instead completes the clear early, then overwrites it after release. + let _ = done_rx.recv_timeout(std::time::Duration::from_millis(100)); + release_tx.send(()).unwrap(); + assert_eq!(first.join().unwrap(), BUS_OK); + second.join().unwrap(); + assert!(!delivered.load(Ordering::SeqCst)); + } + + /// The guest configures node permissions and expects selected scratch words to survive reset. + #[test] + fn permission_writes_latch_errors_and_reset_retains_scratch() { + { + let d = BedrockLb::new(Arc::new(|| 0)); + d.write64(reg::NODE_REGION, (0x5a << 32) | (2 << 46)); + assert_eq!(d.node_number(), 0x5a); + d.write64(reg::NODE_REGION, 3 << 46); + assert_eq!(d.node_number(), 0x5a); + d.write64(reg::PIO_PERMISSION, 0); + d.write64(reg::PIO_PERMISSION_KEY, 0); + assert_eq!(d.read64(reg::PIO_PERMISSION).data, 0); + d.write64(reg::PIO_PERMISSION_KEY, IIO_PROTECT_OVRRD_KEY); + assert_eq!(d.read64(reg::PIO_PERMISSION).data, 1); + d.write64(reg::PIO_WINDOW + 8, 42); + d.write64(reg::ERROR_STATUS, !0); + assert_eq!(d.read64(reg::ERROR_STATUS).data, 0x7ff); + d.power_on(); + assert_eq!(d.read64(reg::PIO_WINDOW + 8).data, 42); + assert!(d.error_level()); + d.write64(reg::ERROR_CLEAR, 0x7ff); + assert!(!d.error_level()); + assert_eq!(d.node_number(), 0); + } + { + let d = BedrockLb::new(Arc::new(|| 0)); + d.write64(reg::ATOMIC_SCRATCH, 42); + d.write64(reg::ATOMIC_SCRATCH, 43); + assert_eq!(d.read64(reg::PIO_WINDOW).data, 42); + d.write64(reg::REGISTER_C030, 0xdead); + d.write64(reg::VECTOR_REQUEST, !0); + assert_eq!(d.read64(reg::VECTOR_STATUS).data, 0); + assert_eq!(d.read64(reg::REGISTER_C030).data, 0xdead); + assert_eq!(d.read32(reg::PIO_WINDOW).status, BUS_ERR); + assert_eq!(d.read64(1).status, BUS_ERR); + } + } + /// The guest polls MicroLAN completion and cannot replace a slot that is still pending. + #[test] + fn microlan_slots_finish_after_one_microsecond_and_reject_overlap() { + let time = Arc::new(AtomicU64::new(0)); + let t = time.clone(); + let d = BedrockLb::new(Arc::new(move || t.load(Ordering::SeqCst))); + let word = (0x66 << 27) | (60 << 10) | (0x41 << 2) | 3; + d.write64(reg::MICRO_LAN, word); + assert_eq!(d.read64(reg::MICRO_LAN).data, word & !3); + d.write64(reg::MICRO_LAN, 8 << 10); + assert_eq!(d.read64(reg::MICRO_LAN).data, word & !3); + time.store(1000, Ordering::SeqCst); + assert_eq!(d.read64(reg::MICRO_LAN).data, (word & !3) | 2); + d.write64(reg::MICRO_LAN, 8 << 10); + time.store(2000, Ordering::SeqCst); + assert_eq!(d.read64(reg::MICRO_LAN).data, (8 << 10) | 3); + d.write64(reg::MICRO_LAN, 60 << 10); + d.power_on(); + time.store(3000, Ordering::SeqCst); + assert_eq!(d.read64(reg::MICRO_LAN).data, 0); + } + /// Node-change callbacks may reenter LB while guest writes race. + #[test] + fn notifications_preserve_concurrent_register_order() { + { + let ni = Arc::new(crate::dev::bedrock_ni::BedrockNi::new()); + let a = ni.clone(); + let b = ni.clone(); + check_order( + ni, + move |cb| a.connect_error(cb), + crate::dev::bedrock_ni::reg::ERROR_STATUS, + 1 << 24, + move || { + b.read64(crate::dev::bedrock_ni::reg::ERROR_CLEAR); + }, + ); + } + { + let d = Arc::new(BedrockLb::new(Arc::new(|| 0))); + let a = d.clone(); + let b = d.clone(); + check_order( + d, + move |cb| a.connect_error(cb), + reg::ERROR_STATUS, + 1, + move || { + b.write64(reg::ERROR_CLEAR, 1); + }, + ); + } + { + let d = Arc::new(crate::dev::bedrock_xb::BedrockXb::new()); + let a = d.clone(); + let b = d.clone(); + check_order( + d, + move |cb| a.connect_error(cb), + crate::dev::bedrock_xb::reg::QUEUE_BASE, + 1, + move || { + b.write64(crate::dev::bedrock_xb::reg::CLEAR_QUEUE_BASE, 0); + b.write64(crate::dev::bedrock_xb::reg::CLEAR_FIRST_ERROR, 0); + }, + ); + } + { + let d = Arc::new(BedrockLb::new(Arc::new(|| 0))); + let delivered = Arc::new(AtomicU8::new(0)); + let sink = delivered.clone(); + let (entered_tx, entered_rx) = std::sync::mpsc::channel(); + let (release_tx, release_rx) = std::sync::mpsc::channel(); + let release_rx = Mutex::new(release_rx); + d.connect_node_changed(Arc::new(move |node| { + if node == 1 { + entered_tx.send(()).unwrap(); + release_rx + .lock() + .recv_timeout(std::time::Duration::from_secs(5)) + .unwrap(); + } + sink.store(node, Ordering::SeqCst); + })); + let io = crate::dev::bedrock_io::BedrockIo::new(); + io.connect_hub(d.clone()); + let first = std::thread::spawn(move || { + io.inbound_write(14, Some(0), (1 << 47) | 0x1800000, 8, 1 << 32) + }); + entered_rx + .recv_timeout(std::time::Duration::from_secs(5)) + .unwrap(); + let (done_tx, done_rx) = std::sync::mpsc::channel(); + let other = d.clone(); + let second = std::thread::spawn(move || { + other.write64(0, 2 << 32); + done_tx.send(()).unwrap(); + }); + let _ = done_rx.recv_timeout(std::time::Duration::from_millis(100)); + release_tx.send(()).unwrap(); + assert_eq!(first.join().unwrap(), BUS_OK); + second.join().unwrap(); + assert_eq!(d.node_number(), 2); + assert_eq!(delivered.load(Ordering::SeqCst), 2); + } + } +} diff --git a/src/dev/bedrock_ni.rs b/src/dev/bedrock_ni.rs new file mode 100644 index 000000000..06f1ee57c --- /dev/null +++ b/src/dev/bedrock_ni.rs @@ -0,0 +1,335 @@ +//! Bedrock NI exposes node routing, sideband and error registers. Fuel runs as +//! one node without a NUMAlink peer, but the guest still reads and programs +//! these local control words during machine setup. +//! +//! # What is approximated here +//! +//! Routing and sideband writes are retained; the network fabric and packet +//! delivery are not simulated. Unspecified power-on fields start at zero, and +//! the sideband word survives reset as the current compatibility behavior. +//! +//! Ported from hw/misc/sgi-bedrock-ni.c and hw/misc/sgi-bedrock-ni-regs.c +//! in jamesbraid/qemu-sgi-origami. + +/// Length of the block-relative register aperture. +pub const SIZE: u64 = 0x2008; + +pub mod reg { + pub const NODE_ID: u64 = 0x0; + pub const LOCAL_RESET: u64 = 0x8; + pub const CONTROL_0: u64 = 0x10; + pub const CONTROL_1: u64 = 0x18; + pub const CONTROL_2: u64 = 0x20; + pub const CONTROL_3: u64 = 0x30; + pub const CONTROL_4: u64 = 0x38; + pub const ERROR_STATUS: u64 = 0x100; + pub const ERROR_STATUS_UPPER: u64 = 0x108; + pub const ERROR_STATUS_LOWER: u64 = 0x110; + pub const SIDE_BAND: u64 = 0x118; + pub const ERROR_CLEAR: u64 = 0x120; + pub const ROUTES: u64 = 0x1000; + pub const ROUTES_END: u64 = 0x1400; + pub const GLOBAL_CONTROL: u64 = 0x2000; +} + +use crate::traits::{BusDevice, BusRead64, Resettable, BUS_ERR, BUS_OK}; +use parking_lot::{Mutex, ReentrantMutex}; +use std::sync::Arc; +type Level = Arc; +#[derive(Clone)] +struct Regs { + control: [u64; 8], + routes: [u64; 128], + global: u64, + errors: u64, + sideband: u64, +} +impl Default for Regs { + fn default() -> Self { + Self { + control: [0; 8], + routes: [0; 128], + global: 0, + errors: 0, + sideband: 0, + } + } +} +pub struct BedrockNi { + regs: Arc>, + error: Mutex>, + delivery: ReentrantMutex<()>, + driven: Mutex>, + reset: Mutex>>, +} +impl Default for BedrockNi { + fn default() -> Self { + Self::new() + } +} +impl BedrockNi { + pub fn new() -> Self { + let regs = Arc::new(Mutex::new(Regs::default())); + crate::locks::register_mutex("bedrock-ni", ®s); + Self { + regs, + error: Mutex::new(None), + delivery: ReentrantMutex::new(()), + driven: Mutex::new(None), + reset: Mutex::new(None), + } + } + pub fn connect_error(&self, cb: Level) { + let _delivery = self.delivery.lock(); + *self.error.lock() = Some(cb); + *self.driven.lock() = None; + self.update_irq(); + } + pub fn connect_local_reset(&self, cb: Arc) { + *self.reset.lock() = Some(cb); + } + pub fn error_level(&self) -> bool { + self.regs.lock().errors & 0x1fffffff000000 != 0 + } + fn update_irq(&self) { + // Serialize the sample and callback, while leaving registers available + // to the callback. Suppress unchanged levels so callbacks can reenter + // register reads that recompute the output. + let _delivery = self.delivery.lock(); + let cb = self.error.lock().clone(); + if let Some(cb) = cb { + let level = self.error_level(); + let changed = { + let mut driven = self.driven.lock(); + let changed = *driven != Some(level); + *driven = Some(level); + changed + }; + if changed { + cb(level); + } + } + } + fn mask(a: u64) -> u64 { + match a { + reg::CONTROL_0 => 1, + reg::CONTROL_1 => 7, + reg::CONTROL_2 => 0x1ff, + reg::CONTROL_3 => 0x7efffffff, + reg::CONTROL_4 => 0xffff, + _ => 0, + } + } +} +impl Resettable for BedrockNi { + fn power_on(&self) { + let mut r = self.regs.lock(); + let sideband = r.sideband; + *r = Regs::default(); + r.sideband = sideband; + drop(r); + self.update_irq(); + } +} +impl BusDevice for BedrockNi { + fn read64(&self, a: u64) -> BusRead64 { + if a & 7 != 0 { + return BusRead64::err(); + } + let mut r = self.regs.lock(); + let v = if Self::mask(a) != 0 { + r.control[a as usize / 8] + } else if (reg::ROUTES..reg::ROUTES_END).contains(&a) { + r.routes[(a as usize - 0x1000) / 8] + } else { + match a { + reg::NODE_ID => 0, + reg::ERROR_STATUS => r.errors, + reg::ERROR_STATUS_UPPER | reg::ERROR_STATUS_LOWER => 0, + reg::SIDE_BAND => r.sideband, + reg::ERROR_CLEAR => { + let v = r.errors; + r.errors = 0; + v + } + reg::GLOBAL_CONTROL => r.global, + _ => { + crate::dlog!(crate::devlog::LogModule::Bedrock, "Bedrock NI unsupported read {a:#x}"); + 0 + } + } + }; + drop(r); + self.update_irq(); + BusRead64::ok(v) + } + fn write64(&self, a: u64, v: u64) -> u32 { + if a & 7 != 0 { + return BUS_ERR; + } + let mut r = self.regs.lock(); + if Self::mask(a) != 0 { + r.control[a as usize / 8] = v & Self::mask(a); + } else if (reg::ROUTES..reg::ROUTES_END).contains(&a) { + r.routes[(a as usize - 0x1000) / 8] = v & 31; + } else { + match a { + reg::LOCAL_RESET => {} + reg::ERROR_STATUS => r.errors = v & ((1 << 53) - 1), + reg::SIDE_BAND => r.sideband = v & 0xfffff, + reg::GLOBAL_CONTROL => r.global = v & 31, + _ => crate::dlog!(crate::devlog::LogModule::Bedrock, "Bedrock NI unsupported or read-only write {a:#x} = {v:#x}"), + } + } + drop(r); + self.update_irq(); + if a == reg::LOCAL_RESET && v & 4 != 0 { + let cb = self.reset.lock().clone(); + if let Some(cb) = cb { + cb(); + } + } + BUS_OK + } +} + +#[cfg(test)] +mod tests { + // Cases ported from tests/qtest/sn1-bedrock-test.c in jamesbraid/qemu-sgi-origami. + use super::*; + use std::sync::atomic::{AtomicU64, Ordering}; + + /// The guest programs route fields and reads the error register to acknowledge faults. + #[test] + fn route_registers_store_fields_and_error_reads_clear_pending_errors() { + let d = Arc::new(BedrockNi::new()); + for (a, mask) in [ + (reg::CONTROL_0, 1), + (reg::CONTROL_1, 7), + (reg::CONTROL_2, 0x1ff), + (reg::CONTROL_3, 0x7efffffff), + (reg::CONTROL_4, 0xffff), + ] { + d.write64(a, !0); + assert_eq!(d.read64(a).data, mask); + } + for a in [reg::ROUTES, reg::ROUTES_END - 8] { + d.write64(a, !0); + assert_eq!(d.read64(a).data, 31); + } + d.write64(reg::GLOBAL_CONTROL, !0); + assert_eq!(d.read64(reg::GLOBAL_CONTROL).data, 31); + let weak = Arc::downgrade(&d); + d.connect_error(Arc::new(move |_| { + let d = weak.upgrade().unwrap(); + let _ = d.read64(reg::SIDE_BAND); + })); + let resets = Arc::new(AtomicU64::new(0)); + let count = resets.clone(); + d.connect_local_reset(Arc::new(move || { + count.fetch_add(1, Ordering::SeqCst); + })); + d.write64(reg::CONTROL_0, 0); + d.write64(reg::LOCAL_RESET, 4); + assert_eq!(resets.load(Ordering::SeqCst), 1); + d.write64(reg::ERROR_STATUS, 0xffffff); + assert!(!d.error_level()); + d.write64(reg::ERROR_STATUS, 1 << 24); + assert!(d.error_level()); + d.read64(reg::ERROR_CLEAR); + assert!(!d.error_level()); + d.write64(reg::SIDE_BAND, !0); + d.power_on(); + assert_eq!(d.read64(reg::SIDE_BAND).data, 0xfffff); + assert_eq!(d.read64(reg::CONTROL_3).data, 0); + } + + /// The PROM's one power-on pass writes a sign-extended port-parameter constant and zero + /// to the error and sideband words; each must read back through its defined bits only. + #[test] + fn power_on_programming_pass_reads_back_through_the_defined_bits() { + let d = BedrockNi::new(); + d.write64(reg::CONTROL_0, 1); + d.write64(reg::CONTROL_1, 0); + d.write64(reg::CONTROL_2, 0x166); + d.write64(reg::CONTROL_3, 0xffff_ffff_a3ff_1990); + d.write64(reg::CONTROL_4, 0xc800); + d.write64(reg::ERROR_STATUS, 0); + d.read64(reg::ERROR_CLEAR); + d.write64(reg::SIDE_BAND, 0); + for (a, want) in [ + (reg::CONTROL_0, 1), + (reg::CONTROL_1, 0), + (reg::CONTROL_2, 0x166), + (reg::CONTROL_3, 0x7_a3ff_1990), + (reg::CONTROL_4, 0xc800), + (reg::ERROR_STATUS, 0), + (reg::SIDE_BAND, 0), + (reg::ERROR_CLEAR, 0), + ] { + assert_eq!(d.read64(a).data, want, "register {a:#x}"); + } + } + + /// Only the local-reset bit of the reset word reaches the node; the other bits must + /// not restart it, and the interface's own registers are left to the board's reset. + #[test] + fn only_the_local_reset_bit_requests_a_node_reset() { + let d = BedrockNi::new(); + let resets = Arc::new(AtomicU64::new(0)); + let count = resets.clone(); + d.connect_local_reset(Arc::new(move || { + count.fetch_add(1, Ordering::SeqCst); + })); + let mut expected = 0; + for enable in 0..2 { + d.write64(reg::CONTROL_0, enable); + for command in 0..8u64 { + d.write64(reg::LOCAL_RESET, command); + expected += u64::from(command & 4 != 0); + assert_eq!(resets.load(Ordering::SeqCst), expected, "command {command:#x}"); + } + } + // With nothing wired to the callback the request does not clear this block. + assert_eq!(d.read64(reg::CONTROL_0).data, 1); + } + + /// A latched port error raises the interrupt line through its interrupting bits only; + /// drivers rely on the line dropping when the status is rewritten or cleared. + #[test] + fn stored_error_status_drives_the_error_line_through_the_interrupting_bits() { + let d = BedrockNi::new(); + let level = Arc::new(AtomicU64::new(0)); + let seen = level.clone(); + d.connect_error(Arc::new(move |high| seen.store(u64::from(high), Ordering::SeqCst))); + for (value, high) in [(1u64 << 30, 1), (0xff, 0), (1 << 52, 1), (0, 0)] { + d.write64(reg::ERROR_STATUS, value); + assert_eq!(level.load(Ordering::SeqCst), high, "status {value:#x}"); + } + // Bit 53 is beyond the register and must not be retained. + d.write64(reg::ERROR_STATUS, 1 << 53); + assert_eq!(d.read64(reg::ERROR_STATUS).data, 0); + } + + /// Reset values of the programmable words are unknown on hardware, so the model clears them; + /// a guest that re-runs its setup after a reset must see zeroes, not stale routes. + #[test] + fn reset_clears_every_programmable_register_and_misaligned_accesses_fault() { + let d = BedrockNi::new(); + let offsets = [ + reg::CONTROL_0, reg::CONTROL_1, reg::CONTROL_2, reg::CONTROL_3, reg::CONTROL_4, + reg::ROUTES, reg::ROUTES_END - 8, reg::GLOBAL_CONTROL, + ]; + for a in offsets { + d.write64(a, !0); + } + // A misaligned write must not land in the neighbouring register. + assert_eq!(d.write64(reg::CONTROL_4 + 1, 0), BUS_ERR); + assert_eq!(d.read64(reg::CONTROL_4 + 1).status, BUS_ERR); + assert_eq!(d.read64(reg::CONTROL_4).data, 0xffff); + d.power_on(); + for a in offsets { + assert_eq!(d.read64(a).data, 0, "register {a:#x}"); + } + } +} diff --git a/src/dev/bedrock_xb.rs b/src/dev/bedrock_xb.rs new file mode 100644 index 000000000..6df8e7b1c --- /dev/null +++ b/src/dev/bedrock_xb.rs @@ -0,0 +1,278 @@ +//! Bedrock XB is the node's internal crossbar status block. It records the +//! fabric error queues and lets software service queued errors; the Fuel board +//! connects its outputs to the Bedrock processor interface. +//! +//! # What is approximated here +//! +//! Arbitration timing and diagnostic outputs are not modeled. Unsupported +//! queue reads return zero, while writes are logged and ignored. +//! +//! Ported from hw/misc/sgi-bedrock-xb.c in jamesbraid/qemu-sgi-origami. + +/// Length of the block-relative register aperture. +pub const SIZE: u64 = 0x198; + +pub mod reg { + pub const CONTROL_BASE: u64 = 0x0; + pub const CONTROL_END: u64 = 0x18; + pub const QUEUE_BASE: u64 = 0x20; + pub const QUEUE_END: u64 = 0x88; + pub const FIRST_ERROR: u64 = 0x90; + pub const DEBUG_BASE: u64 = 0x98; + pub const DEBUG_END: u64 = 0xa0; + pub const CLEAR_QUEUE_BASE: u64 = 0x120; + pub const CLEAR_QUEUE_END: u64 = 0x188; + pub const CLEAR_FIRST_ERROR: u64 = 0x190; +} + +use crate::traits::{BusDevice, BusRead64, Resettable, BUS_ERR, BUS_OK}; +use parking_lot::{Mutex, ReentrantMutex}; +use std::sync::Arc; +type Level = Arc; +const MASKS: [u64; 14] = [ + 0x3333, 0x33, 0, 0, 0x3330, 0, 0, 0xf0, 0xfff0, 0, 0, 0, 0x3303, 0x11, +]; +struct Regs { + control: [u64; 4], + queues: [u64; 14], + first: u64, + debug: [u64; 2], + unserviced: u16, +} +impl Default for Regs { + fn default() -> Self { + Self { + control: [0x0008000080000021, 0, 0, 0], + queues: [0; 14], + first: 0, + debug: [0; 2], + unserviced: 0, + } + } +} +pub struct BedrockXb { + regs: Arc>, + error: Mutex>, + delivery: ReentrantMutex<()>, + driven: Mutex>, +} +impl Default for BedrockXb { + fn default() -> Self { + Self::new() + } +} +impl BedrockXb { + pub fn new() -> Self { + let regs = Arc::new(Mutex::new(Regs::default())); + crate::locks::register_mutex("bedrock-xb", ®s); + Self { + regs, + error: Mutex::new(None), + delivery: ReentrantMutex::new(()), + driven: Mutex::new(None), + } + } + pub fn connect_error(&self, cb: Level) { + let _delivery = self.delivery.lock(); + *self.error.lock() = Some(cb); + *self.driven.lock() = None; + self.update_irq(); + } + pub fn error_level(&self) -> bool { + self.regs.lock().first & 0x1000 != 0 + } + fn update_irq(&self) { + // Serialize the sample and callback, while leaving registers available + // to the callback. Suppress unchanged levels so callbacks can reenter + // register reads that recompute the output. + let _delivery = self.delivery.lock(); + let cb = self.error.lock().clone(); + if let Some(cb) = cb { + let level = self.error_level(); + let changed = { + let mut driven = self.driven.lock(); + let changed = *driven != Some(level); + *driven = Some(level); + changed + }; + if changed { + cb(level); + } + } + } +} +impl Resettable for BedrockXb { + fn power_on(&self) { + *self.regs.lock() = Regs::default(); + self.update_irq(); + } +} +impl BusDevice for BedrockXb { + fn read64(&self, a: u64) -> BusRead64 { + if a & 7 != 0 { + return BusRead64::err(); + } + let r = self.regs.lock(); + let v = match a { + reg::CONTROL_BASE..=reg::CONTROL_END => r.control[a as usize / 8], + reg::QUEUE_BASE..=reg::QUEUE_END => { + let n = (a as usize - 0x20) / 8; + if MASKS[n] == 0 { + crate::dlog!(crate::devlog::LogModule::Bedrock, "Bedrock XB unsupported queue {n}"); + } + r.queues[n] + } + reg::FIRST_ERROR => r.first, + reg::DEBUG_BASE | reg::DEBUG_END => r.debug[(a as usize - 0x98) / 8], + _ => { + crate::dlog!(crate::devlog::LogModule::Bedrock, "Bedrock XB unsupported read {a:#x}"); + 0 + } + }; + BusRead64::ok(v) + } + fn write64(&self, a: u64, v: u64) -> u32 { + if a & 7 != 0 { + return BUS_ERR; + } + let mut r = self.regs.lock(); + match a { + reg::CONTROL_BASE..=reg::CONTROL_END => { + let n = a as usize / 8; + r.control[n] = v & [0x000ffffffffffff1, 0xffffffff, 0x7ff, 0xf][n]; + if a == 0x10 && r.control[n] != 0 { + crate::dlog!(crate::devlog::LogModule::Bedrock, "Bedrock XB traffic observation unsupported"); + } + } + reg::QUEUE_BASE..=reg::QUEUE_END => { + let n = (a as usize - 0x20) / 8; + let v = v & MASKS[n]; + if MASKS[n] == 0 { + crate::dlog!(crate::devlog::LogModule::Bedrock, "Bedrock XB unsupported queue {n}"); + } + if v != 0 { + r.queues[n] |= v; + r.unserviced |= 1 << n; + if r.first & 0x1000 == 0 { + r.first = 0x1000 | ((15 - n) as u64) << 8; + } + } + } + reg::CLEAR_QUEUE_BASE..=reg::CLEAR_QUEUE_END => { + let n = (a as usize - 0x120) / 8; + r.queues[n] = 0; + r.unserviced &= !(1 << n); + } + reg::CLEAR_FIRST_ERROR => { + if r.first & 0x1000 != 0 { + let n = 15 - ((r.first >> 8) & 15) as usize; + if r.unserviced & (1 << n) != 0 { + crate::dlog!(crate::devlog::LogModule::Bedrock, "Bedrock XB first error cleared before queue"); + } else { + r.first = 0; + for n in 0..14 { + if r.queues[n] != 0 { + r.first = 0x1000 | ((15 - n) as u64) << 8; + break; + } + } + } + } + } + reg::DEBUG_BASE | reg::DEBUG_END => { + let n = (a as usize - 0x98) / 8; + r.debug[n] = v & [0xffff, 7][n]; + if r.debug[n] != 0 { + crate::dlog!(crate::devlog::LogModule::Bedrock, "Bedrock XB debug output unsupported"); + } + } + _ => crate::dlog!(crate::devlog::LogModule::Bedrock, "Bedrock XB unsupported or read-only write {a:#x} = {v:#x}"), + } + drop(r); + self.update_irq(); + BUS_OK + } +} + +#[cfg(test)] +mod tests { + // Cases follow tests/qtest/sn1-bedrock-xb-test.c in + // jamesbraid/qemu-sgi-origami: control-storage, service-order and + // cold-reset. + use super::*; + + /// The guest sees masked crossbar control words and cleared errors after reset. + #[test] + fn crossbar_control_masks_apply_and_reset_clears_errors() { + let d = BedrockXb::new(); + for (a, mask) in [ + (reg::CONTROL_BASE, 0x000ffffffffffff1), + (reg::CONTROL_BASE + 8, 0xffffffff), + (reg::CONTROL_BASE + 0x10, 0x7ff), + (reg::CONTROL_END, 15), + (reg::DEBUG_BASE, 0xffff), + (reg::DEBUG_END, 7), + ] { + d.write64(a, !0); + assert_eq!(d.read64(a).data, mask); + } + // Keep expected register values independent of the decoder's mask table. + for (address, expected) in [ + (reg::QUEUE_BASE, 0x3333), + (reg::QUEUE_BASE + 8, 0x0033), + (reg::QUEUE_BASE + 0x10, 0), + (reg::QUEUE_BASE + 0x18, 0), + (reg::QUEUE_BASE + 0x20, 0x3330), + (reg::QUEUE_BASE + 0x28, 0), + (reg::QUEUE_BASE + 0x30, 0), + (reg::QUEUE_BASE + 0x38, 0x00f0), + (reg::QUEUE_BASE + 0x40, 0xfff0), + (reg::QUEUE_BASE + 0x48, 0), + (reg::QUEUE_BASE + 0x50, 0), + (reg::QUEUE_BASE + 0x58, 0), + (reg::QUEUE_BASE + 0x60, 0x3303), + (reg::QUEUE_END, 0x0011), + ] { + d.write64(address, !0); + assert_eq!(d.read64(address).data, expected, "register {address:#x}"); + } + d.power_on(); + assert_eq!(d.read64(reg::CONTROL_BASE).data, 0x0008000080000021); + assert!(!d.error_level()); + assert_eq!(d.read32(reg::CONTROL_BASE).status, BUS_ERR); + } + /// The guest services queued crossbar faults in order, one error at a time. + #[test] + fn servicing_one_crossbar_error_exposes_the_next_pending_error() { + { + let d = BedrockXb::new(); + d.write64(reg::QUEUE_BASE, 1); + assert!(d.error_level()); + assert_eq!(d.read64(reg::FIRST_ERROR).data, 0x1f00); + d.write64(reg::CLEAR_FIRST_ERROR, !0); + assert_eq!(d.read64(reg::FIRST_ERROR).data, 0x1f00); + d.write64(reg::CLEAR_QUEUE_END, !0); + d.write64(reg::CLEAR_QUEUE_BASE, 0); + d.write64(reg::CLEAR_FIRST_ERROR, 0); + assert!(!d.error_level()); + d.power_on(); + assert!(!d.error_level()); + } + { + let d = BedrockXb::new(); + d.write64(reg::QUEUE_END, 0x10); + d.write64(reg::QUEUE_BASE, 1); + assert_eq!(d.read64(reg::FIRST_ERROR).data, 0x1200); + d.write64(reg::CLEAR_QUEUE_END, 0); + d.write64(reg::CLEAR_FIRST_ERROR, 0); + assert_eq!(d.read64(reg::FIRST_ERROR).data, 0x1f00); + d.write64(reg::CLEAR_QUEUE_BASE, 0); + d.write64(reg::QUEUE_BASE, 0x10); + d.write64(reg::CLEAR_FIRST_ERROR, 0); + assert_eq!(d.read64(reg::FIRST_ERROR).data, 0x1f00); + d.write64(reg::CLEAR_QUEUE_BASE, 0); + d.write64(reg::CLEAR_FIRST_ERROR, 0); + assert!(!d.error_level()); + } + } +} diff --git a/src/dev/mod.rs b/src/dev/mod.rs index 013e5b626..8e5552271 100644 --- a/src/dev/mod.rs +++ b/src/dev/mod.rs @@ -60,6 +60,10 @@ pub mod vino; pub mod virtio; pub mod wd33c93a; pub mod z85c30; +pub mod bedrock_io; +pub mod bedrock_ni; +pub mod bedrock_xb; +pub mod bedrock_lb; pub mod bedrock_md; pub mod bedrock_mem; pub mod dp83843; From f7a5d6dff0ed0544f677184fc003e3a92be6e6ce Mon Sep 17 00:00:00 2001 From: James Braid Date: Fri, 30 Oct 2026 08:13:37 +0000 Subject: [PATCH 22/31] bedrock_pi: add processor control and interrupts The Fuel CPU receives interrupts and timing events through Bedrock PI. Model the fitted processor slot and its node callbacks while keeping unconnected graphics control explicit. --- src/dev/bedrock_pi.rs | 757 ++++++++++++++++++++++++++++++++++++ src/dev/bedrock_pi/tests.rs | 276 +++++++++++++ src/dev/mod.rs | 1 + 3 files changed, 1034 insertions(+) create mode 100644 src/dev/bedrock_pi.rs create mode 100644 src/dev/bedrock_pi/tests.rs diff --git a/src/dev/bedrock_pi.rs b/src/dev/bedrock_pi.rs new file mode 100644 index 000000000..7e44a44c4 --- /dev/null +++ b/src/dev/bedrock_pi.rs @@ -0,0 +1,757 @@ +//! Bedrock PI exposes processor-slot control, pending interrupts, real-time +//! counters, timers and shared graphics registers. Fuel uses PI0 slot A for +//! its processor; the other processor slot has no device attached. Node hooks +//! carry cache-alias changes, CPU enable and NMI events to the board. +//! +//! # What is approximated here +//! +//! The model accepts Linux's public `SGIrules` key at offset 0x8 for +//! compatibility. Linux documents the literal for Hub I/O, but the available +//! evidence does not establish that this PI offset maps to the same register. +//! Unconnected graphics registers read as zero and are logged as unsupported. +//! +//! Ported from hw/misc/sgi-bedrock-pi.c in jamesbraid/qemu-sgi-origami. + +use crate::dev::bedrock_io::IIO_PROTECT_OVRRD_KEY; +use crate::dev::heart::Clock; +use crate::hptimer::{TimerId, TimerManager, TimerReturn}; +use crate::irq::InterruptController; +use crate::traits::{BusDevice, BusRead32, BusRead64, Device, Resettable, BUS_ERR, BUS_OK}; +use parking_lot::{Mutex, ReentrantMutex}; +use std::sync::{Arc, OnceLock, Weak}; +use std::time::Duration; +const RT_MASK: u64 = (1 << 55) - 1; +const ERROR_MASK: u64 = (1 << 35) - 1; +const MEMORY_ERROR: u64 = 1 << 24; +const REFLECT: u64 = (1 << 54) | (1 << 55) | (1 << 62) | (1 << 63); +const TICK: u64 = 800; +pub const SIZE: u64 = 0x30108; + +pub mod reg { + pub const NODE_REGION: u64 = 0x0; + pub const PROTECTION_KEY: u64 = 0x8; + pub const OFFSET_0010: u64 = 0x10; + pub const OFFSET_0018: u64 = 0x18; + pub const INTERFACE_ID: u64 = 0x20; + pub const CACHED_ALIAS: u64 = 0x28; + pub const OFFSET_0030: u64 = 0x30; + pub const OFFSET_0038: u64 = 0x38; + pub const INTERFACE_SLOT_A: u64 = 0x40; + pub const INTERFACE_SLOT_B: u64 = 0x48; + pub const CPU_A_ENABLE: u64 = 0x50; + pub const CPU_B_ENABLE: u64 = 0x58; + pub const NMI_SLOT_A: u64 = 0x70; + pub const NMI_SLOT_B: u64 = 0x78; + pub const PENDING_MODIFY: u64 = 0x90; + pub const PENDING_STATUS_A: u64 = 0x98; + pub const PENDING_STATUS_B: u64 = 0xa0; + pub const OFFSET_00A8: u64 = 0xa8; + pub const OFFSET_00B0: u64 = 0xb0; + pub const OFFSET_00B8: u64 = 0xb8; + pub const OFFSET_00C0: u64 = 0xc0; + pub const REQUEST_A: u64 = 0xc8; + pub const REQUEST_B: u64 = 0xd0; + pub const PENDING_A_CLEAR: u64 = 0xd8; + pub const PENDING_B_CLEAR: u64 = 0xe0; + pub const OFFSET_00E8: u64 = 0xe8; + pub const OFFSET_00F0: u64 = 0xf0; + pub const RT_COMPARE_A: u64 = 0x108; + pub const RT_COMPARE_B: u64 = 0x110; + pub const PROFILE_COMPARE: u64 = 0x118; + pub const RT_STATUS_A: u64 = 0x120; + pub const RT_STATUS_B: u64 = 0x128; + pub const PROFILE_STATUS_A: u64 = 0x130; + pub const PROFILE_STATUS_B: u64 = 0x138; + pub const RT_ENABLE_A: u64 = 0x140; + pub const RT_ENABLE_B: u64 = 0x148; + pub const PROFILE_ENABLE_A: u64 = 0x150; + pub const PROFILE_ENABLE_B: u64 = 0x158; + pub const PENDING_MODIFY_B: u64 = 0x180; + pub const ERROR_SUMMARY_ALIAS: u64 = 0x3f8; + pub const ERROR_SUMMARY: u64 = 0x400; + pub const ERROR_CAUSE_A: u64 = 0x408; + pub const ERROR_CAUSE_B: u64 = 0x410; + pub const ERROR_ADDRESS_A: u64 = 0x418; + pub const ERROR_ADDRESS_B: u64 = 0x420; + pub const ERROR_CONTROL: u64 = 0x428; + pub const ERROR_CAPTURE_A: u64 = 0x438; + pub const ERROR_CAPTURE_B: u64 = 0x448; + pub const ERROR_CAPTURE_C: u64 = 0x458; + pub const ERROR_CAPTURE_D: u64 = 0x468; + pub const ERROR_SOURCE_A: u64 = 0x490; + pub const ERROR_SOURCE_B: u64 = 0x4c0; + pub const ERROR_DETAIL_A: u64 = 0x4a8; + pub const ERROR_DETAIL_B: u64 = 0x4b0; + pub const ERROR_DETAIL_C: u64 = 0x4b8; + pub const OFFSET_0430: u64 = 0x430; + pub const OFFSET_0440: u64 = 0x440; + pub const OFFSET_0450: u64 = 0x450; + pub const OFFSET_0460: u64 = 0x460; + pub const OFFSET_04C8: u64 = 0x4c8; + pub const OFFSET_04D0: u64 = 0x4d0; + pub const CREDIT_MODE: u64 = 0x20068; + pub const COUNTER: u64 = 0x30100; + pub const GRAPHICS_BASE: u64 = 0x300; + pub const GRAPHICS_END: u64 = 0x350; +} + +#[derive(Clone, Copy, Debug, Default)] +pub struct BedrockPiConfig { + pub interface: u8, + pub request_region: u8, +} +/// Called outside the state lock. Graphics indices 0..9 are the shared PI +/// graphics registers at offsets 0x300..0x348. Disconnected access is observable. +pub trait NodeHooks: Send + Sync { + fn cached_alias_changed(&self, _size: u8) {} + fn rt_compare_written(&self, _interface: u8, _slot: usize) {} + fn rt_counter_read(&self, _interface: u8) {} + fn cpu_enable(&self, _interface: u8, _slot: usize, _enabled: bool) {} + fn nmi(&self, _interface: u8, _slot: usize) {} + fn gfx_read(&self, _index: usize) -> Option { + None + } + fn gfx_write(&self, _index: usize, _value: u64) -> bool { + false + } + fn gfx_reset(&self) {} +} +struct State { + w: [u64; 160], + pending: [u64; 2], + reflected: u64, + uart: bool, + base: u64, + epoch: u64, + paused: Option, + running: bool, + rt_due: [u128; 2], + profile_due: u128, + levels: [u8; 256], + rearm: [u64; 128], + credit_mode: u64, +} +impl State { + fn new(now: u64) -> Self { + let mut s = Self { + w: [0; 160], + pending: [0; 2], + reflected: 0, + uart: false, + base: 0, + epoch: now, + paused: Some(now), + running: false, + rt_due: [0; 2], + profile_due: 0, + levels: [0; 256], + rearm: [0; 128], + credit_mode: 0, + }; + s.w[0] = !0; + s.w[2] = !0; + s.w[3] = !0; + s.arm_all(now); + s + } + fn time(&self, now: u64) -> u64 { + self.paused.unwrap_or(now) + } + fn count(&self, now: u64) -> u64 { + self.base + .wrapping_add(self.time(now).saturating_sub(self.epoch) / TICK) + & RT_MASK + } + fn arm_rt(&mut self, slot: usize, now: u64) { + let now = self.time(now); + let ticks = (self.w[0x108 / 8 + slot] + .wrapping_sub(self.count(now)) + .wrapping_sub(1) + & RT_MASK) + + 1; + self.rt_due[slot] = u128::from(now - now.saturating_sub(self.epoch) % TICK) + + u128::from(ticks) * u128::from(TICK); + } + fn arm_profile(&mut self, now: u64) { + let now = self.time(now); + let ticks = (self.w[0x118 / 8] as u32).wrapping_sub(self.count(now) as u32); + let ticks = if ticks == 0 { + 1u64 << 32 + } else { + u64::from(ticks) + }; + self.profile_due = u128::from(now - now.saturating_sub(self.epoch) % TICK) + + u128::from(ticks) * u128::from(TICK); + } + fn arm_all(&mut self, now: u64) { + self.arm_rt(0, now); + self.arm_rt(1, now); + self.arm_profile(now); + } + fn sync(&mut self, now: u64) { + let now = self.time(now); + for slot in 0..2 { + if u128::from(now) >= self.rt_due[slot] { + self.w[0x120 / 8 + slot] = 1; + self.arm_rt(slot, now); + } + } + if u128::from(now) >= self.profile_due { + self.w[0x130 / 8] = 1; + self.w[0x138 / 8] = 1; + self.arm_profile(now); + } + for level in 7..128 { + if self.rearm[level] != 0 && self.rearm[level] <= now { + self.rearm[level] = 0; + if self.levels.iter().any(|&v| usize::from(v) == level + 1) { + self.pending[level / 64] |= 1 << (level % 64); + } + } + } + } + fn pending(&self, bank: usize) -> u64 { + if bank == 1 { + (self.pending[1] & !REFLECT) | self.reflected + } else { + self.pending[0] + | if self.uart { 1 << 4 } else { 0 } + | if self.w[0xc8 / 8] & self.w[0xe8 / 8] != 0 { + 1 << 5 + } else { + 0 + } + | if self.w[0xd0 / 8] & self.w[0xe8 / 8] != 0 { + 1 << 6 + } else { + 0 + } + } + } + fn outputs(&self) -> [bool; 10] { + let mut out = [false; 10]; + for slot in 0..2 { + out[slot * 5] = self.pending(0) & self.w[0xa8 / 8 + slot * 2] != 0; + out[slot * 5 + 1] = self.pending(1) & self.w[0xb0 / 8 + slot * 2] != 0; + out[slot * 5 + 2] = self.w[0x120 / 8 + slot] & self.w[0x140 / 8 + slot] != 0; + out[slot * 5 + 3] = self.w[0x130 / 8 + slot] & self.w[0x150 / 8 + slot] != 0; + out[slot * 5 + 4] = self.w[0x400 / 8] & self.w[0x408 / 8 + slot] != 0; + } + out + } + fn modify(&mut self, value: u64, reissue: bool, now: u64) { + let bank = ((value >> 6) & 1) as usize; + let bit = value & 63; + let mask = 1 << bit; + if bank == 1 && mask & REFLECT != 0 { + return; + } + if value & 0x100 != 0 { + self.pending[bank] |= mask; + } else { + self.pending[bank] &= !mask; + let level = bank * 64 + bit as usize; + if reissue && self.levels.iter().any(|&v| usize::from(v) == level + 1) { + self.rearm[level] = now.saturating_add(1_000_000); + } + } + } +} +pub struct BedrockPi { + cfg: BedrockPiConfig, + clock: Clock, + state: Mutex, + hooks: Mutex>>, + irq: Mutex>>, + timers: OnceLock>, + timer: Mutex>, + timer_pass: Mutex<()>, + dispatch: ReentrantMutex<()>, + driven: Mutex<[Option; 10]>, + weak: Weak, +} +impl BedrockPi { + pub fn new(cfg: BedrockPiConfig, clock: Clock) -> Arc { + assert!(cfg.interface < 2 && cfg.request_region < 64); + Arc::new_cyclic(|weak| Self { + cfg, + state: Mutex::new(State::new(clock())), + clock, + hooks: Mutex::new(None), + irq: Mutex::new(None), + timers: OnceLock::new(), + timer: Mutex::new(None), + timer_pass: Mutex::new(()), + dispatch: ReentrantMutex::new(()), + driven: Mutex::new([None; 10]), + weak: weak.clone(), + }) + } + pub fn set_node_hooks(&self, hooks: Arc) { + let _dispatch = self.dispatch.lock(); + *self.hooks.lock() = Some(hooks); + } + pub fn set_timer_manager(&self, tm: Arc) { + let _ = self.timers.set(tm); + self.schedule(); + } + /// Lines 0..4 are slot A IP2..IP6; lines 5..9 are slot B IP2..IP6. + pub fn set_interrupt_controller(&self, irq: Arc) { + let _dispatch = self.dispatch.lock(); + *self.irq.lock() = Some(irq); + *self.driven.lock() = [None; 10]; + self.drive(); + } + fn hooks(&self) -> Option> { + self.hooks.lock().clone() + } + /// Report an access to a register the model does not implement. + fn note(&self, off: u64, write: bool) { + crate::dlog!(crate::devlog::LogModule::Bedrock, + "Bedrock PI{} unsupported {} at {off:#x}", self.cfg.interface, + if write { "write" } else { "read" }); + } + fn drive(&self) { + // Serialize publication, including reentrant reads by a controller. + // Publish one line at a time: a callback may change another line. + let _dispatch = self.dispatch.lock(); + let irq = self.irq.lock().clone(); + if let Some(irq) = irq { + for line in 0..10 { + let level = self.state.lock().outputs()[line]; + let changed = { + let mut driven = self.driven.lock(); + let changed = driven[line] != Some(level); + driven[line] = Some(level); + changed + }; + if changed { + irq.set_line(line as u32, level); + } + } + } + } + fn schedule(&self) { + let _pass = self.timer_pass.lock(); + let Some(tm) = self.timers.get() else { return }; + let mut timer = self.timer.lock(); + if let Some(id) = timer.take() { + tm.remove(id); + } + let s = self.state.lock(); + if !s.running { + return; + } + let next = s + .rt_due + .iter() + .copied() + .chain([s.profile_due]) + .chain(s.rearm.iter().filter(|&&v| v != 0).map(|&v| u128::from(v))) + .min() + .unwrap(); + let delay = next + .saturating_sub(u128::from((self.clock)())) + .min(1_000_000_000) + .max(1) as u64; + drop(s); + *timer = Some( + tm.add_one_shot(Duration::from_nanos(delay), self.weak.clone(), |weak| { + if let Some(pi) = weak.upgrade() { + { + let _pass = pi.timer_pass.lock(); + let mut s = pi.state.lock(); + if !s.running { + return TimerReturn::Delete; + } + s.sync((pi.clock)()); + } + pi.drive(); + pi.schedule(); + } + TimerReturn::Delete + }), + ); + } + pub fn poll(&self) { + self.state.lock().sync((self.clock)()); + self.drive(); + } + pub fn set_pending(&self, bank: usize, bit: u8) { + if bank < 2 && bit < 64 { + self.state.lock().pending[bank] |= 1 << bit; + self.drive(); + } + } + fn reflected(&self, bit: u8, level: bool) { + let mut s = self.state.lock(); + if level { + s.reflected |= 1 << bit; + } else { + s.reflected &= !(1 << bit); + } + drop(s); + self.drive(); + } + pub fn set_lb_error(&self, level: bool) { + self.reflected(55, level); + } + pub fn set_ni_error(&self, level: bool) { + self.reflected(62, level); + } + pub fn set_xb_error(&self, level: bool) { + self.reflected(54, level); + } + pub fn set_uart_irq(&self, level: bool) { + self.state.lock().uart = level; + self.drive(); + } + pub fn set_memory_uncorrectable(&self) { + self.state.lock().w[0x400 / 8] |= MEMORY_ERROR; + self.drive(); + } + pub fn set_level_source(&self, source: usize, level: u8, active: bool) { + if source < 256 && (!active || (7..=127).contains(&level)) { + self.state.lock().levels[source] = if active { level + 1 } else { 0 }; + self.drive(); + } + } + pub fn io_modify(&self, value: u64, region: u8) -> bool { + let mut s = self.state.lock(); + if region >= 64 || s.w[2] & (1 << region) == 0 { + drop(s); + self.note(0x90, true); + return false; + } + let now = s.time((self.clock)()); + s.modify(value, false, now); + drop(s); + self.drive(); + true + } + fn read(&self, off: u64) -> u64 { + if (reg::GRAPHICS_BASE..reg::GRAPHICS_END).contains(&off) { + if let Some(v) = self + .hooks() + .and_then(|h| h.gfx_read(((off - 0x300) / 8) as usize)) + { + return v; + } + self.note(off, false); + return 0; + } + let mut s = self.state.lock(); + s.sync((self.clock)()); + let value = match off { + reg::INTERFACE_ID => u64::from(self.cfg.interface) << 1, + reg::INTERFACE_SLOT_A => u64::from(self.cfg.interface == 0), + reg::INTERFACE_SLOT_B => 0, + reg::PENDING_STATUS_A => s.pending(0), + reg::PENDING_STATUS_B => s.pending(1), + reg::PENDING_A_CLEAR => s.w[reg::REQUEST_A as usize / 8], + reg::PENDING_B_CLEAR => s.w[reg::REQUEST_B as usize / 8], + reg::ERROR_SUMMARY_ALIAS => s.w[reg::ERROR_SUMMARY as usize / 8], + reg::ERROR_CAPTURE_A | reg::ERROR_CAPTURE_B | reg::ERROR_CAPTURE_C | reg::ERROR_CAPTURE_D => s.w[(off - reg::PROTECTION_KEY) as usize / 8], + reg::CREDIT_MODE => s.credit_mode, + reg::COUNTER => s.count((self.clock)()), + _ if readable(off) => s.w[off as usize / 8], + _ => { + self.note(off, false); + 0 + } + }; + drop(s); + self.drive(); + if off == reg::COUNTER { + if let Some(h) = self.hooks() { + h.rt_counter_read(self.cfg.interface); + } + } + value + } + fn write(&self, off: u64, value: u64) { + // Keep register changes and their node notifications in one order. + // The state lock still drops before callbacks; dispatch is reentrant + // so a node callback can read registers and recompute IRQ outputs. + let _dispatch = self.dispatch.lock(); + if (reg::GRAPHICS_BASE..reg::GRAPHICS_END).contains(&off) { + if !self + .hooks() + .is_some_and(|h| h.gfx_write(((off - 0x300) / 8) as usize, value)) + { + self.note(off, true); + } + return; + } + let now = (self.clock)(); + let mut s = self.state.lock(); + s.sync(now); + let mut alias = None; + let mut compare = None; + let mut enable = None; + let mut nmi = None; + match off { + reg::NODE_REGION | reg::OFFSET_0010 | reg::OFFSET_0018 | reg::OFFSET_00A8 | reg::OFFSET_00B0 | reg::OFFSET_00B8 | reg::OFFSET_00C0 | reg::OFFSET_00E8 | reg::OFFSET_04C8 | reg::OFFSET_04D0 => { + s.w[off as usize / 8] = value + } + reg::PROTECTION_KEY => { + if value == IIO_PROTECT_OVRRD_KEY { + s.w[0] |= 1 << self.cfg.request_region; + } + } + reg::CACHED_ALIAS => { + if s.w[5] != value & 15 { + s.w[5] = value & 15; + alias = Some((value & 15) as u8); + } + } + reg::OFFSET_0030 => s.w[6] = value & 255, + reg::OFFSET_0038 => s.w[7] = value & 0xffffff, + reg::CPU_A_ENABLE | reg::CPU_B_ENABLE => { + let slot = ((off - 0x50) / 8) as usize; + s.w[off as usize / 8] = value & 1; + enable = Some((slot, value & 1 != 0)); + } + reg::NMI_SLOT_A | reg::NMI_SLOT_B => { + if value & 1 != 0 { + nmi = Some(((off - 0x70) / 8) as usize); + } else { + self.note(off, true); + } + } + reg::PENDING_MODIFY | reg::PENDING_MODIFY_B => { + let now = s.time(now); + s.modify(value, true, now); + } + reg::REQUEST_A | reg::REQUEST_B => s.w[off as usize / 8] |= 1 << self.cfg.request_region, + reg::PENDING_A_CLEAR | reg::PENDING_B_CLEAR => s.w[(off - reg::OFFSET_0010) as usize / 8] &= !value, + reg::OFFSET_00F0 => { + s.w[0xf0 / 8] = value & 31; + if value & 31 != 0 { + self.note(off, true); + } + } + reg::RT_COMPARE_A | reg::RT_COMPARE_B => { + let slot = ((off - 0x108) / 8) as usize; + s.w[off as usize / 8] = value & RT_MASK; + s.arm_rt(slot, now); + compare = Some(slot); + } + reg::PROFILE_COMPARE => { + s.w[0x118 / 8] = value & 0xffffffff; + s.arm_profile(now); + } + reg::RT_STATUS_A | reg::RT_STATUS_B | reg::PROFILE_STATUS_A | reg::PROFILE_STATUS_B => s.w[off as usize / 8] = 0, + reg::RT_ENABLE_A | reg::RT_ENABLE_B | reg::PROFILE_ENABLE_A | reg::PROFILE_ENABLE_B => s.w[off as usize / 8] = value & 1, + reg::ERROR_SUMMARY_ALIAS => { + s.w[0x400 / 8] = + (s.w[0x400 / 8] & MEMORY_ERROR) | (value & ERROR_MASK & !MEMORY_ERROR) + } + reg::ERROR_SUMMARY => s.w[reg::ERROR_SUMMARY as usize / 8] &= !(value & ERROR_MASK), + reg::ERROR_CAUSE_A | reg::ERROR_CAUSE_B => s.w[off as usize / 8] = value & ERROR_MASK, + reg::ERROR_ADDRESS_A | reg::ERROR_ADDRESS_B => s.w[off as usize / 8] = value & 0x1fffffff8, + reg::ERROR_CONTROL => { + s.w[0x428 / 8] = value & 15; + if value & 15 != 0 { + self.note(off, true); + } + } + reg::ERROR_CAPTURE_A | reg::ERROR_CAPTURE_B | reg::ERROR_CAPTURE_C | reg::ERROR_CAPTURE_D => { + let slot = usize::from(off >= 0x450); + s.w[0x430 / 8 + slot * 4] = 0; + s.w[0x440 / 8 + slot * 4] = 0; + } + reg::ERROR_SOURCE_A | reg::ERROR_SOURCE_B => s.w[off as usize / 8] = value & 63, + reg::ERROR_DETAIL_A | reg::ERROR_DETAIL_B => s.w[off as usize / 8] = value & 0x1fffff, + reg::ERROR_DETAIL_C => s.w[off as usize / 8] = value & 0xfffff, + reg::CREDIT_MODE => s.credit_mode = value & 1, + reg::COUNTER => { + s.base = value & RT_MASK; + s.epoch = s.time(now); + s.arm_all(now); + } + _ => self.note(off, true), + } + drop(s); + if let Some(h) = self.hooks() { + if let Some(size) = alias { + h.cached_alias_changed(size); + } + if let Some(slot) = compare { + h.rt_compare_written(self.cfg.interface, slot); + } + if let Some((slot, value)) = enable { + h.cpu_enable(self.cfg.interface, slot, value); + } + if let Some(slot) = nmi { + h.nmi(self.cfg.interface, slot); + } + } + self.drive(); + self.schedule(); + } +} +fn readable(off: u64) -> bool { + matches!( + off, + reg::NODE_REGION | reg::OFFSET_0010 + | reg::OFFSET_0018 + | reg::CACHED_ALIAS + | reg::OFFSET_0030 + | reg::OFFSET_0038 + | reg::CPU_A_ENABLE + | reg::CPU_B_ENABLE + | reg::OFFSET_00A8 + | reg::OFFSET_00B0 + | reg::OFFSET_00B8 + | reg::OFFSET_00C0 + | reg::REQUEST_A + | reg::REQUEST_B + | reg::OFFSET_00E8 + | reg::OFFSET_00F0 + | reg::RT_COMPARE_A + | reg::RT_COMPARE_B + | reg::PROFILE_COMPARE + | reg::RT_STATUS_A + | reg::RT_STATUS_B + | reg::PROFILE_STATUS_A + | reg::PROFILE_STATUS_B + | reg::RT_ENABLE_A + | reg::RT_ENABLE_B + | reg::PROFILE_ENABLE_A + | reg::PROFILE_ENABLE_B + | reg::ERROR_SUMMARY + | reg::ERROR_CAUSE_A + | reg::ERROR_CAUSE_B + | reg::ERROR_ADDRESS_A + | reg::ERROR_ADDRESS_B + | reg::ERROR_CONTROL + | reg::OFFSET_0430 + | reg::OFFSET_0440 + | reg::OFFSET_0450 + | reg::OFFSET_0460 + | reg::ERROR_SOURCE_A + | reg::ERROR_DETAIL_A + | reg::ERROR_DETAIL_B + | reg::ERROR_DETAIL_C + | reg::ERROR_SOURCE_B + | reg::OFFSET_04C8 + | reg::OFFSET_04D0 + ) +} +impl BusDevice for BedrockPi { + fn read64(&self, addr: u64) -> BusRead64 { + if addr < SIZE && addr & 7 == 0 { + BusRead64::ok(self.read(addr)) + } else { + self.note(addr, false); + BusRead64::err() + } + } + fn read32(&self, addr: u64) -> BusRead32 { + if addr == reg::COUNTER || addr == reg::COUNTER + 4 { + let value = self.read(reg::COUNTER); + BusRead32::ok(if addr & 4 == 0 { + (value >> 32) as u32 + } else { + value as u32 + }) + } else { + self.note(addr, false); + BusRead32::err() + } + } + fn write64(&self, addr: u64, value: u64) -> u32 { + if addr < SIZE && addr & 7 == 0 { + self.write(addr, value); + BUS_OK + } else { + self.note(addr, true); + BUS_ERR + } + } +} +impl Device for BedrockPi { + fn start(&self) { + let now = (self.clock)(); + let mut s = self.state.lock(); + if s.running { + return; + } + if let Some(paused) = s.paused.take() { + let delta = now.saturating_sub(paused); + s.epoch = s.epoch.saturating_add(delta); + for due in &mut s.rt_due { + *due += u128::from(delta); + } + s.profile_due += u128::from(delta); + for due in &mut s.rearm { + if *due != 0 { + *due = due.saturating_add(delta); + } + } + } + s.running = true; + drop(s); + self.schedule(); + } + fn stop(&self) { + { + let _pass = self.timer_pass.lock(); + let mut s = self.state.lock(); + s.sync((self.clock)()); + s.paused = Some(s.time((self.clock)())); + s.running = false; + if let Some(id) = self.timer.lock().take() { + if let Some(tm) = self.timers.get() { + tm.remove(id); + } + } + } + self.drive(); + } + fn is_running(&self) -> bool { + self.state.lock().running + } + fn step(&self, _cycles: u64) { + self.poll(); + } + fn get_clock(&self) -> u64 { + self.state.lock().count((self.clock)()) + } +} +impl Resettable for BedrockPi { + fn power_on(&self) { + let _dispatch = self.dispatch.lock(); + let now = (self.clock)(); + let mut s = self.state.lock(); + // These controls and captures survive the PI block's local reset. + let old = s.w; + let running = s.running; + let reflected = s.reflected; + let uart = s.uart; + *s = State::new(now); + for off in [ + 0, 0x10, 0x18, 0x28, 0x50, 0x58, 0x400, 0x418, 0x420, 0x428, 0x430, 0x440, 0x450, + 0x460, 0x4c8, 0x4d0, + ] { + s.w[off / 8] = old[off / 8]; + } + s.running = running; + s.paused = if running { None } else { Some(now) }; + s.reflected = reflected; + s.uart = uart; + drop(s); + if let Some(h) = self.hooks() { + h.gfx_reset(); + for slot in 0..2 { + let enabled = self.state.lock().w[0x50 / 8 + slot] != 0; + h.cpu_enable(self.cfg.interface, slot, enabled); + } + } + self.drive(); + self.schedule(); + } +} +#[cfg(test)] +mod tests; diff --git a/src/dev/bedrock_pi/tests.rs b/src/dev/bedrock_pi/tests.rs new file mode 100644 index 000000000..5f684ad11 --- /dev/null +++ b/src/dev/bedrock_pi/tests.rs @@ -0,0 +1,276 @@ +// Cases follow tests/qtest/sn1-bedrock-pi-test.c in jamesbraid/qemu-sgi-origami: +// power-on-values, local-reset, pending-reflection, real-time-comparators +// and profile-counter-wrap. + +use super::*; +use std::sync::atomic::{AtomicU64, Ordering}; +fn setup() -> (Arc, Arc) { + let now = Arc::new(AtomicU64::new(0)); + let clock = now.clone(); + let pi = BedrockPi::new( + BedrockPiConfig::default(), + Arc::new(move || clock.load(Ordering::SeqCst)), + ); + pi.start(); + (pi, now) +} +fn r(pi: &BedrockPi, off: u64) -> u64 { + pi.read64(off).data +} +/// The PROM reads PI slot presence and relies on the documented reset survivors. +#[test] +fn the_pi_reports_interface_state_and_retains_registers_across_reset() { + { + let (pi, _) = setup(); + assert_eq!(r(&pi, reg::INTERFACE_ID), 0); + assert_eq!(r(&pi, reg::INTERFACE_SLOT_A), 1); + assert_eq!(r(&pi, reg::INTERFACE_SLOT_B), 0); + pi.write64(reg::NODE_REGION, 0); + pi.write64(reg::PROTECTION_KEY, 0x53474972756c6573); + assert_eq!(r(&pi, reg::NODE_REGION), 1); + let absent = BedrockPi::new( + BedrockPiConfig { + interface: 1, + request_region: 0, + }, + Arc::new(|| 0), + ); + assert_eq!(r(&absent, reg::INTERFACE_ID), 2); + assert_eq!(r(&absent, reg::INTERFACE_SLOT_A), 0); + assert_eq!(pi.read32(reg::INTERFACE_ID).status, BUS_ERR); + assert_eq!(pi.write32(reg::COUNTER, 0), BUS_ERR); + } + { + let (pi, _) = setup(); + pi.write64(reg::PENDING_MODIFY, 0x10c); + assert_eq!(r(&pi, reg::PENDING_STATUS_A), 1 << 12); + pi.write64(reg::PENDING_MODIFY_B, 12); + assert_eq!(r(&pi, reg::PENDING_STATUS_A), 0); + pi.set_lb_error(true); + pi.write64(reg::PENDING_MODIFY, 64 + 55); + assert_eq!(r(&pi, reg::PENDING_STATUS_B), 1 << 55); + pi.write64(reg::PENDING_MODIFY, 0x100 | 64 | 63); + assert_eq!(r(&pi, reg::PENDING_STATUS_B), 1 << 55); + pi.write64(reg::REQUEST_A, 0); + pi.write64(reg::OFFSET_00E8, 1); + assert_eq!(r(&pi, reg::PENDING_STATUS_A), 1 << 5); + pi.write64(reg::PENDING_A_CLEAR, 1); + assert_eq!(r(&pi, reg::PENDING_STATUS_A), 0); + pi.write64(reg::OFFSET_0010, 0); + assert!(!pi.io_modify(0x108, 0)); + pi.write64(reg::OFFSET_0010, 1); + assert!(pi.io_modify(0x108, 0)); + assert_eq!(r(&pi, reg::PENDING_STATUS_A), 1 << 8); + } + { + let (pi, _) = setup(); + pi.write64(reg::CACHED_ALIAS, 9); + pi.write64(reg::CPU_A_ENABLE, 1); + pi.write64(reg::ERROR_ADDRESS_A, !0); + pi.set_memory_uncorrectable(); + pi.write64(reg::ERROR_SUMMARY_ALIAS, 3); + pi.write64(reg::RT_COMPARE_A, 0x24); + pi.power_on(); + assert_eq!(r(&pi, reg::ERROR_SUMMARY), MEMORY_ERROR | 3); + assert_eq!(r(&pi, reg::CACHED_ALIAS), 9); + assert_eq!(r(&pi, reg::CPU_A_ENABLE), 1); + assert_eq!(r(&pi, reg::ERROR_ADDRESS_A), 0x1fffffff8); + assert_eq!(r(&pi, reg::RT_COMPARE_A), 0); + assert_eq!(r(&pi, reg::COUNTER), 0); + pi.write64(reg::ERROR_SUMMARY_ALIAS, 0); + assert_eq!(r(&pi, reg::ERROR_SUMMARY), MEMORY_ERROR); + pi.write64(reg::ERROR_SUMMARY, MEMORY_ERROR); + assert_eq!(r(&pi, reg::ERROR_SUMMARY), 0); + } +} +/// The guest timer drives interrupts even while the processor is stopped. +/// The processor timer can wrap, pause while stopped, and reissue its guest interrupt. +#[test] +fn the_pi_timer_wraps_pauses_and_reissues_interrupts() { + { + let (pi, now) = setup(); + pi.write64(reg::COUNTER, 0x1_0000_0000); + pi.write64(reg::RT_COMPARE_A, 0x1_0000_0002); + now.store(1599, Ordering::SeqCst); + assert_eq!(r(&pi, reg::RT_STATUS_A), 0); + now.store(1600, Ordering::SeqCst); + assert_eq!(r(&pi, reg::RT_STATUS_A), 1); + assert_eq!(pi.read32(reg::COUNTER).data, 1); + assert_eq!(pi.read32(reg::COUNTER + 4).data, 2); + pi.write64(reg::RT_STATUS_A, !0); + assert_eq!(r(&pi, reg::RT_STATUS_A), 0); + pi.write64(reg::COUNTER, RT_MASK); + pi.write64(reg::PROFILE_COMPARE, 0); + now.store(2400, Ordering::SeqCst); + assert_eq!(r(&pi, reg::COUNTER), 0); + assert_eq!(r(&pi, reg::PROFILE_STATUS_A), 1); + assert_eq!(r(&pi, reg::PROFILE_STATUS_B), 1); + pi.write64(reg::PROFILE_STATUS_A, 0); + assert_eq!(r(&pi, reg::PROFILE_STATUS_A), 0); + assert_eq!(r(&pi, reg::PROFILE_STATUS_B), 1); + } + { + let (pi, now) = setup(); + pi.set_level_source(0, 12, true); + pi.write64(reg::PENDING_MODIFY, 0x10c); + pi.write64(reg::PENDING_MODIFY, 12); + now.store(999999, Ordering::SeqCst); + assert_eq!(r(&pi, reg::PENDING_STATUS_A), 0); + now.store(1000000, Ordering::SeqCst); + assert_eq!(r(&pi, reg::PENDING_STATUS_A), 1 << 12); + pi.write64(reg::COUNTER, 0); + pi.write64(reg::RT_COMPARE_A, 2); + now.store(1000400, Ordering::SeqCst); + pi.stop(); + now.store(2000000, Ordering::SeqCst); + assert_eq!(r(&pi, reg::COUNTER), 0); + pi.start(); + now.store(2001200, Ordering::SeqCst); + assert_eq!(r(&pi, reg::RT_STATUS_A), 1); + } +} +/// An interrupt callback may inspect PI while an output changes. +/// An interrupt callback may inspect PI while an output changes. +#[test] +fn pi_interrupt_outputs_follow_masks_and_allow_reentrant_reads() { + struct Probe { + pi: Weak, + bits: AtomicU64, + } + impl InterruptController for Probe { + fn set_line(&self, line: u32, level: bool) { + if let Some(pi) = self.pi.upgrade() { + let _ = pi.read64(reg::PENDING_STATUS_A); + } + if level { + self.bits.fetch_or(1 << line, Ordering::SeqCst); + } else { + self.bits.fetch_and(!(1 << line), Ordering::SeqCst); + } + } + } + let (pi, now) = setup(); + let probe = Arc::new(Probe { + pi: Arc::downgrade(&pi), + bits: AtomicU64::new(0), + }); + pi.set_interrupt_controller(probe.clone()); + pi.write64(reg::OFFSET_00A8, 1 << 12); + pi.write64(reg::OFFSET_00B0, 1 << 55); + pi.write64(reg::PENDING_MODIFY, 0x10c); + pi.set_lb_error(true); + pi.write64(reg::RT_COMPARE_A, 2); + pi.write64(reg::PROFILE_COMPARE, 2); + pi.write64(reg::RT_ENABLE_A, 1); + pi.write64(reg::PROFILE_ENABLE_A, 1); + pi.write64(reg::ERROR_CAUSE_A, MEMORY_ERROR); + pi.set_memory_uncorrectable(); + now.store(1600, Ordering::SeqCst); + pi.poll(); + assert_eq!(probe.bits.load(Ordering::SeqCst), 31); + pi.write64(reg::RT_STATUS_A, 0); + assert_eq!(probe.bits.load(Ordering::SeqCst), 27); + pi.write64(reg::PENDING_MODIFY, 12); + pi.set_lb_error(false); + pi.write64(reg::PROFILE_STATUS_A, 0); + pi.write64(reg::ERROR_SUMMARY, MEMORY_ERROR); + assert_eq!(probe.bits.load(Ordering::SeqCst), 0); +} +/// A guest can wait for a timer interrupt without polling its compare register. +/// The guest may wait for a timer interrupt without polling the compare register. +#[test] +fn the_timer_manager_delivers_expirations_without_mmio_polling() { + struct Probe(std::sync::mpsc::Sender<()>); + impl InterruptController for Probe { + fn set_line(&self, line: u32, level: bool) { + if line == 2 && level { + self.0.send(()).unwrap(); + } + } + } + let pi = BedrockPi::new(BedrockPiConfig::default(), crate::dev::heart::host_clock()); + let tm = Arc::new(TimerManager::new()); + pi.set_timer_manager(tm.clone()); + let (tx, rx) = std::sync::mpsc::channel(); + pi.set_interrupt_controller(Arc::new(Probe(tx))); + pi.write64(reg::RT_COMPARE_A, 12500); + pi.write64(reg::RT_ENABLE_A, 1); + pi.start(); + rx.recv_timeout(Duration::from_secs(2)).unwrap(); + pi.stop(); + assert!(!pi.is_running()); +} +/// Node callbacks may reenter PI, so concurrent writes must keep their observed order. +/// Node callbacks may reenter PI, so concurrent writes must preserve observed order. +#[test] +fn concurrent_pi_updates_publish_callbacks_in_register_order() { + use std::sync::mpsc; + use std::thread; + + struct Hooks { + pi: Weak, + offset: u64, + first_value: u64, + entered: mpsc::Sender<()>, + release: Mutex>, + observed: AtomicU64, + } + impl Hooks { + fn publish(&self, value: u64) { + // A node may inspect PI registers while processing a change. + let _ = self.pi.upgrade().unwrap().read64(self.offset); + if value == self.first_value { + self.entered.send(()).unwrap(); + self.release.lock().recv().unwrap(); + } + self.observed.store(value, Ordering::SeqCst); + } + } + impl NodeHooks for Hooks { + fn cached_alias_changed(&self, size: u8) { + self.publish(u64::from(size)); + } + fn cpu_enable(&self, _: u8, _: usize, enabled: bool) { + self.publish(u64::from(enabled)); + } + } + + for (offset, first_value, second_value) in [(reg::CACHED_ALIAS, 1, 2), (reg::CPU_A_ENABLE, 1, 0), (reg::CPU_B_ENABLE, 1, 0)] { + let (pi, _) = setup(); + let (entered_tx, entered_rx) = mpsc::channel(); + let (release_tx, release_rx) = mpsc::channel(); + let hooks = Arc::new(Hooks { + pi: Arc::downgrade(&pi), + offset, + first_value, + entered: entered_tx, + release: Mutex::new(release_rx), + observed: AtomicU64::new(99), + }); + pi.set_node_hooks(hooks.clone()); + let first_pi = pi.clone(); + let first = thread::spawn(move || first_pi.write64(offset, first_value)); + entered_rx.recv_timeout(Duration::from_secs(2)).unwrap(); + let (started_tx, started_rx) = mpsc::channel(); + let (done_tx, done_rx) = mpsc::channel(); + let second_pi = pi.clone(); + let second = thread::spawn(move || { + started_tx.send(()).unwrap(); + second_pi.write64(offset, second_value); + done_tx.send(()).unwrap(); + }); + started_rx.recv_timeout(Duration::from_secs(2)).unwrap(); + let premature = done_rx.recv_timeout(Duration::from_millis(100)); + // Always release and join both writers before checking results. + release_tx.send(()).unwrap(); + assert_eq!(first.join().unwrap(), BUS_OK); + second.join().unwrap(); + assert_eq!( + hooks.observed.load(Ordering::SeqCst), + r(&pi, offset), + "offset {offset:#x}" + ); + assert!(matches!(premature, Err(mpsc::RecvTimeoutError::Timeout))); + assert_eq!(r(&pi, offset), second_value); + } +} diff --git a/src/dev/mod.rs b/src/dev/mod.rs index 8e5552271..b4ca3d18e 100644 --- a/src/dev/mod.rs +++ b/src/dev/mod.rs @@ -66,6 +66,7 @@ pub mod bedrock_xb; pub mod bedrock_lb; pub mod bedrock_md; pub mod bedrock_mem; +pub mod bedrock_pi; pub mod dp83843; pub mod ics1890; pub mod isp12160; From d9a5e7838d51acd8150126ef360b609f89041a8a Mon Sep 17 00:00:00 2001 From: James Braid Date: Fri, 30 Oct 2026 08:13:37 +0000 Subject: [PATCH 23/31] bedrock_flash: add the Fuel boot flash windows Fuel reaches its AMD flash through executable PROM aliases and programming windows. Share the array bytes and persistent store while preserving the command and erase state of each flash part. --- src/dev/bedrock_flash.rs | 633 +++++++++++++++++++++++++++++++++++++++ src/dev/mod.rs | 1 + 2 files changed, 634 insertions(+) create mode 100644 src/dev/bedrock_flash.rs diff --git a/src/dev/bedrock_flash.rs b/src/dev/bedrock_flash.rs new file mode 100644 index 000000000..05138bfff --- /dev/null +++ b/src/dev/bedrock_flash.rs @@ -0,0 +1,633 @@ +//! Bedrock's boot flash: two independent 8 MiB arrays, selected 32 MiB +//! apart, with the AMD command set and CFI query data. +//! +//! Boot aliases and executable views share the same array bytes. Each array +//! has its own command state; instruction fetches read stored bytes even +//! while register reads return command status. +//! +//! Ported from hw/misc/sgi-bedrock-flash.c and hw/mips/sgi/initial-flash.c +//! in jamesbraid/qemu-sgi-origami. + + + +use super::sflash::amd::{AmdCore, Effect, SECTOR_SIZE}; +use super::sflash::storage::FlashBacking; +use crate::traits::{ + BusDevice, BusRead16, BusRead32, BusRead64, BusRead8, Resettable, BUS_ERR, BUS_OK, +}; +use parking_lot::Mutex; +use std::io; +use std::path::Path; +use std::sync::{ + atomic::{AtomicU64, Ordering}, + Arc, +}; + +pub const PART_SIZE: usize = 8 << 20; +pub const FLASH_SIZE: usize = 2 * PART_SIZE; +pub const SECOND_POSITION: u64 = 32 << 20; + +fn sum(bytes: &[u8]) -> u8 { + bytes.iter().fold(0u8, |s, b| s.wrapping_add(*b)) +} + +/// Apply Fuel's firmware record fields while retaining the configuration sum. +/// This policy also applies when reopening an existing persistent store. +fn configure(data: &mut [u8]) { + let config = &mut data[0x60..0xb0]; + let old = sum(config); + config[0x34..0x38].fill(0); + config[4..8].copy_from_slice(&0x549aa985u32.to_be_bytes()); + config[0x14..0x18].copy_from_slice(&500_000_000u32.to_be_bytes()); + config[0x30..0x34].copy_from_slice(&2u32.to_be_bytes()); + config[0x38..0x3c].copy_from_slice(&1u32.to_be_bytes()); + let adjustment = old.wrapping_sub(sum(config)); + config[0x34..0x38].copy_from_slice(&(adjustment as u32).to_be_bytes()); +} + +struct State { + data: Vec, + parts: [AmdCore; 2], + backing: Option, +} + +impl State { + fn new(data: Vec) -> Self { + Self { + data, + parts: std::array::from_fn(|_| AmdCore::default()), + backing: None, + } + } + // Keep the in-memory array unchanged until each backing write succeeds. + // If a later range fails, restore the old bytes for all ranges already tried. + fn persist(&mut self, ranges: &[(usize, &[u8])]) -> io::Result<()> { + let Some(backing) = &mut self.backing else { + return Ok(()); + }; + for (index, &(offset, bytes)) in ranges.iter().enumerate() { + if let Err(error) = backing.write_range(offset, bytes) { + for &(restore_offset, restore_bytes) in &ranges[..=index] { + let old = &self.data[restore_offset..restore_offset + restore_bytes.len()]; + if let Err(rollback) = backing.write_range(restore_offset, old) { + log::error!( + "bedrock-flash: backing rollback at {restore_offset:#x} failed: {rollback}" + ); + } + } + return Err(error); + } + } + Ok(()) + } + fn replace(&mut self, offset: usize, bytes: &[u8]) -> io::Result { + if self.data[offset..offset + bytes.len()] == *bytes { + return Ok(false); + } + self.persist(&[(offset, bytes)])?; + self.data[offset..offset + bytes.len()].copy_from_slice(bytes); + Ok(true) + } + fn erase(&mut self, part: usize, sectors: &[usize]) -> io::Result { + let erased = vec![0xff; SECTOR_SIZE]; + let ranges: Vec<_> = sectors + .iter() + .map(|sector| (part * PART_SIZE + sector * SECTOR_SIZE, erased.as_slice())) + .collect(); + self.persist(&ranges)?; + let mut changed = false; + for (offset, bytes) in ranges { + let data = &mut self.data[offset..offset + bytes.len()]; + changed |= data.iter().any(|byte| *byte != 0xff); + data.fill(0xff); + } + Ok(changed) + } +} + +pub struct Flash { + st: Mutex, + gen: AtomicU64, +} + +impl Flash { + /// Format a fresh two-position store from a decoded payload. + pub fn new(payload: &[u8]) -> io::Result { + if payload.len() > PART_SIZE { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "Fuel PROM payload exceeds 8 MiB", + )); + } + let mut data = vec![0xff; FLASH_SIZE]; + data[..payload.len()].copy_from_slice(payload); + let label = PART_SIZE + 0x1e0000; + data[label + 0x10..label + 0x14].copy_from_slice(&0x504c4f49u32.to_be_bytes()); + data[label + 0x14..label + 0x18].copy_from_slice(&1u32.to_be_bytes()); + data[label + 0x18..label + 0x1c].copy_from_slice(&1u32.to_be_bytes()); + configure(&mut data); + Ok(Self { + st: Mutex::new(State::new(data)), + gen: AtomicU64::new(0), + }) + } + + /// Load an exact 16 MiB store, or seed a missing/empty one. Wrong sizes are + /// refused without writes. Reopening reapplies Fuel's configuration policy. + /// Completed guest programming and erasure write through to this file. + pub fn with_file(payload: &[u8], path: impl AsRef) -> io::Result { + let mut data = Self::new(payload)?.st.into_inner().data; + let mut backing = FlashBacking::open(path, &mut data).map_err(|error| { + if error.kind() == io::ErrorKind::InvalidData { + io::Error::new( + io::ErrorKind::InvalidData, + "Fuel flash store must be exactly 16 MiB (or empty)", + ) + } else { + error + } + })?; + configure(&mut data); + // Opening an existing store loads it before Fuel's configuration + // fields are refreshed; an empty store was seeded from this same data. + backing.write_range(0, &data)?; + let flash = Self { + st: Mutex::new(State { + data, + parts: std::array::from_fn(|_| AmdCore::default()), + backing: Some(backing), + }), + gen: AtomicU64::new(0), + }; + Ok(flash) + } + + pub fn contents(&self) -> Vec { + self.st.lock().data.clone() + } + pub fn generation(&self) -> u64 { + self.gen.load(Ordering::Acquire) + } + pub fn reset(&self) { + let mut st = self.st.lock(); + for part in &mut st.parts { + part.reset(); + } + self.gen.fetch_add(1, Ordering::Release); + } + /// Advance the model clock independently of flash reads. + pub fn advance_ns(&self, ns: u64) { + let mut st = self.st.lock(); + for part in 0..2 { + let selecting = st.parts[part].selecting(); + let complete = st.parts[part].advance_ns(ns); + if selecting && !st.parts[part].selecting() { + self.gen.fetch_add(1, Ordering::Release); + } + if !complete { + continue; + } + let sectors: Vec<_> = st.parts[part].selected_sectors().collect(); + let success = match st.erase(part, §ors) { + Ok(_) => true, + Err(e) => { + log::error!("bedrock-flash: position {part} erase backing write failed: {e}"); + false + } + }; + st.parts[part].finish_erase(success); + self.gen.fetch_add(1, Ordering::Release); + } + } + pub fn register_locks(self: &Arc) { + let me = self.clone(); + crate::locks::register_lock_fn("bedrock_flash::data", move || me.st.is_locked()); + } + /// Use the same shared flash for local and remote boot spaces. + pub fn boot_view(self: &Arc) -> FlashView { + FlashView { + flash: self.clone(), + range: None, + } + } + /// A contiguous executable alias. Offsets refer to the 16 MiB backing. + pub fn array_view(self: &Arc, offset: usize, len: usize) -> io::Result { + if offset.checked_add(len).is_none_or(|end| end > FLASH_SIZE) { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "Fuel flash alias exceeds backing", + )); + } + Ok(FlashView { + flash: self.clone(), + range: Some((offset, len)), + }) + } +} +impl Resettable for Flash { + fn power_on(&self) { + self.reset(); + } +} + +pub struct FlashView { + flash: Arc, + range: Option<(usize, usize)>, +} +impl FlashView { + fn offset(&self, addr: u64, width: usize) -> Option { + if let Some((base, len)) = self.range { + let off = usize::try_from(addr).ok()?; + return off + .checked_add(width) + .filter(|end| *end <= len) + .map(|_| base + off); + } + let (base, within) = if addr < PART_SIZE as u64 { + (0, addr) + } else if addr >= SECOND_POSITION { + (PART_SIZE, addr - SECOND_POSITION) + } else { + return None; + }; + let within = usize::try_from(within).ok()?; + within + .checked_add(width) + .filter(|end| *end <= PART_SIZE) + .map(|_| base + within) + } + fn read(&self, addr: u64, width: usize) -> Option { + let off = self.offset(addr, width)?; + let mut st = self.flash.st.lock(); + if self.range.is_none() { + let part = off / PART_SIZE; + if !st.parts[part].array_reads() && (width != 2 || off & 1 != 0) { + return None; + } + if width == 2 && off & 1 == 0 { + if let Some(value) = st.parts[part].read16(off % PART_SIZE) { + return Some(value as u64); + } + } + } + Some( + st.data[off..off + width] + .iter() + .fold(0, |v, b| (v << 8) | *b as u64), + ) + } + fn refuse(&self, addr: u64, width: usize, value: u64) -> u32 { + crate::dlog!( + crate::devlog::LogModule::Bedrock, + "bedrock-flash: unsupported {width}-byte store {value:#x} at {addr:#x}; invalid bus cycle or executable view" + ); + BUS_ERR + } +} +impl BusDevice for FlashView { + fn read8(&self, a: u64) -> BusRead8 { + self.read(a, 1) + .map_or_else(BusRead8::err, |v| BusRead8::ok(v as u8)) + } + fn read16(&self, a: u64) -> BusRead16 { + self.read(a, 2) + .map_or_else(BusRead16::err, |v| BusRead16::ok(v as u16)) + } + fn read32(&self, a: u64) -> BusRead32 { + self.read(a, 4) + .map_or_else(BusRead32::err, |v| BusRead32::ok(v as u32)) + } + fn read64(&self, a: u64) -> BusRead64 { + self.read(a, 8).map_or_else(BusRead64::err, BusRead64::ok) + } + fn write8(&self, a: u64, v: u8) -> u32 { + self.refuse(a, 1, v as u64) + } + fn write16(&self, a: u64, v: u16) -> u32 { + self.refuse(a, 2, v as u64) + } + fn write32(&self, a: u64, v: u32) -> u32 { + self.refuse(a, 4, v as u64) + } + fn write64(&self, a: u64, v: u64) -> u32 { + if self.range.is_some() || a & 7 != 0 || v > u16::MAX as u64 { + return self.refuse(a, 8, v); + } + let (part, within) = if a < SECOND_POSITION { + (0, a) + } else { + (1, a - SECOND_POSITION) + }; + let Some(end) = within.checked_add(8) else { + return self.refuse(a, 8, v); + }; + if end > SECOND_POSITION { + return self.refuse(a, 8, v); + } + let offset = (within / 4) as usize; + let mut st = self.flash.st.lock(); + let effect = match st.parts[part].write16(offset, v as u16) { + Ok(effect) => effect, + Err(()) => { + self.flash.gen.fetch_add(1, Ordering::Release); + return BUS_ERR; + } + }; + if let Effect::Program { offset, value } = effect { + let backing = part * PART_SIZE + offset; + let old = u16::from_be_bytes([st.data[backing], st.data[backing + 1]]); + match st.replace(backing, &(old & value).to_be_bytes()) { + Ok(_) => {} + Err(e) => { + log::error!("bedrock-flash: program backing write at {backing:#x} failed: {e}"); + self.flash.gen.fetch_add(1, Ordering::Release); + return BUS_ERR; + } + } + } + self.flash.gen.fetch_add(1, Ordering::Release); + BUS_OK + } + #[cfg(feature = "jitv2")] + fn gen_ptr(&self, _addr: u64) -> *const AtomicU64 { + &self.flash.gen + } +} + +#[cfg(test)] +mod tests { + // Cases follow tests/qtest/sn1-bedrock-flash-test.c in + // jamesbraid/qemu-sgi-origami: program, sector-erase, erase-polling, + // reset-states and backing-file. + use super::*; + fn unlock(v: &FlashView, part: u64) { + assert_eq!(v.write64(part + 8 * 0x555, 0xaa), crate::traits::BUS_OK); + assert_eq!(v.write64(part + 8 * 0x2aa, 0x55), crate::traits::BUS_OK); + } + fn program(v: &FlashView, part: u64, off: u64, value: u16) { + unlock(v, part); + assert_eq!(v.write64(part + 8 * 0x555, 0xa0), crate::traits::BUS_OK); + assert_eq!( + v.write64(part + 4 * off, value as u64), + crate::traits::BUS_OK + ); + } + fn erase_select(v: &FlashView, part: u64, offset: u64) { + unlock(v, part); + assert_eq!(v.write64(part + 8 * 0x555, 0x80), BUS_OK); + unlock(v, part); + assert_eq!(v.write64(part + 4 * offset, 0x30), BUS_OK); + } + + /// The PROM reads CPU configuration and the volume label from its boot flash. + #[test] + fn the_seed_exposes_fuel_configuration_and_volume_label() { + let seed = vec![0x21; 0xb0]; + let f = Flash::new(&seed).unwrap(); + let bytes = f.contents(); + assert_eq!(sum(&bytes[0x60..0xb0]), sum(&seed[0x60..0xb0])); + assert_eq!(&bytes[0x64..0x68], &0x549aa985u32.to_be_bytes()); + assert_eq!(&bytes[0x74..0x78], &500_000_000u32.to_be_bytes()); + assert_eq!(&bytes[0x90..0x94], &2u32.to_be_bytes()); + let label = PART_SIZE + 0x1e0000; + assert_eq!(&bytes[label..label + 16], &[0xff; 16]); + assert_eq!( + &bytes[label + 16..label + 28], + &[0x50, 0x4c, 0x4f, 0x49, 0, 0, 0, 1, 0, 0, 0, 1] + ); + assert!(bytes[label + 28..].iter().all(|b| *b == 0xff)); + assert!(Flash::new(&vec![0; PART_SIZE + 1]).is_err()); + } + /// The PROM programs through one view while executable aliases must + /// keep reading array bytes. + #[test] + fn programming_changes_shared_bytes_without_exposing_status_to_instruction_fetches() { + { + let f = Arc::new(Flash::new(&[]).unwrap()); + let local = f.boot_view(); + let remote = f.boot_view(); + let execute = f.array_view(0, PART_SIZE).unwrap(); + assert_eq!(local.write64(SECOND_POSITION, 0xf0), crate::traits::BUS_OK); + for part in [0, SECOND_POSITION] { + assert_eq!(local.write64(part + 8 * 0x55, 0x98), crate::traits::BUS_OK); + assert_eq!(remote.read16(part + 0x20).data, b'Q' as u16); + assert_eq!(remote.read16((SECOND_POSITION - part) + 0x20).data, 0xffff); + assert_eq!(execute.read16(0x20).data, 0xffff); + assert_eq!(remote.write64(part, 0xf0), crate::traits::BUS_OK); + assert_eq!(local.write64(part + 8 * 0x555, 0xaa), crate::traits::BUS_OK); + assert_eq!( + remote.write64(part + 8 * 0x2aa, 0x55), + crate::traits::BUS_OK + ); + assert_eq!(local.write64(part + 8 * 0x555, 0xa0), crate::traits::BUS_OK); + assert_eq!( + remote.write64(part + 4 * 0x200, 0x1234), + crate::traits::BUS_OK + ); + assert_eq!(local.read16(part + 0x200).data, 0x1234); + } + assert_eq!(execute.read16(0x200).data, 0x1234); + assert!(!local.read8(PART_SIZE as u64).is_ok()); + } + { + let f = Arc::new(Flash::new(&[]).unwrap()); + let v = f.boot_view(); + program(&v, 0, 0x200, 0x1234); + program(&v, 0, 0x200, 0xf0ff); + assert_eq!(v.read16(0x200).data, 0x1034); + unlock(&v, 0); + assert_eq!(v.write64(8 * 0x555, 0xa0), crate::traits::BUS_OK); + let gen = f.generation(); + for (addr, value) in [ + (0x800, 0x10000), + (0x804, 0), + (64 << 20, 0), + ] { + assert_eq!(v.write64(addr, value), BUS_ERR); + } + assert_eq!(v.write16(0x800, 0), BUS_ERR); + assert_eq!(f.generation(), gen); + assert_eq!(v.write64(0x800, 0x00ff), crate::traits::BUS_OK); + assert_eq!(v.read16(0x200).data, 0x0034); + let execute = f.array_view(0, PART_SIZE).unwrap(); + unlock(&v, 0); + assert_eq!(execute.write64(8 * 0x555, 0xa0), BUS_ERR); + assert_eq!(v.write64(8 * 0x555, 0xa0), crate::traits::BUS_OK); + assert_eq!(v.write64(0x808, 0x5555), crate::traits::BUS_OK); + assert_eq!(execute.read16(0x202).data, 0x5555); + } + { + let f = Arc::new(Flash::new(&[]).unwrap()); + let v = f.boot_view(); + assert_eq!(v.write64(0x550, 0x98), BUS_ERR); + assert_eq!(v.read16(0x20).data, 0xffff); + assert_eq!(v.write64(8 * 0x55, 0x98), BUS_OK); + let expected = [ + 0x51, 0x52, 0x59, 2, 0, 0, 0, 0, 0, 0, 0, 0x27, 0x36, 0, 0, 3, 0, 9, 0, 4, 0, 4, 0, 23, + 1, 0, 0, 0, 1, 0x7f, 0, 0, 1, + ]; + for (index, value) in expected.into_iter().enumerate() { + assert_eq!(v.read16(((index + 0x10) * 2) as u64).data, value); + } + assert_eq!(v.read16(0x100).data, 0); + assert!(!v.read8(0x20).is_ok()); + assert!(!v.read16(0x21).is_ok()); + f.reset(); + assert_eq!(v.read16(0x20).data, 0xffff); + } + } + /// The PROM polls erase completion and can reset the chip before + /// selected sectors change. + #[test] + fn erase_completes_only_for_selected_sectors_and_reset_cancels_it() { + let f = Arc::new(Flash::new(&[]).unwrap()); + let v = f.boot_view(); + program(&v, 0, 0x200, 0); + program(&v, 0, 0x10000, 0); + erase_select(&v, 0, 0x200); + assert!(!v.read8(0x200).is_ok()); + let a = v.read16(0x200).data; + let b = v.read16(0x200).data; + assert_eq!((a ^ b) & 0x44, 0x44); + let a = v.read16(0x10000).data; + let b = v.read16(0x10000).data; + assert_eq!((a ^ b) & 0x44, 0x40); + f.advance_ns(40_000); + assert_eq!(v.write64(4 * 0x10000, 0x30), BUS_OK); + f.advance_ns(40_000); + assert_eq!(v.read16(0x10000).data & 8, 0); + assert_eq!(v.write64(SECOND_POSITION + 8 * 0x55, 0x98), BUS_OK); + assert_eq!(v.read16(SECOND_POSITION + 0x20).data, b'Q' as u16); + f.advance_ns(10_000); + assert_eq!(v.read16(0x200).data & 8, 8); + assert_eq!(v.write64(0, 0xf0), BUS_OK); + assert_eq!(v.read16(0x200).data & 0x80, 0); + f.reset(); + f.advance_ns(u64::MAX); + assert_eq!(v.read16(0x200).data, 0); + assert_eq!(v.read16(0x10000).data, 0); + assert_eq!(v.read16(SECOND_POSITION + 0x20).data, 0xffff); + erase_select(&v, 0, 0x200); + assert_eq!(v.write64(0, 0xf0), BUS_OK); + f.advance_ns(u64::MAX); + assert_eq!(v.read16(0x200).data, 0); + erase_select(&v, 0, 0x200); + assert_eq!(v.write64(4 * 0x10000, 0x30), BUS_OK); + f.advance_ns(50_000 + 512_000_000); + assert_eq!(v.read16(0x200).data, 0xffff); + assert_eq!(v.read16(0x10000).data, 0xffff); + unlock(&v, 0); + assert_eq!(v.write64(8 * 0x555, 0xa0), BUS_OK); + f.reset(); + assert_eq!(v.write64(4 * 0x200, 0x1234), BUS_ERR); + assert_eq!(v.read16(0x200).data, 0xffff); + program(&v, 0, 0x200, 0); + program(&v, 0, 0x10000, 0x3456); + erase_select(&v, 0, 0x200); + f.advance_ns(50_000 + 512_000_000); + assert_eq!(v.read16(0x200).data, 0xffff); + assert_eq!(v.read16(0x10000).data, 0x3456); + } + /// Firmware updates must persist across emulator restarts without + /// overwriting a supplied image. + #[test] + fn completed_programming_and_erases_survive_reopening_the_backing_file() { + { + let path = std::env::temp_dir().join(format!("iris-bedrock-store-{}", std::process::id())); + let mut bytes = vec![0x5a; FLASH_SIZE]; + bytes[PART_SIZE] = 0xc7; + std::fs::write(&path, &bytes).unwrap(); + let f = Flash::with_file(&[0x11], &path).unwrap(); + assert_eq!(f.contents()[0], 0x5a); + assert_eq!(f.contents()[PART_SIZE], 0xc7); + assert_eq!(std::fs::read(&path).unwrap(), f.contents()); + drop(f); + std::fs::write(&path, [1, 2, 3]).unwrap(); + assert!(Flash::with_file(&[], &path).is_err()); + assert_eq!(std::fs::read(&path).unwrap(), [1, 2, 3]); + std::fs::write(&path, []).unwrap(); + let g = Flash::with_file(&[0x34], &path).unwrap(); + assert_eq!(g.contents()[0], 0x34); + assert_eq!(std::fs::metadata(&path).unwrap().len(), FLASH_SIZE as u64); + drop(g); + std::fs::remove_file(path).unwrap(); + } + { + let path = std::env::temp_dir().join(format!("iris-amd-persist-{}", std::process::id())); + std::fs::write(&path, []).unwrap(); + let f = Arc::new(Flash::with_file(&[], &path).unwrap()); + let v = f.boot_view(); + program(&v, 0, 0x200, 0x1234); + program(&v, SECOND_POSITION, (PART_SIZE - 2) as u64, 0x5678); + drop(v); + drop(f); + let f = Arc::new(Flash::with_file(&[], &path).unwrap()); + let v = f.boot_view(); + assert_eq!(v.read16(0x200).data, 0x1234); + assert_eq!( + v.read16(SECOND_POSITION + PART_SIZE as u64 - 2).data, + 0x5678 + ); + unlock(&v, SECOND_POSITION); + assert_eq!(v.write64(SECOND_POSITION + 8 * 0x555, 0x80), BUS_OK); + unlock(&v, SECOND_POSITION); + assert_eq!( + v.write64(SECOND_POSITION + 4 * (PART_SIZE as u64 - 2), 0x30), + BUS_OK + ); + f.advance_ns(50_000 + 512_000_000); + drop(v); + drop(f); + let f = Arc::new(Flash::with_file(&[], &path).unwrap()); + let v = f.boot_view(); + assert_eq!(v.read16(0x200).data, 0x1234); + assert_eq!( + v.read16(SECOND_POSITION + PART_SIZE as u64 - 2).data, + 0xffff + ); + std::fs::remove_file(path).unwrap(); + } + } + + /// A host write failure must preserve flash contents and remain visible + /// to the guest. + #[test] + fn failed_backing_writes_leave_array_bytes_unchanged() { + { + let f = Arc::new(Flash::new(&[]).unwrap()); + // A read-only descriptor makes writes fail without a special backend. + let path = std::env::temp_dir().join(format!("iris-amd-read-only-{}", std::process::id())); + std::fs::write(&path, []).unwrap(); + f.st.lock().backing = Some(FlashBacking::from_file( + std::fs::File::open(&path).unwrap(), + FLASH_SIZE, + )); + let v = f.boot_view(); + unlock(&v, 0); + assert_eq!(v.write64(8 * 0x555, 0xa0), BUS_OK); + assert_eq!(v.write64(4 * 0x200, 0x1234), BUS_ERR); + assert_eq!(v.read16(0x200).data, 0xffff); + std::fs::remove_file(path).unwrap(); + } + { + let f = Arc::new(Flash::new(&[]).unwrap()); + let v = f.boot_view(); + program(&v, 0, 0x200, 0); + let path = + std::env::temp_dir().join(format!("iris-amd-erase-failure-{}", std::process::id())); + std::fs::write(&path, []).unwrap(); + f.st.lock().backing = Some(FlashBacking::from_file( + std::fs::File::open(&path).unwrap(), + FLASH_SIZE, + )); + erase_select(&v, 0, 0x200); + f.advance_ns(50_000 + 512_000_000); + assert_eq!(v.read16(0x200).data & 0x20, 0x20); + assert_eq!(f.contents()[0x200], 0); + assert_eq!(v.write64(0, 0xf0), BUS_OK); + assert_eq!(v.read16(0x200).data, 0); + std::fs::remove_file(path).unwrap(); + } + } + +} diff --git a/src/dev/mod.rs b/src/dev/mod.rs index b4ca3d18e..a0a5d060b 100644 --- a/src/dev/mod.rs +++ b/src/dev/mod.rs @@ -60,6 +60,7 @@ pub mod vino; pub mod virtio; pub mod wd33c93a; pub mod z85c30; +pub mod bedrock_flash; pub mod bedrock_io; pub mod bedrock_ni; pub mod bedrock_xb; From f8fd29d0a7010ed2d6b1b956ab33921bc30d5e69 Mon Sep 17 00:00:00 2001 From: James Braid Date: Fri, 30 Oct 2026 08:13:37 +0000 Subject: [PATCH 24/31] xbridge: add the two-head Fuel PCI host bridge Fuel routes PCI configuration, DMA and interrupt packets through two XBridge heads. Keep small sparse PCI decode in window lists so the bridge can share the machine memory map without allocating more Physical tables. A board attached to one of the crossbar's external ports raises its interrupts as crosstalk packets, the same way the PCI heads do. Route them through the same sender, so the Fuel's graphics board reaches the Bedrock's processor interface with no path of its own, and say whether the destination took the packet. The crossbar answers with the Crossbow's widget 0 register map, so it takes its offsets from the Octane's Crossbow model instead of restating them. What stays separate is behavior that differs between the two: the XBridge crossbar's status register carries the requesting port, wide and malformed accesses set the access-error status and answer zero, its interrupts go out as crosstalk packets to any destination, and it resets the heads and ports its fences select. --- src/dev/mod.rs | 1 + src/dev/xbow.rs | 68 +- src/dev/xbridge.rs | 1600 ++++++++++++++++++++++++++++++++++++++ src/dev/xbridge/tests.rs | 738 ++++++++++++++++++ 4 files changed, 2373 insertions(+), 34 deletions(-) create mode 100644 src/dev/xbridge.rs create mode 100644 src/dev/xbridge/tests.rs diff --git a/src/dev/mod.rs b/src/dev/mod.rs index a0a5d060b..db1d9f738 100644 --- a/src/dev/mod.rs +++ b/src/dev/mod.rs @@ -72,3 +72,4 @@ pub mod dp83843; pub mod ics1890; pub mod isp12160; pub mod timekeeper; +pub mod xbridge; diff --git a/src/dev/xbow.rs b/src/dev/xbow.rs index 85f395255..2a6a2b9dd 100644 --- a/src/dev/xbow.rs +++ b/src/dev/xbow.rs @@ -39,51 +39,51 @@ pub const fn widget_id(rev: u32, part: u32, mfgr: u32) -> u32 { } /// The widget HEART answers as, where widget 0's interrupt packets go. -const WIDGET_HEART: u8 = 8; +pub(crate) const WIDGET_HEART: u8 = 8; -const FIRST_WIDGET: u8 = 8; -const LINKS: usize = 8; +pub(crate) const FIRST_WIDGET: u8 = 8; +pub(crate) const LINKS: usize = 8; /// The register block's span in widget 0's window. pub const REG_SPAN: u32 = 0x0100_0000; -const IDENTIFICATION: u32 = 0x00; -const STATUS: u32 = 0x08; -const ERR_UPPER: u32 = 0x10; -const ERR_LOWER: u32 = 0x18; -const CONTROL: u32 = 0x20; -const PACKET_TIMEOUT: u32 = 0x28; -const INT_UPPER: u32 = 0x30; -const INT_LOWER: u32 = 0x38; -const ERR_CMD: u32 = 0x40; -const LLP_CONTROL: u32 = 0x48; -const STATUS_CLEAR: u32 = 0x50; -const ARB_RELOAD: u32 = 0x58; -const PERF_A: u32 = 0x60; -const PERF_B: u32 = 0x68; -const NIC: u32 = 0x70; +pub(crate) const IDENTIFICATION: u32 = 0x00; +pub(crate) const STATUS: u32 = 0x08; +pub(crate) const ERR_UPPER: u32 = 0x10; +pub(crate) const ERR_LOWER: u32 = 0x18; +pub(crate) const CONTROL: u32 = 0x20; +pub(crate) const PACKET_TIMEOUT: u32 = 0x28; +pub(crate) const INT_UPPER: u32 = 0x30; +pub(crate) const INT_LOWER: u32 = 0x38; +pub(crate) const ERR_CMD: u32 = 0x40; +pub(crate) const LLP_CONTROL: u32 = 0x48; +pub(crate) const STATUS_CLEAR: u32 = 0x50; +pub(crate) const ARB_RELOAD: u32 = 0x58; +pub(crate) const PERF_A: u32 = 0x60; +pub(crate) const PERF_B: u32 = 0x68; +pub(crate) const NIC: u32 = 0x70; /// Revision code 5 and later: widget 0's reset fence, then links 8 to 15's /// at a stride of 8, then the resource lock and its clear. -const FENCE_BASE: u32 = 0x78; -const FENCES: usize = 1 + LINKS; -const FENCE_END: u32 = FENCE_BASE + 8 * FENCES as u32; -const LOCK: u32 = 0xc0; -const LOCK_CLEAR: u32 = 0xc8; +pub(crate) const FENCE_BASE: u32 = 0x78; +pub(crate) const FENCES: usize = 1 + LINKS; +pub(crate) const FENCE_END: u32 = FENCE_BASE + 8 * FENCES as u32; +pub(crate) const LOCK: u32 = 0xc0; +pub(crate) const LOCK_CLEAR: u32 = 0xc8; const FENCE_MASK: u32 = 0xff; /// The first revision code with fences and the lock (Crossbow 2.0 in the /// software tables; an Octane2's code-5 part shows as 1.4 in its inventory). const REVISION_FENCES: u32 = 5; -const LINK_BASE: u32 = 0x100; -const LINK_STRIDE: u32 = 0x40; -const LINK_END: u32 = LINK_BASE + LINK_STRIDE * LINKS as u32; -const LINK_FLUSH: u32 = 0x00; -const LINK_CONTROL: u32 = 0x08; -const LINK_STATUS: u32 = 0x10; -const LINK_ARB_UPPER: u32 = 0x18; -const LINK_ARB_LOWER: u32 = 0x20; -const LINK_STATUS_CLEAR: u32 = 0x28; -const LINK_RESET: u32 = 0x30; -const LINK_AUX_STATUS: u32 = 0x38; +pub(crate) const LINK_BASE: u32 = 0x100; +pub(crate) const LINK_STRIDE: u32 = 0x40; +pub(crate) const LINK_END: u32 = LINK_BASE + LINK_STRIDE * LINKS as u32; +pub(crate) const LINK_FLUSH: u32 = 0x00; +pub(crate) const LINK_CONTROL: u32 = 0x08; +pub(crate) const LINK_STATUS: u32 = 0x10; +pub(crate) const LINK_ARB_UPPER: u32 = 0x18; +pub(crate) const LINK_ARB_LOWER: u32 = 0x20; +pub(crate) const LINK_STATUS_CLEAR: u32 = 0x28; +pub(crate) const LINK_RESET: u32 = 0x30; +pub(crate) const LINK_AUX_STATUS: u32 = 0x38; const STATUS_REQUEST: u32 = 1 << 23; const STATUS_LINK_REQUESTS: u32 = 0xff00_0000; diff --git a/src/dev/xbridge.rs b/src/dev/xbridge.rs new file mode 100644 index 000000000..7025b3b20 --- /dev/null +++ b/src/dev/xbridge.rs @@ -0,0 +1,1600 @@ +//! Fuel's XBridge revision 2 contains two independent PCI heads and their +//! shared crossbar. Each head has its own 1024-entry ATE, device windows and +//! interrupt state. Revision 2 places aliases differently from the Octane's +//! single-head Bridge, uses different PCI slot data-phase widths, and sends +//! host DMA through widget 0 with source-aware packets. +//! +//! This remains separate from `bridge.rs` because its two-head topology, +//! alias map and host packet path are externally visible parts of this chip. +//! Generic PCI BAR-run and lane helpers are shared; the ATE and interrupt +//! dispatch remain tied to the XBridge head layout. Sorted BAR runs decode PCI +//! PIO without a `Physical` table; the machine's RAM still uses one. +//! +//! # What is approximated here +//! +//! Slot widths, immediate link initialization, synchronous flushes and +//! revision-2 alias placement follow the project's BSD QEMU model, not a +//! silicon measurement. Timing, external SRAM, flash and diagnostic engines +//! are absent. Undefined accesses latch an error and remain observable. +//! +//! Ported from hw/pci-host/sgi-xbridge.c and hw/misc/sgi-xbridge-xbow.c +//! in jamesbraid/qemu-sgi-origami, revision +//! fe128635b6dfe2511f1bd451684cb30007686df1. + +use crate::dev::w1; +use crate::irq::{InterruptController, IrqLine}; +use crate::pci::{bar_runs, BarRun, PciBar, PciBusDevice, PciDma, PciDmaStatus, PciLanes, Space}; +use crate::physical::{read_bytes, write_bytes}; +use crate::traits::{BusDevice, BusRead16, BusRead32, BusRead64, BusRead8, Resettable, BUS_OK}; +use parking_lot::{Mutex, RwLock}; +use std::collections::HashMap; +use std::sync::{atomic::{AtomicU64, Ordering}, Arc, Weak}; + +pub const HOST_WIDGET: u8 = 10; +pub const HEAD_E: u8 = 14; +pub const HEAD_F: u8 = 15; +pub const ATE_ENTRIES: usize = 1024; +pub const ATE_MAPPED: usize = 512; + +/// Offsets of the PCI head register words, at eight-byte strides. +pub mod reg { + pub const IDENTIFICATION: usize = 0x0; + pub const STATUS: usize = 0x8; + pub const ERR_UPPER: usize = 0x10; + pub const ERR_LOWER: usize = 0x18; + pub const CONTROL: usize = 0x20; + pub const REQUEST_TIMEOUT: usize = 0x28; + pub const INT_DEST_UPPER: usize = 0x30; + pub const INT_DEST_LOWER: usize = 0x38; + pub const LINK_CONFIG: usize = 0x48; + pub const TEST_PIN_CONTROL: usize = 0x70; + pub const DIRECT_MAP: usize = 0x80; + pub const SSRAM_PARITY_ERR: usize = 0x90; + pub const ARBITRATION: usize = 0xa0; + pub const NIC: usize = 0xb0; + pub const BUS_TIMEOUT: usize = 0xc0; + pub const PCI_TYPE1_CONFIG: usize = 0xc8; + pub const PCI_ERR_LOWER: usize = 0xd8; + pub const INTERRUPT_STATUS: usize = 0x100; + pub const INTERRUPT_ENABLE: usize = 0x108; + pub const INTERRUPT_RESET: usize = 0x110; + pub const INTERRUPT_MODE: usize = 0x118; + pub const INTERRUPT_DEVICE: usize = 0x120; + pub const INTERRUPT_ERROR_HOST: usize = 0x128; + pub const INTERRUPT_HOST_BASE: usize = 0x130; + pub const INTERRUPT_HOST_LAST: usize = 0x168; + pub const INTERRUPT_MULTIPLE: usize = 0x170; + pub const FORCE_INTERRUPT_BASE: usize = 0x180; + pub const FORCE_PIN_BASE: usize = 0x1c0; + pub const FORCE_PIN_LAST: usize = 0x1f8; + pub const DEVICE_BASE: usize = 0x200; + pub const DEVICE_LAST: usize = 0x238; + pub const RRB_EVEN: usize = 0x280; + pub const RRB_ODD: usize = 0x288; + pub const RRB_CLEAR: usize = 0x298; + pub const ATE_UPPER_BASE: u64 = 0x10000; + pub const ATE_LOWER_BASE: u64 = 0x12000; + pub const ATE_END: u64 = 0x14000; + +} + +/// The crossbar has the Crossbow's widget 0 register map, which `xbow.rs` +/// owns; the register offsets here are its constants. The access rules and +/// reset values below are this chip's own (the Octane's Crossbow model differs +/// in status, link state, wide accesses and where its interrupts go). +use crate::dev::xbow; + +/// Fences run from widget 0's to link 15's. +const FENCE_LAST: u32 = xbow::FENCE_END - 8; +/// Read-only registers between the resource lock and the first link. +const READ_ONLY_BASE: u32 = xbow::LOCK_CLEAR + 8; +const READ_ONLY_LAST: u32 = xbow::LINK_BASE - 8; +fn ix(off: u32) -> usize { + (off / 8) as usize +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct InterruptPacket { + pub source_widget: u8, + pub widget: u8, + pub address: u64, + pub vector: u8, + pub level: bool, +} +pub type InterruptSender = Arc bool + Send + Sync>; + +/// Source identity and placement hints retained for a host's inbound packet path. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct DmaPacket { + pub source_widget: u8, + pub slot: u8, + pub widget: u8, + pub address: u64, + pub write: bool, + pub half_write_requested: bool, + pub half_read_requested: bool, +} +/// Bytes are in XIO address order after bridge placement. A false return +/// refuses that access. The host applies the requested exchange of doubleword halves. +pub type DmaSender = Arc bool + Send + Sync>; + +struct Routes { + targets: RwLock<[Option>; 16]>, + interrupts: RwLock>, + dma: RwLock>, +} +impl Routes { + fn interrupt(&self, p: InterruptPacket) -> bool { + let sender = self.interrupts.read().clone(); + let accepted = if let Some(send) = sender { + send(p) + } else { + let target = self.targets.read()[p.widget as usize].clone(); + target.is_some_and(|t| { + t.write64( + p.address, + p.vector as u64 | if p.level { 1 << 8 } else { 0 }, + ) == BUS_OK + }) + }; + if !accepted { + crate::dlog!(crate::devlog::LogModule::Bedrock, "xbridge: refused interrupt {p:?}"); + } + accepted + } +} + +/// One chip owns both heads and widget 0; views share routing and reset state. +pub struct XBridge { + routes: Arc, + heads: [Arc; 2], + xbow: Arc, +} +impl XBridge { + pub fn new(memory: Arc) -> Arc { + let routes = Arc::new(Routes { + targets: RwLock::new(std::array::from_fn(|i| { + if i == HOST_WIDGET as usize { + Some(memory.clone()) + } else { + None + } + })), + interrupts: RwLock::new(None), + dma: RwLock::new(None), + }); + let heads = [ + XBridgeHead::new(HEAD_E, 1, routes.clone()), + XBridgeHead::new(HEAD_F, 0, routes.clone()), + ]; + let xbow = Arc::new(Crossbar { + st: Mutex::new(XbowState::new()), + order: Mutex::new(()), + routes: routes.clone(), + heads: heads.each_ref().map(Arc::downgrade), + reset_output: RwLock::new(None), + }); + Arc::new(Self { + routes, + heads, + xbow, + }) + } + pub fn register_locks(self: &Arc) { + use crate::locks::register_lock_fn; + for head in &self.heads { + for (what, probe) in [ + ("state", (|h: &XBridgeHead| h.st.is_locked()) as fn(&XBridgeHead) -> bool), + ("irq_order", |h| h.order.is_locked()), + ("microlan", |h| h.microlan.is_locked()), + ("pio_map", |h| h.pio_map.is_locked()), + ] { + let h = Arc::downgrade(head); + let name = format!("xbridge::head{:x}::{what}", head.widget); + register_lock_fn(name, move || h.upgrade().is_some_and(|h| probe(&h))); + } + } + for (what, probe) in [ + ("state", (|x: &Crossbar| x.st.is_locked()) as fn(&Crossbar) -> bool), + ("irq_order", |x| x.order.is_locked()), + ] { + let x = Arc::downgrade(&self.xbow); + register_lock_fn(format!("xbridge::crossbar::{what}"), move || { + x.upgrade().is_some_and(|x| probe(&x)) + }); + } + } + fn pci_head(&self, widget: u8) -> &Arc { + assert!((HEAD_E..=HEAD_F).contains(&widget)); + &self.heads[(widget - HEAD_E) as usize] + } + pub fn head(&self, widget: u8) -> Arc { + self.pci_head(widget).clone() + } + pub fn crossbar(&self) -> Arc { + self.xbow.clone() + } + /// A reader-specific widget-0 view; the host reads port 10 by default. + pub fn crossbar_for_port(&self, port: u8) -> Arc { + assert!((8..=15).contains(&port)); + Arc::new(CrossbarView { + xbow: self.xbow.clone(), + port, + }) + } + /// Host-port processor projection uses the XBridge widget-0 alias. + pub fn widget(&self, widget: u8) -> Option> { + match widget { + 0 | HOST_WIDGET => Some(self.crossbar()), + HEAD_E | HEAD_F => Some(self.head(widget)), + _ => self + .routes + .targets + .read() + .get(widget as usize) + .and_then(Clone::clone), + } + } + pub fn attach(&self, widget: u8, slot: u8, device: Arc) { + self.pci_head(widget).attach(slot, device); + } + pub fn dma(&self, widget: u8, slot: u8) -> Arc { + assert!(slot < 8); + Arc::new(SlotDma { + head: Arc::downgrade(self.pci_head(widget)), + slot, + }) + } + pub fn pin_line(&self, widget: u8, pin: u8) -> IrqLine { + assert!(pin < 8); + IrqLine::new( + Arc::new(Pin { + head: Arc::downgrade(self.pci_head(widget)), + pci: false, + }), + pin as u32, + ) + } + /// Raw PCI INTx is board wiring, independent of the eight head pins. + pub fn set_pci_irq(&self, widget: u8, slot: u8, pin: u8, output: IrqLine) { + assert!(slot < 8 && pin < 4); + self.pci_head(widget).pci_outputs.write()[(slot * 4 + pin) as usize] = Some(output); + } + pub fn set_interrupt_sender(&self, send: InterruptSender) { + *self.routes.interrupts.write() = Some(send); + } + pub fn set_dma_sender(&self, send: DmaSender) { + *self.routes.dma.write() = Some(send); + } + pub fn set_dma_64bit(&self, widget: u8, slot: u8, wide: bool) { + assert!(slot < 8); + let head = self.pci_head(widget); + { + let mut st = head.st.lock(); + st.dma_64bit = (st.dma_64bit & !(1 << slot)) | ((wide as u8) << slot); + } + head.publish_bars(); + } + pub fn set_wide_slot_mask(&self, widget: u8, mask: u8) { + self.pci_head(widget).st.lock().wide_slots = mask; + } + pub fn set_microlan(&self, widget: u8, bus: w1::Bus) { + *self.pci_head(widget).microlan.lock() = Some(bus); + } + /// An interrupt packet from the widget attached at `p.source_widget`, + /// routed as the heads' packets are. False when nothing takes it. + pub fn send_interrupt(&self, p: InterruptPacket) -> bool { + assert!((8..14).contains(&p.source_widget) && p.source_widget != HOST_WIDGET); + self.routes.interrupt(p) + } + pub fn attach_widget(&self, widget: u8, target: Arc) { + assert!((8..14).contains(&widget) && widget != HOST_WIDGET); + self.routes.targets.write()[widget as usize] = Some(target); + self.set_link_present(widget, true); + } + pub fn set_link_present(&self, widget: u8, present: bool) { + assert!((8..=15).contains(&widget)); + let mut st = self.xbow.st.lock(); + st.present = (st.present & !(1 << (widget - 8))) | ((present as u8) << (widget - 8)); + } + pub fn set_link_8bit(&self, widget: u8, enabled: bool) { + assert!((8..=15).contains(&widget)); + let mut st = self.xbow.st.lock(); + let bit = 1 << (widget - 8); + st.eight_bit = (st.eight_bit & !bit) | if enabled { bit } else { 0 }; + } + /// A pulse for each reset port, outside chip locks. A head's registers, + /// routed PCI lines and function headers are already reset. The board + /// must reset that port's attached engines through this callback. + pub fn set_reset_output(&self, output: Arc) { + *self.xbow.reset_output.write() = Some(output); + } + /// Time is supplied by the machine's virtual clock, in nanoseconds. + pub fn warm_reset(&self, source_port: u8, now_ns: u64) { + self.xbow.warm_reset(source_port, now_ns); + } + pub fn reset(&self) { + for h in &self.heads { + h.reset(); + } + let mut st = self.xbow.st.lock(); + st.reset(); + st.quiet_until = 0; + } +} +impl Resettable for XBridge { + fn power_on(&self) { + self.reset(); + } +} + +struct HeadState { + reg: [u32; 0x300 / 8], + ate: [u64; ATE_ENTRIES], + pins: u8, + dma_64bit: u8, + wide_slots: u8, +} +impl HeadState { + fn r(&self, off: usize) -> u32 { + self.reg[off / 8] + } + fn set(&mut self, off: usize, value: u32) { + self.reg[off / 8] = value; + } + fn reset(&mut self, speed: u8) { + self.reg.fill(0); + self.ate.fill(0); + self.pins = 0; + for (off, value) in [ + (0, 0x2d002049), + (8, 0x20), + (0x20, 0x0f00320f | ((speed as u32) << 4)), + (0x28, 0xfffff), + (reg::LINK_CONFIG, 0x03ff1810), + (0x80, 0x20000), + (0xa0, 0x20008), + (0xc0, 0x10000), + ] { + self.set(off, value); + } + for slot in 0..8 { + self.set( + 0x200 + slot * 8, + 0x18001000 + | if slot < 2 { + slot as u32 * 2 + } else { + slot as u32 + 2 + }, + ); + } + } + fn packet(&self, pin: Option, level: bool) -> InterruptPacket { + let upper = self.r(reg::INT_DEST_UPPER); + let mut address = ((upper as u64 & 0xffff) << 32) | self.r(reg::INT_DEST_LOWER) as u64; + let vector = if let Some(pin) = pin { + let host = self.r(reg::INTERRUPT_HOST_BASE + pin * 8); + address = (address & !(0x3ff << 38)) | (((host >> 8) as u64 & 0x3ff) << 38); + host as u8 + } else { + self.r(reg::INTERRUPT_ERROR_HOST) as u8 + }; + InterruptPacket { + source_widget: (self.r(reg::CONTROL) & 15) as u8, + widget: ((upper >> 16) & 15) as u8, + address, + vector, + level, + } + } +} +// Lock order: order, then microlan (NIC only), then st. +struct XBridgeHead { + widget: u8, + speed: u8, + st: Mutex, + order: Mutex<()>, + routes: Arc, + slots: RwLock<[Option>; 8]>, + pci_outputs: RwLock<[Option; 32]>, + microlan: Mutex>, + pio: Arc, + pio_map: Mutex, +} + +#[derive(Default)] +struct PioMap { + entries: HashMap<(u8, usize, usize), MappedBar>, + runs: Vec<(u64, u64)>, +} + +struct MappedBar { + function: Arc, + lanes: Arc, + device: Arc, +} + +/// Whole BAR-page runs are few and carry no RAM or native fault-handler state. +/// Readers clone one endpoint and release the map lock before calling a device. +#[derive(Default)] +struct PioDecode { + runs: RwLock>, +} +impl PioDecode { + fn publish(&self, mut runs: Vec) -> Vec { + runs.sort_unstable_by_key(|run| run.start); + debug_assert!(runs.windows(2).all(|pair| pair[0].end <= pair[1].start)); + std::mem::replace(&mut *self.runs.write(), runs) + } + fn target(&self, address: u64, size: usize) -> Option> { + let runs = self.runs.read(); + let i = runs.partition_point(|run| run.start <= address).checked_sub(1)?; + let run = &runs[i]; + (address.checked_add(size as u64)? <= run.end).then(|| run.dev.clone()) + } +} + +struct Lanes { + // BAR-relative CPU base, data-phase width and stream/swap policy. + policy: AtomicU64, + alias: bool, + size: u64, +} + +const PIO_INVALID: u64 = 1 << 63; + +impl Lanes { + fn access(&self, f: &dyn PciBusDevice, bar: usize, off: u32, size: usize, value: Option) -> u64 { + let policy = self.policy.load(Ordering::Acquire); + if policy & PIO_INVALID != 0 { + return if value.is_none() { u64::MAX >> (64 - size * 8) } else { 0 }; + } + self.cycle(f, bar, off.wrapping_sub(policy as u32), size, value, policy) + } + + fn cycle(&self, f: &dyn PciBusDevice, bar: usize, off: u32, size: usize, value: Option, policy: u64) -> u64 { + let width = if policy & (1 << 32) != 0 { 8 } else { 4 }; + if size > width { + let hi = self.cycle(f, bar, off, 4, value.map(|v| v >> 32), policy); + let lo = self.cycle(f, bar, off + 4, 4, value.map(|v| v as u32 as u64), policy); + return (hi << 32) | lo; + } + let selected = policy & (1 << 33) != 0; + let reverse = if self.alias { !selected } else { selected }; + let placed = if selected { off } else { off ^ (width - size) as u32 }; + // A reader may have obtained the old table entry before relocation. + // Its new policy must not send the stale address outside the BAR. + if placed as u64 + size as u64 > self.size { + return if value.is_none() { u64::MAX >> (64 - size * 8) } else { 0 }; + } + let mut bytes = [0; 8]; + if let Some(v) = value { + for (i, b) in bytes[..size].iter_mut().enumerate() { + let shift = if self.alias { (size - 1 - i) * 8 } else { i * 8 }; + *b = (v >> shift) as u8; + } + if reverse { bytes[..size].reverse(); } + } + XBridgeHead::bar_cycle(f, bar, placed, &mut bytes[..size], value.is_some()); + if reverse && value.is_none() { bytes[..size].reverse(); } + if self.alias { + bytes[..size].iter().fold(0, |v, &b| (v << 8) | b as u64) + } else { + bytes[..size].iter().enumerate().fold(0, |v, (i, &b)| v | ((b as u64) << (i * 8))) + } + } +} + +impl PciLanes for Lanes { + fn read(&self, f: &dyn PciBusDevice, bar: usize, off: u32, size: usize) -> u64 { + self.access(f, bar, off, size, None) + } + fn write(&self, f: &dyn PciBusDevice, bar: usize, off: u32, size: usize, value: u64) { + self.access(f, bar, off, size, Some(value)); + } +} +struct Pin { + head: Weak, + pci: bool, +} +impl InterruptController for Pin { + fn set_line(&self, line: u32, level: bool) { + if let Some(head) = self.head.upgrade() { + if self.pci { + let output = head + .pci_outputs + .read() + .get(line as usize) + .and_then(Clone::clone); + if let Some(output) = output { + output.set(level); + } else if level { + crate::dlog!(crate::devlog::LogModule::Bedrock, + "xbridge: head {} PCI line {line} has no board wiring", + head.widget + ); + } + } else { + head.pin(line as usize, level); + } + } + } +} +struct SlotDma { + head: Weak, + slot: u8, +} +impl PciDma for SlotDma { + fn dma_read(&self, address: u64, bytes: &mut [u8]) -> PciDmaStatus { + if let Some(h) = self.head.upgrade() { + h.dma(self.slot, address, bytes, false) + } else { + bytes.fill(0); + PciDmaStatus::MasterAbort + } + } + fn dma_write(&self, address: u64, bytes: &[u8]) -> PciDmaStatus { + self.head.upgrade().map_or(PciDmaStatus::MasterAbort, |h| { + h.dma(self.slot, address, &mut bytes.to_vec(), true) + }) + } +} +impl XBridgeHead { + fn new(widget: u8, speed: u8, routes: Arc) -> Arc { + let mut st = HeadState { + reg: [0; 96], + ate: [0; ATE_ENTRIES], + pins: 0, + dma_64bit: 0, + wide_slots: 0, + }; + st.reset(speed); + Arc::new(Self { + widget, + speed, + st: Mutex::new(st), + order: Mutex::new(()), + routes, + slots: RwLock::new(std::array::from_fn(|_| None)), + pci_outputs: RwLock::new(std::array::from_fn(|_| None)), + microlan: Mutex::new(None), + pio: Arc::new(PioDecode::default()), + pio_map: Mutex::new(PioMap::default()), + }) + } + fn attach(self: &Arc, slot: u8, device: Arc) { + assert!(slot < 8); + self.slots.write()[slot as usize] = Some(device.clone()); + device.connect_dma(Arc::new(SlotDma { + head: Arc::downgrade(self), + slot, + })); + let pin = device.config().interrupt_pin(); + if (1..=4).contains(&pin) { + device.connect_interrupt(IrqLine::new( + Arc::new(Pin { + head: Arc::downgrade(self), + pci: true, + }), + slot as u32 * 4 + pin as u32 - 1, + )); + } + self.publish_bars(); + } + fn pin(&self, pin: usize, level: bool) { + if pin >= 8 { + return; + } + let _order = self.order.lock(); + let packet = { + let mut st = self.st.lock(); + let bit = 1 << pin; + let old = st.pins & bit != 0; + st.pins = (st.pins & !bit) | if level { bit } else { 0 }; + let pins = st.pins as u32; + let status = (st.r(reg::INTERRUPT_STATUS) & !255) | pins; + st.set(reg::INTERRUPT_STATUS, status); + (old != level + && st.r(reg::INTERRUPT_ENABLE) & bit as u32 != 0 + && (level || st.r(reg::INTERRUPT_MODE) & bit as u32 != 0)) + .then(|| st.packet(Some(pin), level)) + }; + if let Some(packet) = packet { + self.routes.interrupt(packet); + } + } + fn reset(&self) { + { + let _order = self.order.lock(); + self.st.lock().reset(self.speed); + } + let outputs = self.pci_outputs.read().clone(); + for line in outputs.into_iter().flatten() { + line.set(false); + } + let devices = self.slots.read().clone(); + for d in devices.into_iter().flatten() { + d.config().reset(); + } + self.publish_bars(); + } + fn invalid(&self, address: u64) { + let _order = self.order.lock(); + let packet = { + let mut st = self.st.lock(); + let first = st.r(reg::INTERRUPT_STATUS) & (1 << 24) == 0; + let status = st.r(reg::INTERRUPT_STATUS) | (1 << 24); + st.set(reg::INTERRUPT_STATUS, status); + let multi = st.r(reg::INTERRUPT_MULTIPLE) | (1 << 24); + st.set(reg::INTERRUPT_MULTIPLE, multi); + if first { + st.set(reg::ERR_UPPER, (address >> 32) as u32); + st.set(reg::ERR_LOWER, address as u32); + } + (first && st.r(reg::INTERRUPT_ENABLE) & (1 << 24) != 0).then(|| st.packet(None, true)) + }; + if let Some(p) = packet { + self.routes.interrupt(p); + } + crate::dlog!(crate::devlog::LogModule::Bedrock, + "xbridge: head {} invalid access at {address:#x}", + self.widget + ); + } + fn defined(off: usize) -> bool { + matches!(off, reg::IDENTIFICATION..=reg::TEST_PIN_CONTROL | reg::DIRECT_MAP | reg::SSRAM_PARITY_ERR | reg::ARBITRATION | reg::NIC | reg::BUS_TIMEOUT..=reg::PCI_ERR_LOWER | reg::INTERRUPT_STATUS..=reg::FORCE_PIN_LAST | reg::DEVICE_BASE..=reg::RRB_CLEAR) + } + fn deferred(address: u64) -> bool { + (0x300..0x600).contains(&address) + || (0x30000..0x30008).contains(&address) + || (0xc00000..0x1000000).contains(&address) + } + fn config(&self, address: u64, size: usize, value: Option) -> u64 { + if size == 8 { + let hi = self.config(address, 4, value.map(|v| v >> 32)); + return (hi << 32) | self.config(address + 4, 4, value.map(|v| v & 0xffffffff)); + } + let reg = ((address & 255) as usize) ^ (4 - size); + let slot = if address < 0x28000 { + ((address - 0x20000) >> 12) as usize + } else { + ((self.st.lock().r(reg::PCI_TYPE1_CONFIG) >> 11) & 31) as usize + }; + let bus = if address < 0x28000 { + 0 + } else { + (self.st.lock().r(reg::PCI_TYPE1_CONFIG) >> 16) & 255 + }; + let device = if bus == 0 && address & 0x700 == 0 { + self.slots.read().get(slot).and_then(Clone::clone) + } else { + None + }; + let mask = u32::MAX >> (32 - size * 8); + let shift = (reg & 3) * 8; + if let Some(dev) = device { + if let Some(value) = value { + dev.config().write_masked( + reg / 4, + (value as u32 & mask) << shift, + mask << shift, + ); + self.publish_bars(); + 0 + } else { + ((dev.config().read(reg / 4) >> shift) & mask) as u64 + } + } else { + if value.is_none() { + let mut st = self.st.lock(); + let multi = st.r(reg::INTERRUPT_MULTIPLE) | (1 << 11); + st.set(reg::INTERRUPT_MULTIPLE, multi); + } + mask as u64 + } + } + fn reg_write(&self, address: u64, value: u32) { + let off = address as usize & !7; + let _order = self.order.lock(); + let mut packets = Vec::new(); + if off == 0xb0 { + let pulse = (value >> 10) & 0x3ff; + let sample = (value >> 2) & 255; + let mut bus = self.microlan.lock(); + if let Some(bus) = bus.as_mut() { + let level = w1::slot::slot(bus, pulse, sample); + self.st.lock().set(reg::NIC, 2 | level as u32); + } else { + crate::dlog!(crate::devlog::LogModule::Bedrock, "xbridge: head {} NIC has no board", self.widget); + } + return; + } + { + let mut st = self.st.lock(); + let masked = match off { + reg::CONTROL => (value & 0x0fe1f20f) | ((self.speed as u32) << 4), + reg::REQUEST_TIMEOUT | reg::INT_DEST_UPPER => value & 0xfffff, + reg::INT_DEST_LOWER | reg::RRB_EVEN | reg::RRB_ODD => value, + reg::LINK_CONFIG => value & 0x03ffffff, + reg::TEST_PIN_CONTROL => value & 0xffffff, + reg::DIRECT_MAP => value & 0xf3ffff, + reg::ARBITRATION => value & 0x3ff7f, + reg::BUS_TIMEOUT => value & 0x1f03ff, + reg::PCI_TYPE1_CONFIG => value & 0xfff800, + reg::INTERRUPT_DEVICE => value & 0xffffff, + reg::INTERRUPT_ERROR_HOST | reg::INTERRUPT_MODE => value & 255, + reg::INTERRUPT_ENABLE => { + let value = value & 0x7fffffff; + let newly = value & !st.r(reg::INTERRUPT_ENABLE) & st.pins as u32; + st.set(off, value); + for pin in 0..8 { + if newly & (1 << pin) != 0 { + packets.push(st.packet(Some(pin), true)); + } + } + value + } + reg::INTERRUPT_RESET => { + let clear = (if value & 2 != 0 { 1 << 30 } else { 0 }) + | (if value & 8 != 0 { 1 << 24 } else { 0 }); + let status = st.r(reg::INTERRUPT_STATUS) & !clear; + st.set(reg::INTERRUPT_STATUS, status); + let multi = if value & 64 != 0 { + 0 + } else { + st.r(reg::INTERRUPT_MULTIPLE) & !clear + }; + st.set(reg::INTERRUPT_MULTIPLE, multi); + return; + } + reg::FORCE_INTERRUPT_BASE..=reg::FORCE_PIN_LAST => { + let pin = (off - reg::FORCE_INTERRUPT_BASE) / 8 & 7; + if st.r(reg::INTERRUPT_ENABLE) & (1 << pin) == 0 { + crate::dlog!(crate::devlog::LogModule::Bedrock, "xbridge: masked force semantics unresolved"); + } else if off < reg::FORCE_PIN_BASE || st.pins & (1 << pin) != 0 { + packets.push(st.packet(Some(pin), true)); + } + 0 + } + reg::INTERRUPT_HOST_BASE..=reg::INTERRUPT_HOST_LAST => value & 0x3ffff, + reg::DEVICE_BASE..=reg::DEVICE_LAST => value & 0x1eefffff, + _ => { + crate::dlog!(crate::devlog::LogModule::Bedrock, + "xbridge: head {} ignored register write {off:#x}", + self.widget + ); + return; + } + }; + if !(reg::FORCE_INTERRUPT_BASE..=reg::FORCE_PIN_LAST).contains(&off) { + st.set(off, masked); + } + } + for p in packets { + self.routes.interrupt(p); + } + drop(_order); + if off == reg::CONTROL || (reg::DEVICE_BASE..=reg::DEVICE_LAST).contains(&off) { + self.publish_bars(); + } + } + + fn publish_bars(&self) { + // Keep endpoint identities stable across BAR relocation. Publish the + // complete small decoder after dropping state and attachment locks. + let mut map = self.pio_map.lock(); + let (regs, control, wide) = { + let st = self.st.lock(); + let regs: [u32; 8] = std::array::from_fn(|i| st.r(reg::DEVICE_BASE + i * 8)); + (regs, st.r(reg::CONTROL), st.dma_64bit) + }; + let funcs: Vec<_> = self.slots.read().iter().enumerate() + .filter_map(|(slot, f)| Some((slot, f.clone()?))).collect(); + let windows: Vec<_> = funcs.iter().flat_map(|(slot, f)| { + f.config().windows().into_iter().map(move |(space, bar, base, size)| (*slot, bar, space, base as u64, base as u64 + size as u64)) + }).collect(); + map.runs.clear(); + let mut decoded = Vec::new(); + let mut retired = Vec::new(); + // Host replacement changes an endpoint's function. Guest configuration + // writes only update policy and never allocate another table endpoint. + map.entries.retain(|&(_, slot, _), entry| { + let keep = funcs.iter().any(|(s, f)| *s == slot && Arc::ptr_eq(f, &entry.function)); + if !keep { + entry.lanes.policy.fetch_or(PIO_INVALID, Ordering::AcqRel); + retired.push(entry.device.clone()); + } + keep + }); + for kind in 0u8..11 { + let (cpu, len, space, pci, width, selected) = if kind < 8 { + let slot = kind as usize; + let dev = regs[slot]; + let memory = dev & (1 << 12) != 0; + let mut base = dev & if memory { 0x3ff } else { 0xfff }; + if slot < 2 { base &= !1; } + let (cpu, len) = if slot < 2 { + (0x200000 + slot as u64 * 0x200000, 0x200000) + } else { + (0x600000 + (slot as u64 - 2) * 0x100000, 0x100000) + }; + (cpu, len, if memory { Space::Mem } else { Space::Io }, base << 20, + if dev & (1 << 22) != 0 { 8 } else { 4 }, dev & (1 << 13) != 0) + } else { + let (cpu, len, space, width) = match kind { + 8 => (0x40000000, 0x40000000, Space::Mem, 4), + 9 => (0x80000000, 0x40000000, Space::Mem, 8), + _ => (0x100000000, 0x100000000, Space::Io, 4), + }; + (cpu, len, space, 0, width, control & (1 << if space == Space::Mem { 22 } else { 23 }) != 0) + }; + let runs = bar_runs(&funcs, space, cpu, pci, len, |b| { + let width = if kind == 9 && wide & (1 << b.slot) == 0 { 4 } else { width }; + let policy = b.cpu_base as u32 as u64 | ((width == 8) as u64) << 32 | (selected as u64) << 33; + let entry = map.entries.entry((kind, b.slot, b.bar)).or_insert_with(|| { + let size = windows.iter().find(|&&(slot, bar, _, _, _)| slot == b.slot && bar == b.bar) + .map(|&(_, _, _, lo, hi)| hi - lo).unwrap(); + let lanes = Arc::new(Lanes { policy: AtomicU64::new(policy), alias: kind >= 8, size }); + let device = Arc::new(PciBar::new(lanes.clone(), b.dev.clone(), b.bar, u32::MAX as u64, 0)); + MappedBar { function: b.dev.clone(), lanes, device } + }); + entry.lanes.policy.store(policy, Ordering::Release); + entry.device.clone() + }); + for run in runs { + let lo = run.start - cpu + pci as u64; + let hi = run.end - cpu + pci as u64; + // bar_runs rounds to whole pages. A subpage BAR can still + // overlap one of those pages, so retain the slow decoder there. + let claims = windows.iter().filter(|&&(_, _, s, l, h)| s == space && l < hi && lo < h).count(); + if claims != 1 { continue; } + map.runs.push((run.start, run.end)); + decoded.push(run); + } + } + let previous = self.pio.publish(decoded); + drop(map); + // Replaced functions may run shutdown callbacks from their destructor. + drop(previous); + drop(retired); + } + + fn bar_cycle(dev: &dyn PciBusDevice, bar: usize, off: u32, bytes: &mut [u8], write: bool) { + for (i, part) in bytes.chunks_mut(4).enumerate() { + if write { + let v = part.iter().enumerate().fold(0, |v, (j, &b)| v | ((b as u32) << (j * 8))); + dev.bar_write(bar, off + i as u32 * 4, v, part.len()); + } else { + let v = dev.bar_read(bar, off + i as u32 * 4, part.len()); + for (j, b) in part.iter_mut().enumerate() { *b = (v >> (j * 8)) as u8; } + } + } + } + /// PCI functions see little-endian values; bytes here remain in address order. + fn pci_cycle(&self, space: Space, address: u32, bytes: &mut [u8], write: bool) -> bool { + let claim = self.slots.read().iter().flatten().find_map(|dev| { + let (bar, off) = dev.config().decode(space, address)?; + let (end_bar, end) = dev + .config() + .decode(space, address.checked_add(bytes.len() as u32 - 1)?)?; + (end_bar == bar && end == off + bytes.len() as u32 - 1).then(|| (dev.clone(), bar, off)) + }); + let Some((dev, bar, off)) = claim else { + crate::dlog!(crate::devlog::LogModule::Bedrock, + "xbridge: head {} no PCI BAR for {space:?} {address:#x}", + self.widget + ); + if !write { + bytes.fill(0xff); + } + return false; + }; + Self::bar_cycle(&*dev, bar, off, bytes, write); + true + } + fn window_cycle(&self, slot: usize, off: u32, bytes: &mut [u8], write: bool) { + let dev = self.st.lock().r(reg::DEVICE_BASE + slot * 8); + let width = if dev & (1 << 22) != 0 { 8 } else { 4 }; + if bytes.len() > width { + // Device windows split a doubleword high word first. + let (low, high) = bytes.split_at_mut(4); + self.window_cycle(slot, off, high, write); + self.window_cycle(slot, off + 4, low, write); + return; + } + let memory = dev & (1 << 12) != 0; + let mut base = dev & if memory { 0x3ff } else { 0xfff }; + if slot < 2 { + base &= !1; + } + let stream = dev & (1 << 13) != 0; + let placed = if stream { + off + } else { + off ^ (width - bytes.len()) as u32 + }; + if stream && write { + bytes.reverse(); + } + self.pci_cycle( + if memory { Space::Mem } else { Space::Io }, + (base << 20).wrapping_add(placed), + bytes, + write, + ); + if stream && !write { + bytes.reverse(); + } + } + fn alias_cycle(&self, space: Space, address: u32, width: usize, bytes: &mut [u8], write: bool) { + let width = if width == 8 { + let slots = self.slots.read().clone(); + let access_len = bytes.len(); + let matches: Vec<_> = slots + .iter() + .enumerate() + .flat_map(|(slot, dev)| { + dev.as_ref() + .map(|d| d.config().windows()) + .unwrap_or_default() + .into_iter() + .filter_map(move |(s, _, base, size)| { + (s == space + && address >= base + && (address as u64 + access_len as u64) + <= base as u64 + size as u64) + .then_some(slot) + }) + }) + .collect(); + if matches.len() == 1 && self.st.lock().dma_64bit & (1 << matches[0]) == 0 { + 4 + } else { + 8 + } + } else { + width + }; + if bytes.len() > width { + let (first, last) = bytes.split_at_mut(width); + self.alias_cycle(space, address, width, first, write); + self.alias_cycle(space, address + width as u32, width, last, write); + return; + } + let swap = self.st.lock().r(reg::CONTROL) & (1 << if space == Space::Mem { 22 } else { 23 }) != 0; + let address = if swap { + address + } else { + address ^ (width - bytes.len()) as u32 + }; + if !swap && write { + bytes.reverse(); + } + self.pci_cycle(space, address, bytes, write); + if !swap && !write { + bytes.reverse(); + } + } + fn slow_pio(&self, address: u64, size: usize, value: Option) -> u64 { + let write = value.is_some(); + let mut bytes = [0u8; 8]; + if (0x200000..0xc00000).contains(&address) { + let (slot, off) = if address < 0x600000 { + (((address - 0x200000) >> 21) as usize, address & 0x1fffff) + } else { + (2 + ((address - 0x600000) >> 20) as usize, address & 0xfffff) + }; + if let Some(v) = value { + for (i, b) in bytes[..size].iter_mut().enumerate() { + *b = (v >> (i * 8)) as u8; + } + } + self.window_cycle(slot, off as u32, &mut bytes[..size], write); + return bytes[..size] + .iter() + .enumerate() + .fold(0, |v, (i, &b)| v | ((b as u64) << (i * 8))); + } + let alias = if (0x40000000..0x80000000).contains(&address) { + Some((Space::Mem, address - 0x40000000, 4)) + } else if (0x80000000..0xc0000000).contains(&address) { + Some((Space::Mem, address - 0x80000000, 8)) + } else if (0x100000000..0x200000000).contains(&address) { + Some((Space::Io, address - 0x100000000, 4)) + } else { + None + }; + if let Some((space, off, width)) = alias { + if let Some(v) = value { + for (i, b) in bytes[..size].iter_mut().enumerate() { + *b = (v >> ((size - 1 - i) * 8)) as u8; + } + } + self.alias_cycle(space, off as u32, width, &mut bytes[..size], write); + return bytes[..size].iter().fold(0, |v, &b| (v << 8) | b as u64); + } + 0 + } + fn access(&self, address: u64, size: usize, value: Option) -> u64 { + let write = value.is_some(); + if (0x20000..0x29000).contains(&address) { + return self.config(address, size, value); + } + if (0x200000..0xc00000).contains(&address) + || (0x40000000..0xc0000000).contains(&address) + || (0x100000000..0x200000000).contains(&address) + { + // An unaligned access can straddle a BAR's final mapped page. + if address & 0xfff > 0x1000 - size as u64 { + return self.slow_pio(address, size, value); + } + let Some(endpoint) = self.pio.target(address, size) else { + return self.slow_pio(address, size, value); + }; + return if let Some(v) = value { + match size { + 1 => { endpoint.write8(address, v as u8); } + 2 => { endpoint.write16(address, v as u16); } + 4 => { endpoint.write32(address, v as u32); } + _ => { endpoint.write64(address, v); } + } + 0 + } else { + match size { + 1 => endpoint.read8(address).data as u64, + 2 => endpoint.read16(address).data as u64, + 4 => endpoint.read32(address).data as u64, + _ => endpoint.read64(address).data, + } + }; + } + if size == 8 && address & 7 == 0 && (reg::ATE_UPPER_BASE..reg::ATE_LOWER_BASE).contains(&address) && write { + self.st.lock().ate[((address - reg::ATE_UPPER_BASE) / 8) as usize] = + value.unwrap() & 0x0000ffffffffffff; + return 0; + } + if size == 4 && address & 7 == 4 { + if !write { + if (reg::ATE_UPPER_BASE..reg::ATE_LOWER_BASE).contains(&address) { + return self.st.lock().ate[((address - reg::ATE_UPPER_BASE) / 8) as usize] >> 32; + } + if (reg::ATE_LOWER_BASE..reg::ATE_END).contains(&address) { + return self.st.lock().ate[((address - reg::ATE_LOWER_BASE) / 8) as usize] as u32 as u64; + } + } + if address < 0x300 && Self::defined(address as usize & !7) { + if let Some(value) = value { + self.reg_write(address, value as u32); + return 0; + } + return self.st.lock().r(address as usize & !7) as u64; + } + } + if Self::deferred(address) { + crate::dlog!(crate::devlog::LogModule::Bedrock, + "xbridge: head {} deferred access at {address:#x}", + self.widget + ); + } else { + self.invalid(address); + } + 0 + } + fn translate( + &self, + slot: u8, + address: u64, + write: bool, + packets: bool, + ) -> Option<(u8, u64, u64, bool)> { + let mut st = self.st.lock(); + let dev = st.r(reg::DEVICE_BASE + slot as usize * 8); + let wide = st.dma_64bit & (1 << slot) != 0; + let device_swap = dev & (1 << 19) != 0; + let (widget, target, swap, direct, page, wide64) = if (0x40000000..0x40800000) + .contains(&address) + || (0x60000000..0x60800000).contains(&address) + { + let shift = if st.r(reg::CONTROL) & (1 << 21) != 0 { 14 } else { 12 }; + let index = ((address & 0x1fffffff) >> shift) as usize; + if index >= ATE_MAPPED || st.ate[index] & 1 == 0 { + let v = st.r(reg::INTERRUPT_STATUS) | (1 << 30); + st.set(reg::INTERRUPT_STATUS, v); + let v = st.r(reg::INTERRUPT_MULTIPLE) | (1 << 30); + st.set(reg::INTERRUPT_MULTIPLE, v); + crate::dlog!( + crate::devlog::LogModule::Bedrock, + "xbridge: invalid page DMA {address:#x}, entry {index}" + ); + return None; + } + let ate = st.ate[index]; + let mask = (1 << shift) - 1; + ( + ((ate >> 8) & 15) as u8, + (ate & 0x0000fffffffff000 & !mask) | (address & mask), + ate & 32 != 0 || address & (1 << 29) != 0, + false, + true, + false, + ) + } else if address >= 0x1000000000000000 { + ( + (address >> 60) as u8, + address & 0x0000ffffffffffff, + address & (1 << 55) != 0, + false, + false, + true, + ) + } else if (0x80000000..=0xffffffff).contains(&address) { + let direct = st.r(reg::DIRECT_MAP); + let mut target = ((direct as u64 & 0x1ffff) << 31) | (address & 0x7fffffff); + if direct & 0x1ffff == 0 && direct & (1 << 17) != 0 { + target += 0x20000000; + } + ( + ((direct >> 20) & 15) as u8, + target, + device_swap, + true, + false, + false, + ) + } else { + crate::dlog!(crate::devlog::LogModule::Bedrock, "xbridge: unsupported DMA {address:#x}"); + return None; + }; + let half = packets + && swap + && st.wide_slots & (1 << slot) != 0 + && if write { + direct || page || (wide64 && device_swap) + } else { + direct || (wide64 && wide && device_swap) || (page && address & (1 << 29) != 0) + }; + let mask = if swap || half { + 0 + } else if wide { + 7 + } else { + 3 + }; + Some((widget, target, mask, half)) + } + // Retain the existing memory path's probe before issuing a chunk's writes. + // The neutral helper alone can commit bytes before reporting a refusal. + fn write_memory(mem: &dyn BusDevice, address: u64, data: &[u8]) -> bool { + if data.is_empty() { + return true; + } + let first = address & !7; + let Some(end) = address + .checked_add(data.len() as u64) + .and_then(|a| a.checked_add(7)) + else { + return false; + }; + let mut probe = vec![0; ((end & !7) - first) as usize]; + read_bytes(mem, first, &mut probe) == BUS_OK && write_bytes(mem, address, data) == BUS_OK + } + fn dma(&self, slot: u8, address: u64, bytes: &mut [u8], write: bool) -> PciDmaStatus { + let sender = self.routes.dma.read().clone(); + let mut good = true; + let mut done = 0; + while done < bytes.len() { + let Some(at) = address.checked_add(done as u64) else { + if !write { + bytes[done..].fill(0); + } + return PciDmaStatus::MasterAbort; + }; + let n = (0x1000 - (at as usize & 0xfff)).min(bytes.len() - done); + if (0x3fff0000..0x40000000).contains(&at) { + if !write { + bytes[done..done + n].fill(0); + } + done += n; + continue; + } + if !write { + bytes[done..done + n].fill(0); + } + let translated = self.translate(slot, at, write, sender.is_some()); + let accepted = if let Some((widget, target, mask, half)) = translated { + if let Some(send) = &sender { + // Packet legs use naturally aligned accesses, retaining source identity. + let mut offset = 0; + let mut ok = true; + while offset < n { + let p = target + offset as u64; + let mut size = 8usize; + while size > n - offset || p & (size as u64 - 1) != 0 { + size /= 2; + } + let mut data = [0u8; 8]; + if write { + for i in 0..size { + data[i] = bytes[done + + offset + + if half { + i + } else { + i ^ (mask as usize & (size - 1)) + }]; + } + } + let placed = if half { + p + } else { + p ^ (mask & !(size as u64 - 1)) + }; + let packet = DmaPacket { + source_widget: (self.st.lock().r(reg::CONTROL) & 15) as u8, + slot, + widget, + address: placed, + write, + half_write_requested: half && write && size == 8, + half_read_requested: half && !write && size == 8, + }; + let part = send(packet, &mut data[..size]); + ok &= part; + if !write { + for i in 0..size { + bytes[done + offset + i] = if part { + data[if half { + i + } else { + i ^ (mask as usize & (size - 1)) + }] + } else { + 0 + }; + } + } + offset += size; + } + ok + } else { + let target_device = self.routes.targets.read()[widget as usize].clone(); + target_device.is_some_and(|mem| { + let low = target & !mask; + let high = (target + n as u64 + mask) & !mask; + let mut data = vec![0; (high - low) as usize]; + if write { + for i in 0..n { + data[(((target + i as u64) ^ mask) - low) as usize] = + bytes[done + i]; + } + if low == target && high == target + n as u64 { + Self::write_memory(&*mem, low, &data) + } else { + let mut ok = true; + for i in 0..n { + let p = (target + i as u64) ^ mask; + ok &= Self::write_memory( + &*mem, + p, + &data[(p - low) as usize..(p - low) as usize + 1], + ); + } + ok + } + } else { + // Preserve the old helper's covering-doubleword probe + // and zero the whole failed physical chunk, even if + // the neutral helper returned part of its contents. + let first = low & !7; + let mut fetched = vec![0; (((high + 7) & !7) - first) as usize]; + let ok = read_bytes(&*mem, first, &mut fetched) == BUS_OK; + for i in 0..n { + bytes[done + i] = if ok { + fetched[(((target + i as u64) ^ mask) - first) as usize] + } else { + 0 + }; + } + ok + } + }) + } + } else { + false + }; + if !accepted { + good = false; + } + done += n; + } + if good { + PciDmaStatus::Ok + } else { + PciDmaStatus::MasterAbort + } + } +} + +struct XbowState { + reg: [u32; 96], + status: u32, + present: u8, + eight_bit: u8, + quiet_until: u64, +} +impl XbowState { + fn new() -> Self { + let mut st = Self { + reg: [0; 96], + status: 0, + present: (1 << 2) | (1 << 6) | (1 << 7), + eight_bit: 0, + quiet_until: 0, + }; + st.reset(); + st + } + fn reset(&mut self) { + self.reg.fill(0); + self.status = 0; + for (off, value) in [ + (xbow::IDENTIFICATION, 0x2d000001), + (xbow::CONTROL, 0x80), + (xbow::PACKET_TIMEOUT, 0xfffff), + (xbow::INT_UPPER, 0x80000), + (xbow::LLP_CONTROL, 0x03ff1810), + (xbow::ARB_RELOAD, 2), + (xbow::NIC, 2), + (xbow::FENCE_BASE, 255), + ] { + self.reg[ix(off)] = value; + } + for i in 0..8 { + self.reg[ix(xbow::FENCE_BASE + 8 + i * 8)] = 255; + self.reg[ix(xbow::LINK_BASE + xbow::LINK_CONTROL + i * xbow::LINK_STRIDE)] = 0x80000; + } + } + fn packet(&self, level: bool) -> InterruptPacket { + let upper = self.reg[ix(xbow::INT_UPPER)]; + InterruptPacket { + source_widget: 0, + widget: ((upper >> 16) & 15) as u8, + address: ((upper as u64 & 0xffff) << 32) | self.reg[ix(xbow::INT_LOWER)] as u64, + vector: (upper >> 24) as u8, + level, + } + } +} +// Lock order: order, then st. Reset outputs run with both released. +struct Crossbar { + st: Mutex, + order: Mutex<()>, + routes: Arc, + heads: [Weak; 2], + reset_output: RwLock>>, +} +impl Crossbar { + fn reset_port(&self, port: u8) { + if (14..=15).contains(&port) { + if let Some(h) = self.heads[(port - 14) as usize].upgrade() { + h.reset(); + } + } + let out = self.reset_output.read().clone(); + if let Some(out) = out { + out(port); + } + } + fn warm_reset(&self, source: u8, now: u64) { + assert!((8..=15).contains(&source)); + let ports = { + let mut st = self.st.lock(); + if now < st.quiet_until { + return; + } + st.quiet_until = now.saturating_add(10240); + if st.reg[ix(xbow::FENCE_BASE)] & (1 << (source - 8)) == 0 { + return; + } + let ports: Vec<_> = (0..8) + .filter(|&i| st.reg[ix(xbow::FENCE_BASE + 8 + i * 8)] & (1 << (source - 8)) != 0) + .map(|i| i as u8 + 8) + .collect(); + st.reset(); + ports + }; + for port in ports { + self.reset_port(port); + } + } + fn access(&self, port: u8, address: u64, size: usize, value: Option) -> u64 { + let _order = self.order.lock(); + let mut packet = None; + let mut reset = None; + let mut access_error = false; + let result = { + let mut st = self.st.lock(); + let off = address as u32 & !7; + if size != 4 || address & 7 != 4 || address >= xbow::LINK_END as u64 || (READ_ONLY_BASE..=READ_ONLY_LAST).contains(&off) { + st.status |= 32; + access_error = true; + if st.reg[ix(xbow::CONTROL)] & 32 != 0 && st.status & (1 << 23) == 0 { + st.status |= 1 << 23; + packet = Some(st.packet(true)); + } + crate::dlog!( + crate::devlog::LogModule::Bedrock, + "xbridge-xbow: invalid access {address:#x} width {size}" + ); + 0 + } else if let Some(v) = value { + let v = v as u32; + if off >= xbow::LINK_BASE { + match (off - xbow::LINK_BASE) % xbow::LINK_STRIDE { + xbow::LINK_CONTROL => st.reg[ix(off)] = v & 0xbf7f01ff, + xbow::LINK_ARB_UPPER | xbow::LINK_ARB_LOWER => st.reg[ix(off)] = v, + xbow::LINK_RESET => reset = Some((8 + (off - xbow::LINK_BASE) / xbow::LINK_STRIDE) as u8), + _ => { + st.status |= 32; + access_error = true; + crate::dlog!( + crate::devlog::LogModule::Bedrock, + "xbridge-xbow: read-only link register offset {off:#x}" + ); + } + } + } else { + match off { + xbow::CONTROL => st.reg[ix(off)] = v & 0xfe, + xbow::PACKET_TIMEOUT => st.reg[ix(off)] = v & 0xfffff, + xbow::INT_UPPER => st.reg[ix(off)] = (v & 0xff07ffff) | 0x80000, + xbow::INT_LOWER => st.reg[ix(off)] = v, + xbow::ERR_CMD => { + st.reg[ix(xbow::ERR_CMD)] = 0; + st.reg[ix(xbow::ERR_UPPER)] = 0; + st.reg[ix(xbow::ERR_LOWER)] = 0; + } + xbow::LLP_CONTROL => st.reg[ix(off)] = v & 0x3ffffff, + xbow::ARB_RELOAD => st.reg[ix(off)] = v & 0x3f, + xbow::PERF_A | xbow::PERF_B => st.reg[ix(off)] = v & 0x700000, + xbow::NIC => { + st.reg[ix(off)] = (v & 0xffffc) | 2; + if v & 0xffc00 != 0 { + crate::dlog!( + crate::devlog::LogModule::Bedrock, + "xbridge-xbow: one-wire pulse unimplemented at register offset 0x70" + ); + } + } + xbow::FENCE_BASE..=FENCE_LAST => st.reg[ix(off)] = v & 255, + xbow::LOCK => { + if st.reg[ix(off)] == 0 { + st.reg[ix(off)] = v & 255; + } + } + xbow::LOCK_CLEAR => st.reg[ix(xbow::LOCK)] = 0, + _ => { + st.status |= 32; + access_error = true; + crate::dlog!( + crate::devlog::LogModule::Bedrock, + "xbridge-xbow: read-only register offset {off:#x}" + ); + } + } + } + 0 + } else if off >= xbow::LINK_BASE { + let link = (off - xbow::LINK_BASE) / xbow::LINK_STRIDE; + let present = st.present & (1 << link) != 0; + match (off - xbow::LINK_BASE) % xbow::LINK_STRIDE { + xbow::LINK_FLUSH => 0, + xbow::LINK_STATUS | xbow::LINK_STATUS_CLEAR => { + if present { + 1 << 31 + } else { + 1 << 5 + } + } + xbow::LINK_AUX_STATUS => { + (if present { 1 << 5 } else { 1 << 6 }) + | (if st.eight_bit & (1 << link) != 0 { + 1 << 4 + } else { + 0 + }) + } + xbow::LINK_RESET => { + st.status |= 32; + access_error = true; + 0 + } + _ => st.reg[ix(off)] as u64, + } + } else { + match off { + xbow::STATUS => (st.status | ((port as u32) << 6)) as u64, + xbow::STATUS_CLEAR => { + let v = st.status | ((port as u32) << 6); + if st.status & (1 << 23) != 0 { + packet = Some(st.packet(false)); + } + st.status &= !((1 << 23) | 0x3f); + v as u64 + } + xbow::LOCK_CLEAR => st.reg[ix(xbow::LOCK)] as u64, + _ => st.reg[ix(off)] as u64, + } + } + }; + if access_error && packet.is_none() { + let mut st = self.st.lock(); + if st.reg[ix(xbow::CONTROL)] & 32 != 0 && st.status & (1 << 23) == 0 { + st.status |= 1 << 23; + packet = Some(st.packet(true)); + } + } + if let Some(p) = packet { + self.routes.interrupt(p); + } + // Reset outputs may call back into the chip, so release ordering too. + drop(_order); + if let Some(port) = reset { + self.reset_port(port); + } + result + } +} +struct CrossbarView { + xbow: Arc, + port: u8, +} + +macro_rules! bus_view { + ($ty:ty, $read:expr) => { + impl BusDevice for $ty { + fn read8(&self, a: u64) -> BusRead8 { + BusRead8::ok(($read)(self, a, 1, None) as u8) + } + fn read16(&self, a: u64) -> BusRead16 { + BusRead16::ok(($read)(self, a, 2, None) as u16) + } + fn read32(&self, a: u64) -> BusRead32 { + BusRead32::ok(($read)(self, a, 4, None) as u32) + } + fn read64(&self, a: u64) -> BusRead64 { + BusRead64::ok(($read)(self, a, 8, None)) + } + fn write8(&self, a: u64, v: u8) -> u32 { + ($read)(self, a, 1, Some(v as u64)); + BUS_OK + } + fn write16(&self, a: u64, v: u16) -> u32 { + ($read)(self, a, 2, Some(v as u64)); + BUS_OK + } + fn write32(&self, a: u64, v: u32) -> u32 { + ($read)(self, a, 4, Some(v as u64)); + BUS_OK + } + fn write64(&self, a: u64, v: u64) -> u32 { + ($read)(self, a, 8, Some(v)); + BUS_OK + } + } + }; +} +bus_view!(XBridgeHead, |s: &XBridgeHead, a, n, v| s.access(a, n, v)); +bus_view!(Crossbar, |s: &Crossbar, a, n, v| s.access( + HOST_WIDGET, + a, + n, + v +)); +bus_view!(CrossbarView, |s: &CrossbarView, a, n, v| s + .xbow + .access(s.port, a, n, v)); + +#[cfg(test)] +mod tests; diff --git a/src/dev/xbridge/tests.rs b/src/dev/xbridge/tests.rs new file mode 100644 index 000000000..1caeff8ce --- /dev/null +++ b/src/dev/xbridge/tests.rs @@ -0,0 +1,738 @@ +// Cases follow tests/qtest/sn1-xbridge-test.c (page-dma, interrupt-packets, +// access-errors), sn1-xbridge-chip-test.c (head-dma-reaches-the-host, +// warm-reset-reaches-the-heads) and sn1-xbridge-xbow-test.c (reset-values, +// lock, fences, link-reset-one-shot) in jamesbraid/qemu-sgi-origami. +use super::*; +use crate::traits::BUS_ERR; +struct Memory(Mutex>); +impl BusDevice for Memory { + fn read32(&self, a: u64) -> BusRead32 { + let mem = self.0.lock(); + if a as usize + 4 > mem.len() { + return BusRead32::err(); + } + BusRead32::ok( + mem[a as usize..a as usize + 4] + .iter() + .fold(0, |v, &b| (v << 8) | b as u32), + ) + } + fn write32(&self, a: u64, v: u32) -> u32 { + let mut mem = self.0.lock(); + if a as usize + 4 > mem.len() { + return BUS_ERR; + } + for i in 0..4 { + mem[a as usize + i] = (v >> (24 - i * 8)) as u8; + } + BUS_OK + } + + fn read64(&self, a: u64) -> BusRead64 { + let mem = self.0.lock(); + if a as usize + 8 > mem.len() { + return BusRead64::err(); + } + BusRead64::ok( + mem[a as usize..a as usize + 8] + .iter() + .fold(0, |v, &b| (v << 8) | b as u64), + ) + } + fn write64(&self, a: u64, v: u64) -> u32 { + let mut mem = self.0.lock(); + if a as usize + 8 > mem.len() { + return BUS_ERR; + } + for i in 0..8 { + mem[a as usize + i] = (v >> (56 - i * 8)) as u8; + } + BUS_OK + } + fn write64_masked(&self, a: u64, v: u64, mask: u64) -> u32 { + let old = self.read64(a); + if old.status != BUS_OK { + return BUS_ERR; + } + self.write64(a, (old.data & !mask) | (v & mask)) + } + + fn read8(&self, a: u64) -> BusRead8 { + match self.0.lock().get(a as usize) { + Some(&value) => BusRead8 { + data: value, + status: BUS_OK, + }, + None => BusRead8 { + data: 0, + status: BUS_ERR, + }, + } + } + fn write8(&self, a: u64, v: u8) -> u32 { + match self.0.lock().get_mut(a as usize) { + Some(b) => { + *b = v; + BUS_OK + } + None => BUS_ERR, + } + } +} +fn chip() -> (Arc, Arc) { + let mem = Arc::new(Memory(Mutex::new(vec![0; 0x8000]))); + (XBridge::new(mem.clone()), mem) +} + +/// The machine lock registry must not extend the lifetime of a chip or attached function. +#[test] +fn lock_registry_does_not_keep_the_chip_or_its_devices_alive() { + let memory = Arc::new(Memory(Mutex::new(vec![0; 0x8000]))); + let memory_ref = Arc::downgrade(&memory); + let chip = XBridge::new(memory); + let chip_ref = Arc::downgrade(&chip); + let heads = chip.heads.each_ref().map(Arc::downgrade); + let pio = chip.heads.each_ref().map(|head| Arc::downgrade(&head.pio)); + let crossbar = Arc::downgrade(&chip.xbow); + let card = Card::with_bar(crate::pci::Bar::Mem(0x1000), 0); + let card_ref = Arc::downgrade(&card); + chip.attach(15, 1, card.clone()); + chip.set_reset_output(Arc::new(move |_| card.config.reset())); + chip.head(15).write32(0x21010, 0x02000000); + chip.head(15).write32(0x21004, 2); + assert!(!chip.pci_head(15).pio_map.lock().runs.is_empty()); + chip.register_locks(); + + assert!(heads.iter().all(|head| head.upgrade().is_some())); + assert!(crossbar.upgrade().is_some()); + assert!(card_ref.upgrade().is_some()); + assert!(memory_ref.upgrade().is_some()); + drop(chip); + assert!(chip_ref.upgrade().is_none()); + assert!(heads.iter().all(|head| head.upgrade().is_none())); + assert!(pio.iter().all(|view| view.upgrade().is_none())); + assert!(crossbar.upgrade().is_none()); + assert!(card_ref.upgrade().is_none()); + assert!(memory_ref.upgrade().is_none()); +} +/// A DMA packet reports failure without losing the valid prefix already transferred. +#[test] +fn page_dma_keeps_valid_prefix_and_refuses_invalid_suffix() { + physical_chunk_refusal_does_not_publish_a_prefix(); + let (c, mem) = chip(); + let f = c.head(15); + f.write64(reg::ATE_UPPER_BASE as u64, 0x2000 | (10 << 8) | 0x21); + let dma = c.dma(15, 2); + let bytes = [1, 2, 3, 4, 5, 6, 7, 8]; + assert_eq!(dma.dma_write(0x40000ffc, &bytes), PciDmaStatus::MasterAbort); + assert_eq!(&mem.0.lock()[0x2ffc..0x3000], &[1, 2, 3, 4]); + assert_eq!(&mem.0.lock()[0x3000..0x3004], &[0; 4]); + let mut read = [0xff; 8]; + assert_eq!( + dma.dma_read(0x40000ffc, &mut read), + PciDmaStatus::MasterAbort + ); + assert_eq!(read, [1, 2, 3, 4, 0, 0, 0, 0]); + assert_ne!(f.read32((reg::INTERRUPT_STATUS + 4) as u64).data & (1 << 30), 0); + f.write32((reg::INTERRUPT_RESET + 4) as u64, 2); + assert_eq!(f.read32((reg::INTERRUPT_STATUS + 4) as u64).data & (1 << 30), 0); + f.write64((reg::ATE_UPPER_BASE + 800 * 8) as u64, 0xffff123456789abc); + assert_eq!(f.read32((reg::ATE_UPPER_BASE + 4 + 800 * 8) as u64).data, 0x1234); + assert_eq!(f.read32((reg::ATE_LOWER_BASE + 4 + 800 * 8) as u64).data, 0x56789abc); + f.write64((reg::ATE_UPPER_BASE + 512 * 8) as u64, 0x2a21); + f.write32((reg::DIRECT_MAP + 4) as u64, 9 << 20); + for (address, status) in [(0x80002000, PciDmaStatus::MasterAbort), + (0x3fff8000, PciDmaStatus::Ok), (0x40800000, PciDmaStatus::MasterAbort), + (0x40200000, PciDmaStatus::MasterAbort)] { + let mut bytes = [0xff; 4]; + assert_eq!(dma.dma_read(address, &mut bytes), status); + assert_eq!(bytes, [0; 4]); + } +} +// A mapped memory range whose middle word refuses accesses. +struct FaultMemory(Arc); +impl BusDevice for FaultMemory { + fn read8(&self, a: u64) -> BusRead8 { + if (0x2004..0x2008).contains(&a) { + BusRead8::err() + } else { + self.0.read8(a) + } + } + fn write8(&self, a: u64, v: u8) -> u32 { + if (0x2004..0x2008).contains(&a) { + BUS_ERR + } else { + self.0.write8(a, v) + } + } + fn read32(&self, a: u64) -> BusRead32 { + if (0x2004..0x2008).contains(&a) { + BusRead32::err() + } else { + self.0.read32(a) + } + } + fn write32(&self, a: u64, v: u32) -> u32 { + if (0x2004..0x2008).contains(&a) { + BUS_ERR + } else { + self.0.write32(a, v) + } + } +} +fn physical_chunk_refusal_does_not_publish_a_prefix() { + let mem = Arc::new(Memory(Mutex::new(vec![0xaa; 0x8000]))); + let c = XBridge::new(Arc::new(FaultMemory(mem.clone()))); + let f = c.head(15); + f.write32((reg::DIRECT_MAP + 4) as u64, 10 << 20); + f.write32((reg::DEVICE_BASE + 4) as u64, 1 << 19); + let dma = c.dma(15, 0); + let mut bytes = [0xff; 8]; + assert_eq!( + dma.dma_read(0x80002002, &mut bytes), + PciDmaStatus::MasterAbort + ); + assert_eq!( + bytes, [0; 8], + "a failed physical chunk zeroes its readable prefix and suffix" + ); + assert_eq!( + dma.dma_write(0x80002002, &[1; 8]), + PciDmaStatus::MasterAbort + ); + assert_eq!( + &mem.0.lock()[0x2000..0x2010], + &[0xaa; 16], + "the probe prevents writes before or after the refused word" + ); +} +/// Enabling an asserted guest interrupt pin delivers its programmed destination. +#[test] +fn interrupt_keeps_destination_and_enabling_a_live_pin_asserts() { + let (c, _) = chip(); + let f = c.head(15); + let packets = Arc::new(Mutex::new(Vec::new())); + let capture = packets.clone(); + c.set_interrupt_sender(Arc::new(move |p| { + capture.lock().push(p); + true + })); + f.write32((reg::INT_DEST_UPPER + 4) as u64, 0xa0001); + f.write32((reg::INT_DEST_LOWER + 4) as u64, 0x23450080); + f.write32((reg::INTERRUPT_HOST_BASE + 4) as u64, (0x155 << 8) | 0x42); + c.pin_line(15, 0).set(true); + assert!(packets.lock().is_empty()); + f.write32((reg::INTERRUPT_ENABLE + 4) as u64, 1); + f.write32((reg::INTERRUPT_MODE + 4) as u64, 1); + c.pin_line(15, 0).set(false); + let p = packets.lock(); + assert_eq!(p.len(), 2); + assert_eq!(p[0].source_widget, 15); + assert_eq!(p[0].widget, 10); + assert_eq!(p[0].vector, 0x42); + assert_eq!(p[0].address, (0x123450080 & !(0x3ff << 38)) | (0x155 << 38)); + assert!(p[0].level); + assert!(!p[1].level); + drop(p); + let card = Card::new(); + c.attach(15, 2, card.clone()); + let (probe, line) = crate::irq::LevelProbe::line(); + c.set_pci_irq(15, 2, 0, line); + card.irq.lock().as_ref().unwrap().set(true); + assert!(probe.level()); + assert_eq!(f.read32((reg::INTERRUPT_STATUS + 4) as u64).data & 255, 0); + c.reset(); + assert!(!probe.level()); +} + +/// The guest receives host DMA packets with their original widget and address placement. +#[test] +fn host_dma_places_packets_and_rejects_invalid_addresses() { + let (c, mem) = chip(); + let f = c.head(15); + let d = c.dma(15, 0); + f.write32((reg::DIRECT_MAP + 4) as u64, 10 << 20); + f.write64(reg::ATE_UPPER_BASE as u64, 0x3000 | (10 << 8) | 1); + for (wide, address, target, expected) in [ + (false, 0x80002000, 0x2000, [3, 2, 1, 0, 7, 6, 5, 4]), + (true, 0x80002000, 0x2000, [7, 6, 5, 4, 3, 2, 1, 0]), + (true, 0x60000000, 0x3000, [0, 1, 2, 3, 4, 5, 6, 7]), + (true, (10 << 60) | (1 << 55) | 0x2000, 0x2000, [0, 1, 2, 3, 4, 5, 6, 7]), + ] { + c.set_dma_64bit(15, 0, wide); + assert_eq!(d.dma_write(address, &[0, 1, 2, 3, 4, 5, 6, 7]), PciDmaStatus::Ok); + assert_eq!(&mem.0.lock()[target..target + 8], &expected); + } + let records = Arc::new(Mutex::new(Vec::new())); + let out = records.clone(); + c.set_dma_sender(Arc::new(move |p, b| { + out.lock().push((p, b.to_vec())); + true + })); + c.set_wide_slot_mask(15, 1 << 3); + c.set_dma_64bit(15, 3, true); + f.write32((reg::CONTROL + 4) as u64, 8); + f.write32((reg::DIRECT_MAP + 4) as u64, 10 << 20); + f.write32((reg::DEVICE_BASE + 3 * 8 + 4) as u64, 1 << 19); + assert_eq!( + c.dma(15, 3) + .dma_write(0x80002000, &[0, 1, 2, 3, 4, 5, 6, 7]), + PciDmaStatus::Ok + ); + let r = records.lock(); + assert_eq!(r.len(), 1); + assert_eq!(r[0].0.source_widget, 8); + assert_eq!(r[0].0.slot, 3); + assert_eq!(r[0].0.widget, 10); + assert_eq!(r[0].0.address, 0x2000); + assert!(r[0].0.half_write_requested); + assert_eq!(r[0].1, [0, 1, 2, 3, 4, 5, 6, 7]); + drop(r); + c.set_dma_sender(Arc::new(|p, b| { + if p.address >= 0x2008 { + return false; + } + b.fill(0x42); + true + })); + let mut bytes = [0xff; 16]; + assert_eq!( + c.dma(15, 0).dma_read(0x80002000, &mut bytes), + PciDmaStatus::MasterAbort + ); + assert_eq!( + bytes, + [0x42, 0x42, 0x42, 0x42, 0x42, 0x42, 0x42, 0x42, 0, 0, 0, 0, 0, 0, 0, 0] + ); +} +/// A PCI function may inspect XBridge from its link-reset callback. +#[test] +fn link_reset_callback_resets_a_function_and_can_reenter_the_chip() { + use crate::dev::isp1040::{mbox, reg as isp_reg, CTRL_INT_ENABLE, CTRL_RISC_INT_ENABLE, HOST_CMD_SHIFT}; + use crate::dev::isp12160::Isp12160; + + let (c, _) = chip(); + let isp = Arc::new(Isp12160::new(Isp12160::new_bus(), Isp12160::new_bus())); + c.attach(15, 1, isp.clone()); + let (probe, line) = crate::irq::LevelProbe::line(); + c.set_pci_irq(15, 1, 0, line); + isp.config().write(1, 6); + isp.config().write(5, 0x02000000); + isp.bar_write(1, isp_reg::BUS_CTRL, u32::from(CTRL_INT_ENABLE | CTRL_RISC_INT_ENABLE), 2); + isp.bar_write(1, isp_reg::MAILBOX_BASE, u32::from(mbox::NOP), 2); + isp.bar_write(1, isp_reg::HOST_CMD, 0x5 << HOST_CMD_SHIFT, 2); + assert!(probe.level()); + assert_eq!(isp.bar_read(1, isp_reg::MAILBOX_BASE, 2), u32::from(mbox::COMPLETE)); + + let weak = Arc::downgrade(&c); + let reset_isp = isp.clone(); + c.set_reset_output(Arc::new(move |port| { + assert_eq!(port, 15); + let chip = weak.upgrade().unwrap(); + // Callbacks own attached engines and may inspect or rewire the chip. + assert_eq!(chip.head(port).read32((reg::DIRECT_MAP + 4) as u64).data, 0x20000); + assert_eq!(chip.crossbar().read32((xbow::IDENTIFICATION + 4) as u64).data, 0x2d000001); + assert_eq!(reset_isp.config().command(), 0); + reset_isp.power_on(); + chip.set_reset_output(Arc::new(|_| {})); + })); + c.head(15).write32((reg::DIRECT_MAP + 4) as u64, 0xa00000); + c.crossbar().write32((xbow::LINK_BASE + 7 * xbow::LINK_STRIDE + xbow::LINK_RESET + 4) as u64, 1); + assert!(!probe.level()); + assert_eq!(isp.config().bar_base(1), 0); + assert_eq!(isp.bar_read(1, isp_reg::BUS_CTRL, 2), 0); + assert_eq!(isp.bar_read(1, isp_reg::BUS_STATUS, 2), 0); + assert_eq!(isp.bar_read(1, isp_reg::MAILBOX_BASE, 2), 0); + assert_eq!(isp.bar_read(1, isp_reg::MAILBOX_BASE + 2, 2), 0x4953); +} + +/// Only admitted guest ports receive warm-reset notifications after both heads reset. +#[test] +fn warm_reset_notifies_only_admitted_ports_after_head_state_is_reset() { + let (c, _) = chip(); + let ports = Arc::new(Mutex::new(Vec::new())); + let output = ports.clone(); + let weak = Arc::downgrade(&c); + c.set_reset_output(Arc::new(move |port| { + if port >= 14 { + assert_eq!(weak.upgrade().unwrap().head(port).read32((reg::DIRECT_MAP + 4) as u64).data, 0x20000); + } + output.lock().push(port); + })); + let x = c.crossbar(); + x.write32((xbow::FENCE_BASE + 8 + 6 * 8 + 4) as u64, 0xff & !(1 << 2)); + c.head(14).write32((reg::DIRECT_MAP + 4) as u64, 0xa00000); + c.head(15).write32((reg::DIRECT_MAP + 4) as u64, 0xa00000); + c.warm_reset(10, 0); + assert_eq!(*ports.lock(), vec![8, 9, 10, 11, 12, 13, 15]); + assert_eq!(c.head(14).read32((reg::DIRECT_MAP + 4) as u64).data, 0xa00000); + ports.lock().clear(); + c.warm_reset(10, 10239); + assert!(ports.lock().is_empty()); + c.warm_reset(10, 10240); + assert_eq!(*ports.lock(), vec![8, 9, 10, 11, 12, 13, 14, 15]); +} +/// The crossbar reports the first error once and retains its programmed destination. +#[test] +fn crossbar_errors_deliver_once_then_clear_the_original_destination() { + let (c, _) = chip(); + let x = c.crossbar(); + let packets = Arc::new(Mutex::new(Vec::new())); + let out = packets.clone(); + c.set_interrupt_sender(Arc::new(move |p| { + out.lock().push(p); + true + })); + x.write32((xbow::INT_UPPER + 4) as u64, 0x550a0000); + x.write32((xbow::INT_LOWER + 4) as u64, 0x80); + x.write32((xbow::CONTROL + 4) as u64, 0x20); + x.read64(xbow::IDENTIFICATION as u64); + x.write32((xbow::IDENTIFICATION + 4) as u64, 1); + assert_eq!(packets.lock().len(), 1); + assert_eq!(x.read32((xbow::STATUS_CLEAR + 4) as u64).data & ((1 << 23) | 32), (1 << 23) | 32); + let p = packets.lock(); + assert_eq!(p.len(), 2); + assert_eq!(p[0].vector, 0x55); + assert_eq!(p[0].widget, 10); + assert_eq!(p[0].address, 0x80); + assert!(p[0].level); + assert!(!p[1].level); +} + +struct Card { + config: crate::pci::PciConfig, + bytes: Mutex>, + irq: Mutex>, +} +impl Card { + fn new() -> Arc { + Self::with_status_events(0) + } + fn with_status_events(mask: u16) -> Arc { + Self::with_bar(crate::pci::Bar::Mem(32), mask) + } + fn with_bar(bar: crate::pci::Bar, mask: u16) -> Arc { + use crate::pci::{Bar, PciConfig}; + let size = match bar { Bar::Mem(size) | Bar::Io(size) => size as usize, Bar::Unused => 0 }; + Arc::new(Self { + config: PciConfig::new_with_policy( + 0x1234, + 0x5678, + 0, + 0, + [ + bar, + Bar::Unused, + Bar::Unused, + Bar::Unused, + Bar::Unused, + Bar::Unused, + ], + 1, + crate::pci::PciConfigPolicy { status_w1c_mask: mask, ..Default::default() }, + ), + bytes: Mutex::new((0..size).map(|i| i as u8).collect()), + irq: Mutex::new(None), + }) + } +} +impl PciBusDevice for Card { + fn config(&self) -> &crate::pci::PciConfig { + &self.config + } + fn bar_read(&self, _: usize, off: u32, width: usize) -> u32 { + let b = self.bytes.lock(); + (0..width).fold(0, |v, i| v | ((b[off as usize + i] as u32) << (i * 8))) + } + fn bar_write(&self, _: usize, off: u32, value: u32, width: usize) { + let mut b = self.bytes.lock(); + for i in 0..width { + b[off as usize + i] = (value >> (i * 8)) as u8; + } + } + fn connect_interrupt(&self, line: IrqLine) { + *self.irq.lock() = Some(line); + } +} +/// The guest can identify the first invalid head access and its error destination. +#[test] +fn invalid_register_captures_first_address_and_reports_error_destination() { + let (c, _) = chip(); + let f = c.head(15); + let packets = Arc::new(Mutex::new(Vec::new())); + let out = packets.clone(); + c.set_interrupt_sender(Arc::new(move |p| { + out.lock().push(p); + true + })); + f.write32((reg::INT_DEST_UPPER + 4) as u64, 0xa0000); + f.write32((reg::INT_DEST_LOWER + 4) as u64, 0x80); + f.write32((reg::INTERRUPT_ERROR_HOST + 4) as u64, 73); + f.write32((reg::INTERRUPT_ENABLE + 4) as u64, 1 << 24); + f.read32(0x1000004); + f.read32(0x2000004); + assert_eq!(f.read32((reg::ERR_LOWER + 4) as u64).data, 0x1000004); + assert_eq!(packets.lock().len(), 1); + assert_eq!(packets.lock()[0].vector, 73); + f.write32((reg::INTERRUPT_RESET + 4) as u64, 8); + assert_eq!(f.read32((reg::INTERRUPT_STATUS + 4) as u64).data & (1 << 24), 0); +} +/// Narrow guest configuration writes preserve status bits outside the written lanes. +#[test] +fn partial_configuration_writes_preserve_unwritten_status_events() { + let (c, _) = chip(); + let f = c.head(15); + let card = Card::with_status_events(0xc000); + c.attach(15, 0, card.clone()); + for (bus, expected) in [(0, 0x56781234), (1, u32::MAX)] { + f.write32((reg::PCI_TYPE1_CONFIG + 4) as u64, bus << 16); + assert_eq!(f.read32(0x28000).data, expected); + } + card.config.set_status_bits(0xc000); + assert_eq!(f.write16(0x20006, 0x0102), BUS_OK); + assert_eq!(f.read32(0x20004).data, 0xc200_0102); + assert_eq!(f.write8(0x20007, 3), BUS_OK); + assert_eq!(f.read32(0x20004).data, 0xc200_0103); + assert_eq!(f.write8(0x20006, 2), BUS_OK); + assert_eq!(f.read32(0x20004).data, 0xc200_0203); + assert_eq!(f.write8(0x20005, 0xff), BUS_OK); + assert_eq!(f.read32(0x20004).data, 0xc200_0203); + assert_eq!(f.write8(0x20004, 0x80), BUS_OK); + assert_eq!(f.read32(0x20004).data, 0x4200_0203); + assert_eq!(f.write16(0x20004, 0x4000), BUS_OK); + assert_eq!(f.read32(0x20004).data, 0x0200_0203); + card.config.set_status_bits(0xc000); + assert_eq!(f.write32(0x20004, 0x8000_0002), BUS_OK); + assert_eq!(f.read32(0x20004).data, 0x4200_0002); +} + +fn assert_pio_matches_slow(head: &XBridgeHead, card: &Card, address: u64, mask: usize) { + for size in [1, 2, 4, 8] { + for offset in (0..32).step_by(size) { + let at = address + offset as u64; + let before = card.bytes.lock().clone(); + let expected = (0..size).fold(0u64, |v, i| (v << 8) | before[(offset + i) ^ mask] as u64); + assert_eq!(head.access(at, size, None), expected, "read {at:#x} width {size}"); + assert_eq!(head.slow_pio(at, size, None), expected); + head.access(at, size, Some(0x0123456789abcdef)); + let fast = card.bytes.lock().clone(); + *card.bytes.lock() = before.clone(); + head.slow_pio(at, size, Some(0x0123456789abcdef)); + assert_eq!(*card.bytes.lock(), fast, "write {at:#x} width {size}"); + *card.bytes.lock() = before; + } + } +} + +/// Native BAR dispatch preserves bridge data lanes at every supported access width. +#[test] +fn native_bar_pages_preserve_all_pio_lanes() { + for memory in [false, true] { + let (chip, _) = chip(); + let head = chip.pci_head(15); + let bar = if memory { crate::pci::Bar::Mem(0x2000) } else { crate::pci::Bar::Io(0x2000) }; + let card = Card::with_bar(bar, 0); + // Device window 0 reaches a function owned by slot 3. + chip.attach(15, 3, card.clone()); + head.write32(0x23010, 0x02000000 | u32::from(!memory)); + head.write32(0x23004, if memory { 2 } else { 1 }); + for wide in [false, true] { + chip.set_dma_64bit(15, 3, wide); + for stream in [false, true] { + head.write32((reg::DEVICE_BASE + 4) as u64, 0x20 | if memory { 1 << 12 } else { 0 } + | if wide { 1 << 22 } else { 0 } | if stream { 1 << 13 } else { 0 }); + assert!(head.pio_map.lock().runs.contains(&(0x200000, 0x202000))); + assert_pio_matches_slow(head, &card, 0x200000, if stream { 0 } else if wide { 7 } else { 3 }); + for swap in [false, true] { + head.write32((reg::CONTROL + 4) as u64, if swap { 1 << if memory { 22 } else { 23 } } else { 0 }); + let aliases = if memory { + vec![(0x42000000, false), (0x82000000, wide)] + } else { vec![(0x102000000, false)] }; + for (address, wide) in aliases { + assert_pio_matches_slow(head, &card, address, if swap { 0 } else if wide { 7 } else { 3 }); + } + } + } + } + } +} + +/// A moved or reset BAR stops reaching its former endpoint. +#[test] +fn bar_moves_disable_and_reset_revoke_native_pages_without_new_endpoints() { + let (chip, _) = chip(); + let head = chip.pci_head(15); + let card = Card::with_bar(crate::pci::Bar::Mem(0x1000), 0); + let old_ref = Arc::downgrade(&card); + chip.attach(15, 0, card); + head.write32(0x20010, 0x02000000); + head.write32(0x20004, 2); + head.write32((reg::DEVICE_BASE + 4) as u64, 0x1020); + let original: Vec<_> = head.pio_map.lock().entries.iter() + .map(|(key, entry)| (*key, entry.device.clone())).collect(); + let stale = head.pio_map.lock().entries[&(8, 0, 0)].device.clone(); + head.write32((reg::DEVICE_BASE + 4) as u64, 0x1022); + assert_eq!(head.read32(0x200000).data, u32::MAX); + head.write32((reg::DEVICE_BASE + 4) as u64, 0x1020); + assert_eq!(head.read32(0x200000).data, 0x03020100); + for i in 0..128 { + let base = 0x02000000 + i * 0x1000; + head.write32(0x20010, base); + assert_eq!(head.read32(0x40000000 + base as u64).data, 0x03020100); + if i != 0 { + assert_eq!(head.read32(0x40000000 + base as u64 - 0x1000).data, u32::MAX); + } + } + assert_eq!(stale.read32(0x42000000).data, u32::MAX); + let entries = head.pio_map.lock(); + assert_eq!(entries.entries.len(), original.len()); + for (key, device) in &original { assert!(Arc::ptr_eq(device, &entries.entries[key].device)); } + drop(entries); + head.write8(0x20007, 0); + assert!(head.pio_map.lock().runs.is_empty()); + assert_eq!(head.read32(0x4207f000).data, u32::MAX); + head.write8(0x20007, 2); + assert!(!head.pio_map.lock().runs.is_empty()); + chip.crossbar().write32((xbow::LINK_BASE + 7 * xbow::LINK_STRIDE + xbow::LINK_RESET + 4) as u64, 1); + assert!(head.pio_map.lock().runs.is_empty()); + assert_eq!(head.read32(0x4207f000).data, u32::MAX); + head.write32(0x20010, 0x02000000); + head.write32(0x20004, 2); + let stale = head.pio_map.lock().entries[&(8, 0, 0)].device.clone(); + let replacement = Card::with_bar(crate::pci::Bar::Mem(0x1000), 0); + replacement.bytes.lock().fill(0x5a); + chip.attach(15, 0, replacement); + assert_eq!(head.read32(0x42000000).data, u32::MAX); + assert_eq!(stale.read32(0x42000000).data, u32::MAX); + stale.write32(0x42000000, 0xdeadbeef); + assert_eq!(old_ref.upgrade().unwrap().bytes.lock()[..4], [0, 1, 2, 3]); + head.write32(0x20010, 0x02000000); + head.write32(0x20004, 2); + assert_eq!(head.read32(0x42000000).data, 0x5a5a5a5a); +} + +/// A BAR page shared with another mapping must retain the slow decoder. +#[test] +fn partial_or_overlapping_bar_pages_use_the_slow_decoder() { + let (chip, _) = chip(); + let head = chip.pci_head(15); + let small = Card::new(); + let large = Card::with_bar(crate::pci::Bar::Mem(0x2000), 0); + small.bytes.lock().fill(0xa5); + chip.attach(15, 0, small); + chip.attach(15, 1, large.clone()); + for slot in 0..2 { + head.write32(0x20010 + slot * 0x1000, 0x02000000); + head.write32(0x20004 + slot * 0x1000, 2); + } + head.write32((reg::DEVICE_BASE + 4) as u64, 0x1020); + assert!(head.pio_map.lock().runs.is_empty()); + assert_eq!(head.read32(0x42000000).data, 0xa5a5a5a5); + assert_eq!(head.read32(0x42000020).data, 0x23222120); + head.write32(0x20004, 0); + assert!(!head.pio_map.lock().runs.is_empty()); + assert_pio_matches_slow(head, &large, 0x42000000, 3); + head.write32(0x21004, 0); + head.write32(0x20004, 2); + assert!(head.pio_map.lock().runs.is_empty()); + assert_eq!(head.read32(0x42000000).data, 0xa5a5a5a5); +} + +/// BAR holes and cross-boundary cycles must use the original slow decoder, +/// including after the host publishes a replacement set of BAR runs. +#[test] +fn sparse_bar_lookup_rejects_holes_and_cross_boundary_cycles() { + let decode = PioDecode::default(); + let first: Arc = Arc::new(Memory(Mutex::new(vec![0; 0x4000]))); + let second: Arc = Arc::new(Memory(Mutex::new(vec![0; 0x4000]))); + drop(decode.publish(vec![ + BarRun { start: 0x3000, end: 0x4000, dev: second.clone() }, + BarRun { start: 0x1000, end: 0x2000, dev: first.clone() }, + ])); + assert!(Arc::ptr_eq(&decode.target(0x1000, 8).unwrap(), &first)); + assert!(Arc::ptr_eq(&decode.target(0x1ff8, 8).unwrap(), &first)); + assert!(decode.target(0x1ffc, 8).is_none()); + assert!(decode.target(0x2000, 8).is_none()); + assert!(decode.target(0x2ff8, 8).is_none()); + assert!(Arc::ptr_eq(&decode.target(0x3000, 8).unwrap(), &second)); + assert!(decode.target(0x4000, 8).is_none()); + assert!(decode.target(u64::MAX, 8).is_none()); + drop(decode.publish(vec![BarRun { start: 0x2000, end: 0x3000, dev: first.clone() }])); + assert!(decode.target(0x1000, 8).is_none()); + assert!(Arc::ptr_eq(&decode.target(0x2000, 8).unwrap(), &first)); + assert!(decode.target(0x3000, 8).is_none()); +} + +fn xbow_read(x: &Arc, off: u32) -> u32 { + x.read32((off + 4) as u64).data +} + +/// Software probing the crossbar after reset sees these values. Each +/// link's control word resets to 0x80000; that value belongs in the control +/// word, not in the link reset register, which no read can see. +#[test] +fn the_crossbar_resets_to_its_documented_values() { + let (c, _) = chip(); + let x = c.crossbar(); + for (off, value) in [ + (xbow::ERR_UPPER, 0), + (xbow::ERR_LOWER, 0), + (xbow::CONTROL, 0x80), + (xbow::PACKET_TIMEOUT, 0xfffff), + (xbow::INT_UPPER, 0x80000), + (xbow::INT_LOWER, 0), + (xbow::ERR_CMD, 0), + (xbow::LLP_CONTROL, 0x03ff1810), + (xbow::ARB_RELOAD, 2), + (xbow::PERF_A, 0), + (xbow::PERF_B, 0), + (xbow::NIC, 2), + (xbow::FENCE_BASE, 0xff), + (xbow::LOCK, 0), + ] { + assert_eq!(xbow_read(&x, off), value, "{off:#x}"); + } + for link in 0..8 { + let base = xbow::LINK_BASE + link * xbow::LINK_STRIDE; + assert_eq!(xbow_read(&x, xbow::FENCE_BASE + 8 + link * 8), 0xff); + assert_eq!(xbow_read(&x, base + xbow::LINK_CONTROL), 0x80000, "link {}", link + 8); + assert_eq!(xbow_read(&x, base + xbow::LINK_ARB_UPPER), 0); + assert_eq!(xbow_read(&x, base + xbow::LINK_ARB_LOWER), 0); + } +} + +/// Software shares the crossbar between processors with the resource lock: +/// a write takes it only while it is free, so a second holder is refused +/// until the first writes the clear register. +#[test] +fn the_resource_lock_admits_one_holder_until_cleared() { + let (c, _) = chip(); + let x = c.crossbar(); + let other = c.crossbar_for_port(HEAD_E); + x.write32((xbow::LOCK + 4) as u64, 0x5a); + assert_eq!(xbow_read(&x, xbow::LOCK), 0x5a); + other.write32((xbow::LOCK + 4) as u64, 0x3c); + assert_eq!(xbow_read(&x, xbow::LOCK), 0x5a); + assert_eq!(xbow_read(&x, xbow::LOCK_CLEAR), 0x5a); + x.write32((xbow::LOCK_CLEAR + 4) as u64, 0); + assert_eq!(xbow_read(&x, xbow::LOCK), 0); + other.write32((xbow::LOCK + 4) as u64, 0x3c); + assert_eq!(xbow_read(&x, xbow::LOCK), 0x3c); +} + +/// Every reset fence bit is read and write, a link's own bit included, and +/// only the low eight bits exist. +#[test] +fn reset_fences_keep_their_low_eight_bits() { + let (c, _) = chip(); + let x = c.crossbar(); + for off in (xbow::FENCE_BASE..=FENCE_LAST).step_by(8) { + x.write32((off + 4) as u64, 0); + assert_eq!(xbow_read(&x, off), 0); + x.write32((off + 4) as u64, off as u32); + assert_eq!(xbow_read(&x, off), off as u32 & 0xff); + x.write32((off + 4) as u64, u32::MAX); + assert_eq!(xbow_read(&x, off), 0xff); + } +} From 8d2ed830ae5f438c8221329fc6d863a5f3c3bce3 Mon Sep 17 00:00:00 2001 From: James Braid Date: Fri, 30 Oct 2026 08:13:37 +0000 Subject: [PATCH 25/31] l1: add the Fuel system controller The Fuel console first passes through the L1 system controller. Reuse the shared 16550 core and keep controller framing, power commands and identity records in its protocol model. --- debug.md | 5 +- src/dev/l1.rs | 449 ++++++++++++++++++++++++++ src/dev/l1/model.rs | 764 ++++++++++++++++++++++++++++++++++++++++++++ src/dev/l1/tests.rs | 381 ++++++++++++++++++++++ src/dev/mod.rs | 1 + src/devlog.rs | 8 +- 6 files changed, 1604 insertions(+), 4 deletions(-) create mode 100644 src/dev/l1.rs create mode 100644 src/dev/l1/model.rs create mode 100644 src/dev/l1/tests.rs diff --git a/debug.md b/debug.md index 5d795820d..4cfa05cf2 100644 --- a/debug.md +++ b/debug.md @@ -541,10 +541,11 @@ Per-module logging; all of it needs a developer build to produce output. | `log file ` | Send a module's log to a file | Modules: `net hpc3 seeq hal2 mc rex3 mips ioc scsi pdma vino dcb vc2 cmap -xmap bt445 scc ps2 rtc eeprom l1i l1d l2c bedrock`. Mask categories: `pdma` — +xmap bt445 scc ps2 rtc eeprom l1i l1d l2c bedrock l1`. Mask categories: `pdma` — `hal enet scsi`; `mips` — `insn tlb mem fpu cp0`; `l1i`/`l1d`/`l2c` — `hit miss op`; every module also takes `on`/`all`, `off`/`none` or a hex mask. -`bedrock` covers the Fuel's devices on its Bedrock hub. +`bedrock` covers the Fuel's devices on its Bedrock hub, and `l1` its system +controller. `IRIS_DEBUG_LOG` or `[debug] debug_log` set the same thing at startup. ### Performance and locks diff --git a/src/dev/l1.rs b/src/dev/l1.rs new file mode 100644 index 000000000..c23b6c256 --- /dev/null +++ b/src/dev/l1.rs @@ -0,0 +1,449 @@ +//! Fuel's L1 system controller and its processor-side UART. +//! +//! The internal L1 connection is separate from IOC3's external serial ports. +//! The Ns16550 owns the UART registers, FIFO, timeout and interrupt state; +//! this device keeps the controller protocol and its eight-byte register stride. +//! Controller identities and power transitions are functional approximations. +//! Remote routing and controller diagnostics are absent. +//! +//! Ported from hw/misc/sgi-l1.c in jamesbraid/qemu-sgi-origami. + +use crate::dev::ns16550::{com_reg, Ns16550}; +use crate::dev::serial_host::{null_serial_backend, SerialBackend}; +use crate::devlog::LogModule; +use crate::irq::IrqLine; +use crate::traits::{BusDevice, BusRead64, Device, Resettable, BUS_ERR, BUS_OK}; +use parking_lot::Mutex; +use std::collections::VecDeque; +use std::io; +use std::sync::{ + atomic::{AtomicBool, AtomicU64, Ordering}, + Arc, +}; + +mod model; +use model::State; +pub use model::{parse_revision, L1Config, L1Event, DEFAULT_BRICK_SERIAL, DEFAULT_REVISION}; + +fn fcs(mut crc: u16, byte: u8) -> u16 { + crc ^= byte as u16; + for _ in 0..8 { + crc = (crc >> 1) ^ if crc & 1 != 0 { 0x8408 } else { 0 }; + } + crc +} + +fn encode(payload: &[u8]) -> Vec { + let crc = !payload.iter().fold(0xffff, |value, byte| fcs(value, *byte)); + let mut out = vec![0x7e]; + for byte in payload.iter().copied().chain([crc as u8, (crc >> 8) as u8]) { + if matches!(byte, 0x7e | 0x7d | 0x0d) { + out.extend_from_slice(&[0x7d, byte ^ 0x20]); + } else { + out.push(byte); + } + } + out.push(0x7e); + out +} + +const UART_BAUDBASE: u32 = 115200; +const UART_WINDOW_SIZE: u64 = 0x40; + +mod uart_offset { + use super::com_reg; + pub const DATA: u64 = 0x00; + pub const IER: u64 = 0x08; + pub const IIR_FCR: u64 = 0x10; + pub const LCR: u64 = 0x18; + pub const MCR: u64 = 0x20; + pub const LSR: u64 = 0x28; + pub const MSR: u64 = 0x30; + pub const SCR: u64 = 0x38; + + pub fn register(offset: u64) -> u32 { + match offset { + DATA => com_reg::THR, + IER => com_reg::IER, + IIR_FCR => com_reg::IIR_FCR, + LCR => com_reg::LCR, + MCR => com_reg::MCR, + LSR => com_reg::LSR, + MSR => com_reg::MSR, + SCR => com_reg::SCR, + _ => unreachable!("validated L1 UART offset"), + } + } +} + +type EventHandler = Arc; + +#[derive(Default)] +struct Dispatch { + active: bool, + pending: bool, + level: Option, +} + +/// The UART backend does not connect guest L1 commands to a host terminal. +/// Guest transmit bytes are handed to the protocol parser after the UART lock +/// is released; receive bytes come from the protocol reply queue. +struct ProtocolBackend { + guest_tx: Arc>>, + protocol_rx: Arc>>, +} + +impl SerialBackend for ProtocolBackend { + fn send_byte(&self, byte: u8) { + self.guest_tx.lock().push_back(byte); + } + + fn recv_byte(&self) -> io::Result { + self.protocol_rx + .lock() + .pop_front() + .ok_or_else(|| io::ErrorKind::WouldBlock.into()) + } +} + +/// The internal L1 connection is independent of IOC3's external serial ports. +/// Register offsets are relative to the 0x40-byte UART window. Only aligned +/// 64-bit accesses are decoded; register values occupy the low eight bits. +pub struct L1 { + // UART RX takes UART -> protocol queue. Model replies take state -> protocol + // queue. No path holds the protocol queue while acquiring either lock. + state: Arc>, + // Take dispatch before state when draining, release it before the UART or callbacks. + dispatch: Arc>, + uart: Arc>, + guest_tx: Arc>>, + backend: Arc>>, + irq: Arc>, + handler: Arc>>, + running: Arc, + clock: Arc, + worker: Mutex>>, + wake: Arc<(Mutex<()>, parking_lot::Condvar)>, +} +impl L1 { + pub fn new(config: L1Config) -> Result { + if config.rack > 0x3ff || config.bay > 0x3f || config.node_position > 3 { + return Err("L1 rack, bay or node position out of range".into()); + } + if config.mac == [0; 6] || config.mac[0] & 1 != 0 { + return Err("L1 MAC must be nonzero unicast".into()); + } + if config + .brick_serial + .as_ref() + .is_some_and(|s| s.len() > 37 || s.as_bytes().contains(&0)) + { + return Err("L1 brick serial must fit a 37-byte text field".into()); + } + + let protocol_rx = Arc::new(Mutex::new(VecDeque::new())); + let state = Arc::new(Mutex::new(State::new(config, protocol_rx.clone()))); + let guest_tx = Arc::new(Mutex::new(VecDeque::new())); + let protocol_backend = Arc::new(ProtocolBackend { + guest_tx: guest_tx.clone(), + protocol_rx, + }); + let mut uart = Ns16550::new(UART_BAUDBASE); + uart.set_backend(protocol_backend); + + Ok(Self { + state, + dispatch: Arc::new(Mutex::new(Dispatch::default())), + uart: Arc::new(Mutex::new(uart)), + guest_tx, + backend: Arc::new(Mutex::new(null_serial_backend())), + irq: Arc::new(Mutex::new(IrqLine::unconnected())), + handler: Arc::new(Mutex::new(None)), + running: Arc::new(AtomicBool::new(false)), + clock: Arc::new(AtomicU64::new(0)), + worker: Mutex::new(None), + wake: Arc::new((Mutex::new(()), parking_lot::Condvar::new())), + }) + } + + /// Install the separate host-side L1 console connection. + pub fn set_serial_backend(&self, backend: Arc) { + *self.backend.lock() = backend; + } + + pub fn set_irq(&self, line: IrqLine) { + *self.irq.lock() = line; + self.dispatch.lock().level = None; + self.flush(); + } + + /// Power/reset requests reach the machine only after the reply is queued + /// and after the device state lock is released. + pub fn set_event_handler(&self, handler: EventHandler) { + *self.handler.lock() = Some(handler); + } + + #[cfg(test)] + pub fn node_powered(&self) -> bool { + self.state.lock().powered + } + + pub fn station_mac(&self) -> [u8; 6] { + self.state.lock().config.mac + } + + #[cfg(test)] + pub fn display_line(&self) -> String { + self.state.lock().display.clone() + } + + #[cfg(test)] + pub fn record(&self, component: usize, area: u32) -> Option> { + let index = match area { + 1 => 0, + 2 => 1, + 4 => 2, + _ => return None, + }; + self.state.lock().records.get(component).and_then(|r| { + if r[index].is_empty() { + None + } else { + Some(r[index].clone()) + } + }) + } + + /// Install a raw area before execution. It replaces only the named area; + /// station identity remains the configured MAC, as in the Fuel model. + #[cfg(test)] + pub fn set_record(&self, component: usize, area: u32, bytes: Vec) -> Result<(), String> { + let index = match area { + 1 => 0, + 2 => 1, + 4 => 2, + _ => return Err("unsupported L1 area".into()), + }; + if component >= 16 || bytes.len() > 65535 { + return Err("L1 record out of range".into()); + } + self.state.lock().records[component][index] = bytes; + Ok(()) + } + + pub fn send_event(&self, code: u32, number: u32, text: &str) { + self.state.lock().event(code, number, text); + self.service_protocol_uart(); + self.flush(); + } + + /// Deterministic input path shared with the polled host serial backend. + pub fn receive_console(&self, bytes: &[u8]) { + for &byte in bytes { + self.state.lock().console_input(byte); + self.service_protocol_uart(); + self.flush(); + } + } + + pub fn poll_serial(&self) { + let backend = self.backend.lock().clone(); + for _ in 0..256 { + if self.state.lock().queue.lock().len() > 4088 { + break; + } + match backend.recv_byte() { + Ok(byte) => self.receive_console(&[byte]), + Err(error) => { + if error.kind() != io::ErrorKind::WouldBlock { + log::debug!("L1 serial input: {error}"); + } + break; + } + } + } + } + + /// Let the native UART pull reply bytes from the model's shared bounded queue. + fn service_protocol_uart(&self) { + let mut uart = self.uart.lock(); + uart.pump(); + uart.check_timeout(std::time::Instant::now()); + } + + /// Parse guest transmit only after the UART lock is released. The UART + /// callback only appends to this independent queue. + fn process_guest_transmit(&self) { + let bytes: Vec<_> = self.guest_tx.lock().drain(..).collect(); + if bytes.is_empty() { + return; + } + { + let mut state = self.state.lock(); + for byte in bytes { + state.parse(byte); + } + } + self.service_protocol_uart(); + } + + fn flush(&self) { + let mut dispatch = self.dispatch.lock(); + dispatch.pending = true; + if dispatch.active { + return; + } + dispatch.active = true; + loop { + dispatch.pending = false; + let (output, actions) = { + let mut state = self.state.lock(); + ( + std::mem::take(&mut state.output), + std::mem::take(&mut state.actions), + ) + }; + drop(dispatch); + let level = self.uart.lock().irq(); + + // Reentrant reads request another drain, but must not repeat an + // unchanged IRQ and recursively trigger the same machine callback. + dispatch = self.dispatch.lock(); + let changed = dispatch.level != Some(level); + dispatch.level = Some(level); + drop(dispatch); + + if changed { + let irq = self.irq.lock().clone(); + irq.set(level); + } + let backend = self.backend.lock().clone(); + for byte in output { + backend.send_byte(byte); + } + let handler = self.handler.lock().clone(); + for action in actions { + if let Some(handler) = &handler { + handler(action); + } else { + crate::dlog!(LogModule::L1, "unconnected L1 action {action:?}"); + } + } + + dispatch = self.dispatch.lock(); + if !dispatch.pending { + dispatch.active = false; + break; + } + } + } +} +impl BusDevice for L1 { + fn read64(&self, addr: u64) -> BusRead64 { + if addr >= UART_WINDOW_SIZE || addr & 7 != 0 { + return BusRead64::err(); + } + let value = self.uart.lock().read(uart_offset::register(addr)); + self.flush(); + BusRead64::ok(value as u64) + } + + fn write64(&self, addr: u64, value: u64) -> u32 { + if addr >= UART_WINDOW_SIZE || addr & 7 != 0 { + return BUS_ERR; + } + let register = uart_offset::register(addr); + let value = value as u8; + if register == com_reg::IIR_FCR && value & com_reg::FCR_RFR != 0 { + self.state.lock().queue.lock().clear(); + } + self.uart.lock().write(register, value); + self.process_guest_transmit(); + self.flush(); + BUS_OK + } +} +impl Resettable for L1 { + fn power_on(&self) { + self.state.lock().reset(); + self.guest_tx.lock().clear(); + self.uart.lock().reset(); + self.clock.store(0, Ordering::Relaxed); + self.flush(); + } +} +impl Device for L1 { + fn start(&self) { + let mut worker = self.worker.lock(); + if worker.is_some() { + return; + } + self.running.store(true, Ordering::Release); + let device = Self { + state: self.state.clone(), + dispatch: self.dispatch.clone(), + uart: self.uart.clone(), + guest_tx: self.guest_tx.clone(), + backend: self.backend.clone(), + irq: self.irq.clone(), + handler: self.handler.clone(), + running: self.running.clone(), + clock: self.clock.clone(), + worker: Mutex::new(None), + wake: self.wake.clone(), + }; + *worker = Some( + std::thread::Builder::new() + .name("l1-serial".into()) + .spawn(move || { + while device.is_running() { + device.poll_serial(); + let mut guard = device.wake.0.lock(); + if device.is_running() { + device + .wake + .1 + .wait_for(&mut guard, std::time::Duration::from_millis(2)); + } + } + }) + .expect("start L1 serial worker"), + ); + } + + fn stop(&self) { + let mut slot = self.worker.lock(); + { + let _guard = self.wake.0.lock(); + self.running.store(false, Ordering::Release); + self.wake.1.notify_all(); + } + let worker = slot.take(); + if let Some(worker) = worker { + if worker.thread().id() != std::thread::current().id() { + let _ = worker.join(); + } + } + } + + fn is_running(&self) -> bool { + self.running.load(Ordering::Acquire) + } + + fn get_clock(&self) -> u64 { + self.clock.load(Ordering::Relaxed) + } + + fn step(&self, cycles: u64) { + if self.is_running() { + self.clock.fetch_add(cycles, Ordering::Relaxed); + } + } +} +impl Drop for L1 { + fn drop(&mut self) { + self.stop(); + } +} +#[cfg(test)] +mod tests; diff --git a/src/dev/l1/model.rs b/src/dev/l1/model.rs new file mode 100644 index 000000000..907fd59dd --- /dev/null +++ b/src/dev/l1/model.rs @@ -0,0 +1,764 @@ +//! Fuel's L1 controller protocol parser and reply model. +//! +//! This state owns packet framing, controller identity records, console +//! commands and power events. The UART register file and receive pacing live +//! in the shared Ns16550 core. +//! +//! Ported from hw/misc/sgi-l1.c in jamesbraid/qemu-sgi-origami. + +use super::*; + +/// Firmware revision the model reports for request 0x0005. +/// +/// Model choice: 1.48.1 is the revision a retained real Fuel L1 session +/// prints (the Fuel contract's L1 section). The model reproduces that +/// firmware's replies, not its code, so the revision is a label. +pub const DEFAULT_REVISION: [u32; 3] = [1, 48, 1]; + +/// Brick serial the model reports for request 0x0004. +/// +/// Model choice: the marked synthetic text form, system-board letter `N`, +/// instance 0 (the component taxonomy's synthetic serials). The real format +/// is unknown, and the value must not pass as a real SGI serial. +pub const DEFAULT_BRICK_SERIAL: &str = "EMUN00"; + +/// Parse a "major.minor.patch" revision string, as a `[fuel] l1_revision` +/// override spells it. Each part is a decimal number. +pub fn parse_revision(text: &str) -> Result<[u32; 3], String> { + let bad = || format!("L1 revision \"{text}\" must be major.minor.patch, each a decimal number"); + let mut parts = text.split('.'); + let mut rev = [0u32; 3]; + for slot in &mut rev { + let part = parts.next().ok_or_else(bad)?; + if part.is_empty() || !part.bytes().all(|b| b.is_ascii_digit()) { + return Err(bad()); + } + *slot = part.parse().map_err(|_| bad())?; + } + if parts.next().is_some() { + return Err(bad()); + } + Ok(rev) +} + +#[derive(Clone, Debug)] +pub struct L1Config { + pub rack: u32, + pub bay: u32, + pub node_position: u8, + pub type_code: u32, + pub revision: Option<[u32; 3]>, + pub mac: [u8; 6], + pub brick_serial: Option, + pub auto_power: bool, +} +impl Default for L1Config { + fn default() -> Self { + Self { + rack: 1, + bay: 1, + node_position: 0, + type_code: 0x41, + revision: Some(DEFAULT_REVISION), + mac: [8, 0, 0x69, 0x12, 0x34, 0x56], + brick_serial: Some(DEFAULT_BRICK_SERIAL.into()), + auto_power: true, + } + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum L1Event { + Power(bool), + Reset, + SoftReset, + Nmi, + Shutdown, +} + +pub(super) struct State { + pub config: L1Config, + pub queue: Arc>>, + frame: Vec, + in_frame: bool, + escape: bool, + overflow: bool, + pub display: String, + pub records: [[Vec; 3]; 16], + pub actions: Vec, + pub output: Vec, + pub powered: bool, + console: bool, + escaped: bool, + command: Vec, + port: u8, + debug: u32, + partition: u32, + two_bit: u32, + reasons: [u32; 2], + events: Option, + power_down_pending: bool, +} +fn text_field(out: &mut Vec, text: &[u8]) { + out.push(1); + out.extend_from_slice(text); + out.push(0); +} +pub(super) fn ints(values: &[u32]) -> Vec { + let mut out = Vec::new(); + for v in values { + out.push(0); + out.extend_from_slice(&v.to_be_bytes()); + } + out +} +fn word(p: &[u8]) -> u32 { + u32::from_be_bytes(p[..4].try_into().unwrap()) +} +fn arguments(mut p: &[u8], count: u8) -> Option> { + let mut fields = Vec::new(); + for _ in 0..count { + let tag = *p.first()?; + let len = match tag { + 0 => 5, + 1 => p[1..].iter().position(|b| *b == 0)? + 2, + t if t & 0x80 != 0 => 1 + (t & 0x7f) as usize, + _ => return None, + }; + fields.push(p.get(..len)?); + p = &p[len..]; + } + if p.is_empty() { + Some(fields) + } else { + None + } +} +fn area_finish(mut area: Vec) -> Vec { + while (area.len() + 1) % 8 != 0 { + area.push(0); + } + let sum = area.iter().fold(0u8, |s, b| s.wrapping_add(*b)); + area.push(sum.wrapping_neg()); + area +} +fn ia_text(area: &mut Vec, text: &str) { + area.push(0xc0 | text.len() as u8); + area.extend_from_slice(text.as_bytes()); +} +fn board(mac: Option<[u8; 6]>) -> Vec { + let mut b = vec![0; 6]; + for s in ["SYNTHETIC", "IP34", "EMUN00", "030-1707-003"] { + ia_text(&mut b, s); + } + b.push(0); + ia_text(&mut b, "-G"); + b.extend_from_slice(&[1, 0]); + ia_text(&mut b, "00"); + for _ in 0..3 { + b.extend_from_slice(&[4, 0, 0, 0, 0]); + } + if let Some(mac) = mac { + ia_text( + &mut b, + &mac.iter().map(|v| format!("{v:02X}")).collect::(), + ); + } + b.push(0xc1); + // The count includes the checksum and padding, not a separate header. + b[1] = (b.len() + 1).div_ceil(8) as u8; + area_finish(b) +} +impl State { + pub fn new(config: L1Config, queue: Arc>>) -> Self { + let mut records: [[Vec; 3]; 16] = + std::array::from_fn(|_| std::array::from_fn(|_| Vec::new())); + let chassis = area_finish(vec![0, 2, 0, 0xc2, b'N', b'A', 0xc2, b'N', b'A', 0xc1]); + for c in 0..2 { + records[c][0] = chassis.clone(); + records[c][1] = board(if c == 1 { Some(config.mac) } else { None }); + } + let powered = config.auto_power; + let port = config.node_position << 5; + Self { + config, + queue, + frame: Vec::new(), + in_frame: false, + escape: false, + overflow: false, + display: "L1 running".into(), + records, + actions: Vec::new(), + output: Vec::new(), + powered, + console: powered, + escaped: false, + command: Vec::new(), + port, + debug: 0, + partition: 0, + two_bit: 0, + reasons: [0; 2], + events: None, + power_down_pending: false, + } + } + pub fn reset(&mut self) { + // Controller settings survive the controller's local boot. The UART and + // packet parser reset independently of those persistent settings. + let persistent = (self.debug, self.partition, self.two_bit, self.reasons); + let events = self.events.filter(|&channel| channel >= 4); + let records = std::mem::replace( + &mut self.records, + std::array::from_fn(|_| std::array::from_fn(|_| Vec::new())), + ); + let queue = self.queue.clone(); + queue.lock().clear(); + *self = Self::new(self.config.clone(), queue); + (self.debug, self.partition, self.two_bit, self.reasons) = persistent; + self.events = events; + self.records = records; + } + fn emit(&mut self, payload: &[u8]) { + if payload.len() > 128 { + crate::dlog!(LogModule::L1, "L1 reply exceeds frame capacity"); + return; + } + let bytes = encode(payload); + let mut queue = self.queue.lock(); + if queue.len() + bytes.len() <= 4096 { + queue.extend(bytes); + } else { + crate::dlog!(LogModule::L1, "L1 receive queue full"); + } + } + fn reply(&mut self, channel: u8, code: i32, count: u8, fields: &[u8]) { + let mut p = vec![0x20 | channel]; + p.extend_from_slice(&code.to_be_bytes()); + p.push(count); + p.extend_from_slice(fields); + self.emit(&p); + } + pub fn parse(&mut self, mut byte: u8) { + if byte == 0x7e { + if self.in_frame + && !self.escape + && !self.overflow + && self.frame.len() >= 3 + && self.frame.iter().fold(0xffff, |v, b| fcs(v, *b)) == 0xf0b8 + { + let payload = self.frame[..self.frame.len() - 2].to_vec(); + self.request(&payload); + } + self.in_frame = true; + self.escape = false; + self.overflow = false; + self.frame.clear(); + return; + } + if !self.in_frame || self.overflow { + return; + } + if byte == 0x7d { + self.escape = true; + return; + } + if self.escape { + byte ^= 0x20; + self.escape = false; + } + if self.frame.len() == 258 { + self.overflow = true; + } else { + self.frame.push(byte); + } + } + fn eeprom(&mut self, channel: u8, a: &[&[u8]]) { + if a.len() < 4 || a[..4].iter().any(|v| v[0] != 0) { + self.reply(channel, -100, 0, &[]); + return; + } + let component = word(&a[0][1..]) as usize; + let area = word(&a[1][1..]); + let offset = word(&a[2][1..]) as usize; + let requested = word(&a[3][1..]) as usize; + if component >= 16 || self.records[component].iter().all(Vec::is_empty) { + self.reply(channel, -104, 0, &[]); + return; + } + if area > 4 { + self.reply(channel, -105, 0, &[]); + return; + } + let index = match area { + 1 => 0, + 2 => 1, + 4 => 2, + _ => { + self.reply(channel, -104, 0, &[]); + return; + } + }; + let record = &self.records[component][index]; + if record.is_empty() { + self.reply(channel, -104, 0, &[]); + return; + } + let checksum_len = if area == 4 { + record.len().min(64) + } else { + record.len() + }; + if record[..checksum_len] + .iter() + .fold(0u8, |s, b| s.wrapping_add(*b)) + != 0 + { + self.reply(channel, -107, 0, &[]); + return; + } + if offset >= record.len() { + self.reply(channel, -105, 0, &[]); + return; + } + let len = requested.min(record.len() - offset); + if len > 116 { + crate::dlog!(LogModule::L1, "L1 EEPROM reply exceeds frame capacity"); + self.reply(channel, -105, 0, &[]); + return; + } + let mut fields = ints(&[(record.len() - offset - len) as u32]); + if requested != 0 { + fields.push(0x80 | len as u8); + fields.extend_from_slice(&record[offset..offset + len]); + } + self.reply(channel, 0, if requested == 0 { 1 } else { 2 }, &fields); + } + fn request(&mut self, p: &[u8]) { + let channel = p[0] & 31; + if p[0] & 0xe0 == 0x40 { + self.port = (self.port & 0x60) | channel; + if self.console && self.powered { + self.output.extend_from_slice(&p[1..]); + } else { + crate::dlog!(LogModule::L1, "L1 console output while operator is in L1 mode"); + } + return; + } + if p[0] & 0xe0 != 0 || p.len() < 7 { + return; + } + let address = word(&p[1..]); + let cmd = u16::from_be_bytes([p[5], p[6]]); + let key = address >> 12; + if key != 0xffff && key != ((self.config.rack << 6) | self.config.bay) { + crate::dlog!(LogModule::L1, "L1 unsupported destination {address:#x}"); + self.emit(&[0x20 | channel, 255, 255, 255, 255, 0, 0, 0, 8]); + return; + } + let count = if p.len() == 7 { 0 } else { p[7] }; + let raw = if p.len() > 8 { &p[8..] } else { &[] }; + let task = address & 31; + if task == 5 || task == 21 { + if self.powered { + self.emit(&[ + ((address >> 5) & 31) as u8, + raw.get(4).copied().unwrap_or(0), + ]); + } else { + let mut field = Vec::new(); + text_field( + &mut field, + format!( + "WARNING: power on {:03}c{:02} appears off!", + self.config.rack, self.config.bay + ) + .as_bytes(), + ); + self.reply(channel, -1, 1, &field); + } + return; + } + if task != 6 && task != 3 && task != 18 { + crate::dlog!(LogModule::L1, "L1 unsupported task {task}"); + return; + } + let Some(a) = arguments(raw, count) else { + self.reply(channel, -100, 0, &[]); + return; + }; + if task == 3 { + if a.len() != 1 || a[0][0] != 1 { + self.reply(channel, -100, 0, &[]); + return; + } + let text = String::from_utf8_lossy(&a[0][1..a[0].len() - 1]); + let (code, output) = self.run_line(&text, false); + let mut fields = Vec::new(); + let mut n = 0; + for line in output.split("\r\n").filter(|s| !s.is_empty()) { + if fields.len() + line.len() + 2 > 121 { + break; + } + text_field(&mut fields, line.as_bytes()); + n += 1; + } + self.reply(channel, code, n, &fields); + return; + } + if task == 18 { + let length = a.iter().take(3).map(|v| v.len()).sum::().min(121); + self.reply(channel, cmd as i32, count.min(3), &raw[..length]); + return; + } + let mut values = Vec::new(); + let mut fields = Vec::new(); + let mut field_count = 0; + let mut error = 0; + match cmd { + 1 => values.push(self.debug), + 2 => values.push((self.config.rack << 6) | self.config.bay), + 3 => { + if self.config.type_code == 0 { + error = -104; + } else { + values.extend([ + (self.config.rack << 6) | self.config.bay, + self.config.type_code, + ]); + } + } + 4 => { + if let Some(serial) = &self.config.brick_serial { + text_field(&mut fields, serial.as_bytes()); + field_count = 1; + } else { + error = -104; + } + } + 5 => { + if let Some(rev) = self.config.revision { + values.extend(rev); + } else { + crate::dlog!(LogModule::L1, "L1 revision metadata not configured"); + error = -104; + } + } + 6 => { + self.eeprom(channel, &a); + return; + } + 7 => error = if a.is_empty() { -100 } else { -104 }, + // The system serial is the station address as twelve uppercase + // hexadecimal digits in address order, no separators or OUI + // added: the text the stock PROM converts straight back into the + // Ethernet address (the Fuel contract's MAC section). The + // uppercase case is the contract's choice for the model. + 8 => { + let mac = self + .config + .mac + .iter() + .map(|b| format!("{b:02X}")) + .collect::(); + text_field(&mut fields, mac.as_bytes()); + field_count = 1; + } + 9 => values.push(self.partition), + 0xa => values.extend([0, 0]), + 0xb => values.push(self.two_bit), + 0xc => values.extend([0; 8]), + 0xd => values.extend([0; 4]), + 0xe => values.push((self.port >> 5) as u32 & 3), + 0xf => values.extend(self.reasons), + 0x10 => values.extend([1, 1, 1, 1]), + 0x11 => { + error = -106; + values.push(1); + } + 0x1001 | 0x1002 | 0x1006 | 0x1007 | 0x1008 => { + if a.is_empty() { + error = -100; + } else if a[0][0] != 0 { + error = -105; + } else { + let v = word(&a[0][1..]); + match cmd { + 0x1001 => self.debug = v & 0xffff, + 0x1002 => self.port = ((self.port as u32 & 0x60) | v) as u8 & 0x7f, + 0x1006 => self.partition = v, + 0x1007 => self.events = Some(v as u8), + _ => self.two_bit = v & 3, + } + } + } + 0x1003 => {} + 0x1004 | 0x1005 => { + if a.is_empty() { + error = -100; + } else if a[0][0] != 1 { + error = -102; + } else { + self.display = a[0][1..a[0].len() - 1] + .iter() + .take(72) + .map(|b| { + if (0x20..=0x7e).contains(b) { + *b as char + } else { + '?' + } + }) + .collect(); + } + } + 0x1009 => { + if a.len() < 2 { + error = -100; + } else if a[0][0] != 0 || a[1][0] != 0 || word(&a[1][1..]) & 15 > 2 { + error = -105; + } + } + 0x100a => self.events = None, + 0x100b => { + if a.len() != 3 { + error = -100; + } else { + self.events = None; + } + } + 0x100c => { + if a.len() != 2 { + error = -100; + } + } + 0x2001 | 0x2005 | 0x2006 => { + self.reply(channel, 0, 0, &[]); + match cmd { + 0x2001 => self.reset_node(), + 0x2005 => self.power_down(true, false), + _ => { + self.power_down(false, false); + self.power_up(true); + } + } + return; + } + 0x2002..=0x2004 => error = -100, + 0x2007 | 0x2008 => error = -108, + 0x4000 => error = -104, + 0x4002 => { + self.reply(channel, 0, 0, &[]); + self.reboot(); + return; + } + _ => { + crate::dlog!(LogModule::L1, "L1 unsupported request {cmd:#x}"); + error = -101; + } + } + if !values.is_empty() { + field_count = values.len() as u8; + fields = ints(&values); + } + self.reply(channel, error, field_count, &fields); + } + fn reason(&mut self) { + self.reasons[1] = self.reasons[0]; + self.reasons[0] = 3; + } + fn reset_node(&mut self) { + if self.powered { + self.reason(); + self.display = "Reset done".into(); + self.actions.push(L1Event::Reset); + } + } + fn power_down(&mut self, shutdown: bool, confirm: bool) { + if self.powered { + if confirm && !self.power_down_pending { + self.power_down_pending = true; + self.event( + 0x8000, + 0, + "Power Down: issue 'pwr d' again to immediately power down.", + ); + return; + } + self.power_down_pending = false; + self.reason(); + self.powered = false; + self.display = "Powered Down".into(); + self.actions.push(L1Event::Power(false)); + if shutdown { + self.actions.push(L1Event::Shutdown); + } + } + } + fn power_up(&mut self, restart: bool) { + self.reason(); + self.powered = true; + self.power_down_pending = false; + self.port &= 0x60; + self.console = true; + self.display = "Powered Up".into(); + self.actions.push(L1Event::Power(true)); + if restart { + self.actions.push(L1Event::Reset); + } + } + fn reboot(&mut self) { + self.power_down_pending = false; + self.console = false; + self.escaped = false; + self.command.clear(); + self.frame.clear(); + self.in_frame = false; + self.port = self.config.node_position << 5; + self.display = "L1 running".into(); + if self.events.is_some_and(|c| c < 4) { + self.events = None; + } + } + pub fn event(&mut self, code: u32, number: u32, text: &str) { + let Some(channel) = self.events else { + return; + }; + let mut p = vec![0x40 | (channel & 31)]; + p.extend_from_slice( + &((self.config.rack << 18) | (self.config.bay << 12) | 4).to_be_bytes(), + ); + p.extend_from_slice(&code.to_be_bytes()); + p.push(2); + p.extend(ints(&[number])); + text_field( + &mut p, + format!( + "\r\n{:03}c{:02} ATTN: {text}\r\n", + self.config.rack, self.config.bay + ) + .as_bytes(), + ); + if p.len() <= 127 { + self.emit(&p); + } + } + fn run_line(&mut self, line: &str, interactive: bool) -> (i32, String) { + let words = line.split_whitespace().collect::>(); + let local = if words.first() == Some(&"*") { + &words[1..] + } else { + &words[..] + }; + match local { + ["pwr" | "power", "d" | "down"] => { + self.power_down(true, interactive); + (0, String::new()) + } + ["pwr" | "power", "u" | "up"] => { + if !self.powered { + self.power_up(true); + } + (0, String::new()) + } + ["reset"] => { + self.reset_node(); + (0, String::new()) + } + ["softreset"] => { + self.actions.push(L1Event::SoftReset); + self.display = "Sft rst done".into(); + (0, String::new()) + } + ["nmi"] => { + self.actions.push(L1Event::Nmi); + self.display = "NMI done".into(); + (0, String::new()) + } + ["reboot_l1"] => { + self.reboot(); + (0, String::new()) + } + ["l1"] => { + self.escaped = false; + (0, String::new()) + } + ["version"] => { + if let Some(r) = self.config.revision { + ( + 0, + format!("L1 Controller: Revision {}.{}.{}\r\n", r[0], r[1], r[2]), + ) + } else { + ( + -104, + "L1 firmware revision metadata not configured\r\n".into(), + ) + } + } + ["debug"] if words.len() == 1 => ( + 0, + format!("debug switches set to 0x{:04x}\n\r\n", self.debug), + ), + ["debug", v] if words.len() == 2 => { + let n = if let Some(hex) = v.strip_prefix("0x") { + u32::from_str_radix(hex, 16) + } else { + v.parse() + }; + match n { + Ok(v) => { + self.debug = v & 0xffff; + ( + 0, + format!("debug switches set to 0x{:04x}\n\r\n", self.debug), + ) + } + Err(_) => (-1200, "ERROR: command not found.\n\r\n".into()), + } + } + _ => { + crate::dlog!(LogModule::L1, "L1 unsupported operator command {line:?}"); + (-1, "ERROR: command not found.\r\n".into()) + } + } + } + fn prompt(&mut self) { + self.output.extend_from_slice( + format!("{:03}c{:02}-L1>", self.config.rack, self.config.bay).as_bytes(), + ); + } + pub fn console_input(&mut self, byte: u8) { + if self.console { + if byte == 0x14 { + self.console = false; + self.escaped = true; + self.output + .extend_from_slice(b"escaping to L1 system controller\r\n"); + self.prompt(); + } else if self.powered { + self.emit(&[self.port & 31, byte]); + } else { + crate::dlog!(LogModule::L1, "L1 console input while powered off"); + } + } else if byte == 4 { + self.command.clear(); + self.console = true; + } else if byte == b'\r' || byte == b'\n' { + let line = String::from_utf8_lossy(&self.command).into_owned(); + self.command.clear(); + let (_, output) = self.run_line(&line, true); + self.output.extend_from_slice(output.as_bytes()); + if self.escaped { + self.console = true; + } else { + self.prompt(); + } + } else if byte == 8 || byte == 127 { + self.command.pop(); + } else if self.command.len() < 255 { + self.command.push(byte); + } + } +} diff --git a/src/dev/l1/tests.rs b/src/dev/l1/tests.rs new file mode 100644 index 000000000..64d206fd5 --- /dev/null +++ b/src/dev/l1/tests.rs @@ -0,0 +1,381 @@ +// Cases follow tests/qtest/sn1-l1-test.c in jamesbraid/qemu-sgi-origami: +// uart, frames, identity, eeprom, errors and reset. + +use super::*; + +fn send(l1: &L1, bytes: &[u8]) { + for &b in bytes { + assert_eq!(l1.write64(0, b as u64), BUS_OK); + } +} +fn drain(l1: &L1) -> Vec { + let mut out = Vec::new(); + while l1.read64(0x28).data & 1 != 0 { + out.push(l1.read64(0).data as u8); + } + out +} +fn request(l1: &L1, command: u16, args: &[u8], count: u8) -> Vec { + let mut p = vec![ + 0x10, + 0x0f, + 0xff, + 0xf0, + 6, + (command >> 8) as u8, + command as u8, + count, + ]; + p.extend_from_slice(args); + send(l1, &encode(&p)); + drain(l1) +} +fn integer_reply(values: &[u32]) -> Vec { + let mut p = vec![0x30, 0, 0, 0, 0, values.len() as u8]; + p.extend(model::ints(values)); + encode(&p) +} + +/// The PROM accesses the L1 UART through byte-sized register operations. +#[test] +fn the_l1_uart_frames_bytes_and_prioritizes_interrupts() { + let l1 = L1::new(Default::default()).unwrap(); + assert_eq!(l1.read64(1).status, BUS_ERR); + assert_eq!(l1.read8(0).status, BUS_ERR); + assert_eq!(l1.read32(0).status, BUS_ERR); + assert_eq!(l1.read64(0x28).data, 0x60); + send(&l1, &[0x7e, 0x10, 0x0f, 0xff, 0xf0, 6, 0, 0x0e, 0, 0xf9, 0x72, 0x7e]); + assert_eq!(drain(&l1), [0x7e, 0x30, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0x30, 0x98, 0x7e]); + let (irq, line) = crate::irq::LevelProbe::line(); + l1.set_irq(line); + l1.write64(8, 3); + assert!(irq.level()); + assert_eq!(l1.read64(0x10).data, 2); + assert!(!irq.level()); + l1.write64(0x18, 0x80); + l1.write64(0, 42); + l1.write64(8, 7); + assert_eq!(l1.read64(0).data, 42); + assert_eq!(l1.read64(8).data, 7); + l1.write64(0x18, 3); + send(&l1, &encode(&[0x10, 0x0f, 0xff, 0xf0, 6, 0, 0x0e, 0])); + assert_eq!(l1.read64(0x10).data, 4); + drain(&l1); + assert_eq!(l1.read64(0x10).data, 2); + assert!(!irq.level()); +} + +/// The guest reads controller identity and checks the EEPROM record checksum. +#[test] +fn the_l1_returns_identity_and_validated_eeprom_records() { + let l1 = L1::new(Default::default()).unwrap(); + assert_eq!(request(&l1, 3, &[], 0), integer_reply(&[65, 65])); + let board = l1.record(1, 2).unwrap(); + assert_eq!(board.iter().fold(0u8, |s, x| s.wrapping_add(*x)), 0); + for text in [b"IP34".as_slice(), b"EMUN00", b"080069123456"] { + assert!(board.windows(text.len()).any(|x| x == text)); + } + let mut expected = vec![0x30, 0, 0, 0, 0, 2]; + expected.extend(model::ints(&[0])); + expected.push(0x80 | board.len() as u8); + expected.extend(board); + assert_eq!(request(&l1, 6, &model::ints(&[1, 2, 0, 116]), 4), encode(&expected)); + for (args, count, error) in [([2, 5, 0, 1], 4, 152), ([1, 5, 0, 1], 4, 151), ([1, 2, 0, 1], 3, 156)] { + assert_eq!(request(&l1, 6, &model::ints(&args), count), encode(&[0x30, 255, 255, 255, error, 0])); + } + l1.power_on(); + assert_eq!(request(&l1, 5, &[], 0), integer_reply(&[1, 48, 1])); +} + +/// A successful reply carrying one text field. +fn text_reply(text: &str) -> Vec { + let mut p = vec![0x30, 0, 0, 0, 0, 1]; + p.push(1); + p.extend_from_slice(text.as_bytes()); + p.push(0); + encode(&p) +} + +/// With no configuration the controller reports the labelled model defaults: +/// firmware 1.48.1 and the synthetic brick serial. +#[test] +fn the_l1_reports_the_default_revision_and_brick_serial() { + let l1 = L1::new(Default::default()).unwrap(); + l1.power_on(); + assert_eq!(request(&l1, 5, &[], 0), integer_reply(&[1, 48, 1])); + assert_eq!(request(&l1, 4, &[], 0), text_reply("EMUN00")); +} + +/// Both identity values can be replaced. +#[test] +fn the_l1_revision_and_brick_serial_can_be_overridden() { + let l1 = L1::new(L1Config { + revision: Some(parse_revision("2.13.40").unwrap()), + brick_serial: Some("EMUN07".into()), + ..Default::default() + }) + .unwrap(); + l1.power_on(); + assert_eq!(request(&l1, 5, &[], 0), integer_reply(&[2, 13, 40])); + assert_eq!(request(&l1, 4, &[], 0), text_reply("EMUN07")); + assert_eq!( + L1::new(L1Config { brick_serial: Some("x".repeat(38)), ..Default::default() }).err().as_deref(), + Some("L1 brick serial must fit a 37-byte text field") + ); +} + +/// The revision override is exactly three decimal numbers. +#[test] +fn the_l1_revision_text_is_three_decimal_parts() { + assert_eq!(parse_revision("1.48.1"), Ok([1, 48, 1])); + for bad in ["", "1", "1.48", "1.48.1.0", "1..1", "a.b.c", "1.48.-1", "1.48.+1", " 1.48.1", "1.48.99999999999"] { + assert!(parse_revision(bad).is_err(), "{bad:?}"); + } +} + +/// Request 0x0008 is the configured station address as twelve uppercase +/// hexadecimal digits in address order, the text the PROM turns back into the +/// Ethernet address. +#[test] +fn the_l1_system_serial_is_the_station_address_in_hex() { + let l1 = L1::new(Default::default()).unwrap(); + l1.power_on(); + assert_eq!(request(&l1, 8, &[], 0), text_reply("080069123456")); + let l1 = L1::new(L1Config { mac: [0x00, 0xA0, 0xc9, 0x1f, 0x02, 0xbe], ..Default::default() }).unwrap(); + l1.power_on(); + assert_eq!(request(&l1, 8, &[], 0), text_reply("00A0C91F02BE")); +} + +/// The L1 parser recovers after malformed frames and preserves escaped console text. +#[test] +fn the_l1_resynchronizes_after_bad_frames_and_preserves_escaped_text() { + let l1 = L1::new(Default::default()).unwrap(); + send(&l1, &[0x7e, 1, 2, 3, 0x7e]); + send(&l1, &vec![0x41; 259]); + send(&l1, &[0x7e]); + assert!(drain(&l1).is_empty()); + assert_eq!(request(&l1, 0x2000, &[], 0), encode(&[0x30, 255, 255, 255, 155, 0])); + send(&l1, &encode(&[0x10, 0x4f, 0xff, 0xf0, 6, 0, 3, 0])); + assert_eq!(drain(&l1), encode(&[0x30, 255, 255, 255, 255, 0, 0, 0, 8])); + let mut text = vec![1]; + text.extend_from_slice(b"hi~}\r\0"); + assert_eq!(request(&l1, 0x1004, &text, 1), integer_reply(&[])); + assert_eq!(l1.display_line(), "hi~}?"); + text.truncate(1); + text.extend_from_slice(b"abcdefghijkl\x01\x7f"); + text.extend(std::iter::repeat_n(b'z', 70)); + text.push(0); + assert_eq!(request(&l1, 0x1004, &text, 1), integer_reply(&[])); + assert_eq!(l1.display_line(), format!("abcdefghijkl??{}", "z".repeat(58))); +} + +/// The PROM may reset L1 during boot; controller settings and channel configuration must remain available. +#[test] +fn l1_reset_clears_uart_state_but_preserves_settings_and_channels() { + for channel in [3, 23] { + let l1 = L1::new(Default::default()).unwrap(); + let (irq, line) = crate::irq::LevelProbe::line(); + l1.set_irq(line); + for (set, value) in [(0x1001, 0x1234), (0x1006, 42), (0x1008, 2), (0x1007, channel)] { + request(&l1, set, &model::ints(&[value]), 1); + } + request(&l1, 0x2001, &[], 0); + let reasons = request(&l1, 0xf, &[], 0); + assert_eq!(reasons, integer_reply(&[3, 0])); + l1.write64(8, 1); + l1.send_event(0x8000, 7, "before reset"); + assert!(irq.level()); + let stale = encode(&[0x10, 0x0f, 0xff, 0xf0, 6, 0, 9, 0]); + send(&l1, &stale[..4]); + l1.write64(0x18, 0x80); + l1.write64(0, 42); + l1.write64(8, 7); + l1.power_on(); + assert!(!irq.level()); + assert_eq!(l1.read64(0x18).data, 0); + assert_eq!(l1.read64(8).data, 0); + assert!(drain(&l1).is_empty()); + l1.write64(0x18, 0x80); + // Ns16550 resets its divisor latch to 0x000c for 9600 baud. + assert_eq!(l1.read64(0).data, 0x0c); + assert_eq!(l1.read64(8).data, 0); + l1.write64(0x18, 0); + send(&l1, &stale[4..]); + assert!(drain(&l1).is_empty()); + for (get, value) in [(1, 0x1234), (9, 42), (0xb, 2)] { + assert_eq!(request(&l1, get, &[], 0), integer_reply(&[value])); + } + assert_eq!(request(&l1, 0xf, &[], 0), reasons); + l1.write64(8, 1); + l1.send_event(0x8000, 7, "after reset"); + assert_eq!(irq.level(), channel == 23); + let emitted = drain(&l1); + assert_eq!(emitted.is_empty(), channel == 3); + assert!(!irq.level()); + } +} + +/// A local console power-down request requires a second confirmation command. +#[test] +fn interactive_power_down_waits_for_a_second_command() { + let l1 = Arc::new(L1::new(Default::default()).unwrap()); + let actions = Arc::new(Mutex::new(Vec::new())); + let observed = actions.clone(); + let weak = Arc::downgrade(&l1); + l1.set_event_handler(Arc::new(move |event| { + assert!(!weak.upgrade().unwrap().node_powered()); + observed.lock().push(event); + })); + request(&l1, 0x1007, &model::ints(&[23]), 1); + l1.receive_console(&[0x14]); + l1.receive_console(b"pwr d\r"); + assert!(l1.node_powered()); + assert!(actions.lock().is_empty()); + let mut warning = vec![0x57, 0, 4, 0x10, 4, 0, 0, 0x80, 0, 2]; + warning.extend(model::ints(&[0])); + warning.push(1); + warning.extend_from_slice(b"\r\n001c01 ATTN: Power Down: issue 'pwr d' again to immediately power down.\r\n\0"); + assert_eq!(drain(&l1), encode(&warning)); + l1.power_on(); + l1.receive_console(&[0x14]); + l1.receive_console(b"pwr d\r"); + assert!(l1.node_powered(), "reset clears the first confirmation"); + drain(&l1); + l1.receive_console(&[0x14]); + l1.receive_console(b"power down\r"); + assert!(!l1.node_powered()); + assert_eq!(*actions.lock(), [L1Event::Power(false), L1Event::Shutdown]); +} + +/// Protocol power-down actions must finish before the L1 request reply. +#[test] +fn protocol_and_task_three_power_down_commands_are_immediate() { + let protocol = vec![0x10, 0x0f, 0xff, 0xf0, 6, 0x20, 5, 0]; + let mut task_three = vec![0x10, 0x0f, 0xff, 0xf0, 3, 0, 0, 1, 1]; + task_three.extend_from_slice(b"pwr d\0"); + for (case, packet) in [ + ("protocol command", protocol), + ("task three console", task_three), + ] { + let l1 = L1::new(Default::default()).unwrap(); + let actions = Arc::new(Mutex::new(Vec::new())); + let observed = actions.clone(); + l1.set_event_handler(Arc::new(move |event| observed.lock().push(event))); + send(&l1, &encode(&packet)); + assert_eq!(drain(&l1), integer_reply(&[]), "{case}"); + assert!(!l1.node_powered(), "{case}"); + assert_eq!( + *actions.lock(), + [L1Event::Power(false), L1Event::Shutdown], + "{case}" + ); + } +} + +/// A CPU may read the UART in its interrupt callback while the worker updates the line. +#[test] +fn l1_worker_irq_delivery_stays_ordered_across_reentrant_cpu_reads() { + use std::sync::mpsc; + struct BlockingIrq { + entered: mpsc::SyncSender<()>, + resume: std::sync::Mutex>, + blocked: AtomicBool, + levels: Mutex>, + device: std::sync::Weak, + } + impl crate::irq::InterruptController for BlockingIrq { + fn set_line(&self, _: u32, level: bool) { + if level && !self.blocked.swap(true, Ordering::AcqRel) { + self.entered.send(()).unwrap(); + self.resume.lock().unwrap().recv().unwrap(); + // A machine interrupt handler may inspect the UART immediately. + assert_eq!(self.device.upgrade().unwrap().read64(0x28).data, 0x60); + } + self.levels.lock().push(level); + } + } + let l1 = Arc::new(L1::new(Default::default()).unwrap()); + let backend = Arc::new(crate::dev::serial_host::CiSerialBackend::new()); + let (entered_tx, entered_rx) = mpsc::sync_channel(1); + let (resume_tx, resume_rx) = mpsc::sync_channel(1); + let irq = Arc::new(BlockingIrq { + entered: entered_tx, + resume: std::sync::Mutex::new(resume_rx), + blocked: AtomicBool::new(false), + levels: Mutex::new(Vec::new()), + device: Arc::downgrade(&l1), + }); + l1.set_irq(IrqLine::new(irq.clone(), 0)); + l1.set_serial_backend(backend.clone()); + l1.write64(8, 1); + l1.start(); + backend.push_host(b"x"); + let entered = entered_rx.recv_timeout(std::time::Duration::from_secs(2)); + if entered.is_ok() { + l1.write64(0x10, 2); + } + resume_tx.send(()).unwrap(); + l1.stop(); + entered.unwrap(); + assert_eq!(*irq.levels.lock(), [false, true, false]); +} + +/// Concurrent worker output and platform actions must preserve their order. +#[test] +fn concurrent_l1_outputs_and_actions_are_delivered_in_order() { + use std::sync::mpsc; + struct BlockingBackend { + entered: mpsc::SyncSender<()>, + resume: std::sync::Mutex>, + output: Mutex>, + } + impl SerialBackend for BlockingBackend { + fn send_byte(&self, byte: u8) { + if byte == b'A' { + self.entered.send(()).unwrap(); + self.resume.lock().unwrap().recv().unwrap(); + } + self.output.lock().push(byte); + } + fn recv_byte(&self) -> std::io::Result { + Err(std::io::ErrorKind::WouldBlock.into()) + } + } + let l1 = Arc::new(L1::new(Default::default()).unwrap()); + let (entered_tx, entered_rx) = mpsc::sync_channel(1); + let (resume_tx, resume_rx) = mpsc::sync_channel(1); + let backend = Arc::new(BlockingBackend { + entered: entered_tx, + resume: std::sync::Mutex::new(resume_rx), + output: Mutex::new(Vec::new()), + }); + l1.set_serial_backend(backend.clone()); + let actions = Arc::new(Mutex::new(Vec::new())); + let captured = actions.clone(); + let weak = Arc::downgrade(&l1); + l1.set_event_handler(Arc::new(move |action| { + // Reentrant register reads must not recursively dispatch notifications. + assert_eq!(weak.upgrade().unwrap().read64(0x28).data, 0x60); + captured.lock().push(action); + })); + { + let mut s = l1.state.lock(); + s.output.extend(b"AB"); + s.actions.push(L1Event::Reset); + } + let worker_device = l1.clone(); + let worker = std::thread::spawn(move || worker_device.flush()); + let entered = entered_rx.recv_timeout(std::time::Duration::from_secs(2)); + if entered.is_ok() { + let mut s = l1.state.lock(); + s.output.extend(b"CD"); + s.actions.push(L1Event::Nmi); + drop(s); + l1.flush(); + } + resume_tx.send(()).unwrap(); + worker.join().unwrap(); + entered.unwrap(); + assert_eq!(*backend.output.lock(), b"ABCD"); + assert_eq!(*actions.lock(), [L1Event::Reset, L1Event::Nmi]); +} diff --git a/src/dev/mod.rs b/src/dev/mod.rs index db1d9f738..714490145 100644 --- a/src/dev/mod.rs +++ b/src/dev/mod.rs @@ -71,5 +71,6 @@ pub mod bedrock_pi; pub mod dp83843; pub mod ics1890; pub mod isp12160; +pub mod l1; pub mod timekeeper; pub mod xbridge; diff --git a/src/devlog.rs b/src/devlog.rs index b9ee4f6fa..406858d4f 100644 --- a/src/devlog.rs +++ b/src/devlog.rs @@ -56,6 +56,7 @@ pub enum LogModule { /// model does not support. Odyssey = 28, Bedrock = 29, + L1 = 30, } // Mask bits shared by L1i / L1d / L2c @@ -64,7 +65,7 @@ pub const CACHE_LOG_MISS: u32 = 0x2; // log misses and fills pub const CACHE_LOG_OP: u32 = 0x4; // log CACHE instruction ops impl LogModule { - pub const COUNT: usize = 30; + pub const COUNT: usize = 31; pub fn from_str(s: &str) -> Option { match s { @@ -98,6 +99,7 @@ impl LogModule { "rad1" => Some(Self::Rad1), "odyssey" => Some(Self::Odyssey), "bedrock" => Some(Self::Bedrock), + "l1" => Some(Self::L1), _ => None, } } @@ -134,6 +136,7 @@ impl LogModule { Self::Rad1 => "rad1", Self::Odyssey => "odyssey", Self::Bedrock => "bedrock", + Self::L1 => "l1", } } @@ -144,7 +147,7 @@ impl LogModule { Self::Vino, Self::Dcb, Self::Vc2, Self::Cmap, Self::Xmap, Self::Bt445, Self::Scc, Self::Ps2, Self::Rtc, Self::Eeprom, Self::L1i, Self::L1d, Self::L2c, Self::Ultra, Self::Scsi1, - Self::Nveeprom, Self::Gr2, Self::Rad1, Self::Odyssey, Self::Bedrock, + Self::Nveeprom, Self::Gr2, Self::Rad1, Self::Odyssey, Self::Bedrock, Self::L1, ] } @@ -287,6 +290,7 @@ impl DevLog { ModuleLog::new(), // Rad1 ModuleLog::new(), // Odyssey ModuleLog::new(), // Bedrock + ModuleLog::new(), // L1 ], } } From d11a99a1f754f8056305050f75c51801dd9b2d33 Mon Sep 17 00:00:00 2001 From: James Braid Date: Fri, 30 Oct 2026 08:13:37 +0000 Subject: [PATCH 26/31] ohci: add the Lucent USB host controller and HID boot devices Fuel enumerates a Lucent OHCI controller on PCI. Provide its registers and descriptor scheduling through the USB function interface while leaving devices unattached in the machine configuration. The frame counter follows the machine clock the board hands the controller, and every register read and write first delivers the frame boundaries that clock has passed. Advancing it only from the board's poll, which runs between CPU batches, left HcFmNumber at zero after the guest's own 1 ms wait had expired, and the Fuel PROM reported that SOF did not start within 2 ms. The sync holds one lock from reading the clock to applying the frames, so a timer thread cannot leave a concurrent reader behind it. Entering UsbOperational from Reset or Suspend re-anchors the frame: it loads HcFmRemaining from HcFmInterval, toggle included, and increments HcFmNumber, with the first SOF one frame later (OpenHCI 1.0a 6.2.1, 7.3.2, 7.3.3). The tests cover that entry with frame numbers at and around the overflow flag. Add two low-speed USB functions for the OHCI root ports: a HID boot keyboard and a HID boot mouse, each with one configuration, one boot interface and one interrupt-IN endpoint. They answer the standard requests (descriptors, SET_ADDRESS taking effect after its status stage, SET_CONFIGURATION, status and interface queries) and the HID class requests (GET/SET_REPORT, GET/SET_IDLE, GET/SET_PROTOCOL), and report host keys and pointer moves as boot reports. Reports queue one per change so a key tapped between two polls still shows both edges. Written from the USB 1.1 and HID 1.11 specifications (boot descriptors from appendix B) and the HID Usage Tables, with no code from another emulator. The wheel, remote wakeup and the host layout translation that the PS/2 path has are left out. Ohci::attach loses its test-only gate so a machine can plug the devices in. --- debug.md | 6 +- src/dev/mod.rs | 2 + src/dev/ohci.rs | 589 +++++++++++++++++++++++++++++++++++++ src/dev/ohci/engine.rs | 223 ++++++++++++++ src/dev/ohci/schedule.rs | 98 +++++++ src/dev/ohci/tests.rs | 554 +++++++++++++++++++++++++++++++++++ src/dev/ps2.rs | 70 ++++- src/dev/usb_hid.rs | 609 +++++++++++++++++++++++++++++++++++++++ src/dev/usb_hid/tests.rs | 448 ++++++++++++++++++++++++++++ src/devlog.rs | 8 +- 10 files changed, 2600 insertions(+), 7 deletions(-) create mode 100644 src/dev/ohci.rs create mode 100644 src/dev/ohci/engine.rs create mode 100644 src/dev/ohci/schedule.rs create mode 100644 src/dev/ohci/tests.rs create mode 100644 src/dev/usb_hid.rs create mode 100644 src/dev/usb_hid/tests.rs diff --git a/debug.md b/debug.md index 4cfa05cf2..f444d05c4 100644 --- a/debug.md +++ b/debug.md @@ -541,11 +541,11 @@ Per-module logging; all of it needs a developer build to produce output. | `log file ` | Send a module's log to a file | Modules: `net hpc3 seeq hal2 mc rex3 mips ioc scsi pdma vino dcb vc2 cmap -xmap bt445 scc ps2 rtc eeprom l1i l1d l2c bedrock l1`. Mask categories: `pdma` — +xmap bt445 scc ps2 rtc eeprom l1i l1d l2c bedrock l1 usb`. Mask categories: `pdma` — `hal enet scsi`; `mips` — `insn tlb mem fpu cp0`; `l1i`/`l1d`/`l2c` — `hit miss op`; every module also takes `on`/`all`, `off`/`none` or a hex mask. -`bedrock` covers the Fuel's devices on its Bedrock hub, and `l1` its system -controller. +`bedrock` covers the Fuel's devices on its Bedrock hub, `l1` its system +controller and `usb` its USB host controller. `IRIS_DEBUG_LOG` or `[debug] debug_log` set the same thing at startup. ### Performance and locks diff --git a/src/dev/mod.rs b/src/dev/mod.rs index 714490145..33c4182d3 100644 --- a/src/dev/mod.rs +++ b/src/dev/mod.rs @@ -56,6 +56,7 @@ pub mod seeq8003; pub mod testdev; pub mod timer; pub mod ultra64; +pub mod usb_hid; pub mod vino; pub mod virtio; pub mod wd33c93a; @@ -72,5 +73,6 @@ pub mod dp83843; pub mod ics1890; pub mod isp12160; pub mod l1; +pub mod ohci; pub mod timekeeper; pub mod xbridge; diff --git a/src/dev/ohci.rs b/src/dev/ohci.rs new file mode 100644 index 000000000..2625c5b0f --- /dev/null +++ b/src/dev/ohci.rs @@ -0,0 +1,589 @@ +// Derived from MAME's xbox_usb.cpp and xbox_usb.h (the OHCI host controller), +// BSD-3-Clause, copyright Samuele Zannoli. + +//! The Lucent USS-312 USB controller: two root ports, OHCI registers and +//! descriptor processing through the PCI bridge's DMA interface. +//! +//! Functions plug into a root port with `attach`: the Fuel fits a keyboard and +//! a mouse (`usb_hid`), and the tests drive the descriptor and packet +//! scheduling engine through a test-only function. Isochronous +//! traffic, suspend/resume, external hubs, USB storage and legacy keyboard/SMI +//! routing are unsupported. Scheduling uses synthetic work units rather than +//! wire time. Register behavior follows OpenHCI 1.0a and the USS-312 revision 5 +//! data sheet. +//! +//! Ported from src/mame/shared/xbox_usb.cpp and xbox_usb.h in MAME, +//! revision 2ee9b1e30299b122b131cab0bd7912abc2b095ba. + +use crate::dev::heart::{host_clock, Clock}; +use crate::irq::IrqLine; +use crate::pci::{Bar, PciBusDevice, PciConfig, PciConfigPolicy, PciDma, PCI_INTA}; +use crate::traits::{Device, Resettable}; +use parking_lot::Mutex; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Arc, OnceLock}; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +#[repr(u32)] +pub enum Pid { + Setup = 0, + Out = 1, + In = 2, +} +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct UsbPacket { + pub endpoint: u8, + pub pid: Pid, + pub toggle: bool, + /// IN requests name capacity; OUT and SETUP carry their bytes. + pub capacity: usize, + pub data: Vec, + pub time_ms: u64, +} +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum UsbOutcome { + Ack(Vec), + Nak, + Stall, + NoResponse, + CrcError, + BitStuffError, +} +pub trait UsbFunction: Send + Sync { + fn reset(&self); + fn address(&self) -> u8; + fn low_speed(&self) -> bool { + false + } + fn transact(&self, packet: UsbPacket) -> UsbOutcome; +} + +mod engine; +mod schedule; + +const EVENTS: u32 = 0x4000_007f; +const WDH: u32 = 2; +const UE: u32 = 16; +const RHSC: u32 = 64; +const PPS: u32 = 0x100; +const CHANGES: u32 = 0x1f0000; +#[derive(Clone)] +struct State { + r: [u32; 23], + ports: [Option>; 2], + low_speed: [bool; 2], + reset_ms: [u8; 2], + done_delay: u8, + now_ms: u64, + active_fi: u32, + bit_credit: u64, +} +impl State { + fn new() -> Self { + let mut r = [0; 23]; + r[0] = 0x110; + r[13] = 0x2edf; + r[17] = 0x628; + r[18] = 0x1000_0902; + r[19] = 0x60000; + Self { + r, + ports: [None, None], + low_speed: [false; 2], + reset_ms: [0; 2], + done_delay: 7, + now_ms: 0, + active_fi: 0x2edf, + bit_credit: 0, + } + } + fn soft_reset(&mut self) { + let control = (self.r[1] & 0x300) | 0xc0; + let keep = [self.r[18], self.r[19], self.r[20], self.r[21], self.r[22]]; + let ports = self.ports.clone(); + let low_speed = self.low_speed; + let resets = self.reset_ms; + let time = self.now_ms; + *self = Self::new(); + self.r[1] = control; + self.r[18..23].copy_from_slice(&keep); + self.ports = ports; + self.low_speed = low_speed; + self.reset_ms = resets; + self.now_ms = time; + } + fn irq(&self) -> bool { + self.r[1] & 0x100 == 0 && self.r[4] & 0x80000000 != 0 && self.r[3] & self.r[4] & EVENTS != 0 + } + fn powered(&self, p: usize) -> bool { + self.r[18] & 0x200 != 0 || self.r[21 + p] & PPS != 0 + } + fn individual(&self, p: usize) -> bool { + self.r[18] & 0x100 != 0 && self.r[19] & (1 << (17 + p)) != 0 + } + fn power(&mut self, p: usize, on: bool) { + if self.r[18] & 0x200 != 0 || on { + self.r[21 + p] |= PPS; + if self.ports[p].is_some() && self.r[21 + p] & 1 == 0 { + self.r[21 + p] |= 1 | 0x10000 | if self.low_speed[p] { 0x200 } else { 0 }; + self.r[3] |= RHSC; + } + } else { + let previous = self.r[21 + p]; + self.r[21 + p] &= !(PPS | 0x17); + if previous & 1 != 0 { + self.r[21 + p] |= 0x10000; + } + if previous & 2 != 0 { + self.r[21 + p] |= 0x20000; + } + if previous & 3 != 0 { + self.r[3] |= RHSC; + } + self.reset_ms[p] = 0; + } + } +} + +pub struct Ohci { + config: PciConfig, + state: Mutex, + pass: Mutex<()>, + dma: Mutex>>, + irq: Mutex, + running: AtomicBool, + /// Source of the frame count: the machine's own clock, so the guest's + /// idea of a millisecond and the controller's agree. + time: OnceLock, + /// `time` reading at the last frame boundary delivered; `None` while stopped. + clock: Mutex>, +} +impl Ohci { + pub fn new() -> Arc { + let config = PciConfig::new_with_policy( + 0x11c1, + 0x5802, + 0x0c0310, + 0x10, + [ + Bar::Mem(0x1000), + Bar::Unused, + Bar::Unused, + Bar::Unused, + Bar::Unused, + Bar::Unused, + ], + PCI_INTA, + PciConfigPolicy { + status: 0x0200, + status_w1c_mask: 0x8000, + command_mask: 0x0006, + latency_mask: 0, + cache_line_mask: 0, + registers: &[], + }, + ); + config.set_subsystem_identity(0xffff_ffff); + Arc::new(Self { + config, + state: Mutex::new(State::new()), + pass: Mutex::new(()), + dma: Mutex::new(None), + irq: Mutex::new(IrqLine::unconnected()), + running: AtomicBool::new(false), + time: OnceLock::new(), + clock: Mutex::new(None), + }) + } + /// Use `clock` for frame timing. Call before `start`. + pub fn set_clock(&self, clock: Clock) { + assert!( + !self.running.load(Ordering::Acquire) && self.time.set(clock).is_ok(), + "connect the OHCI clock once, before start" + ); + } + fn now(&self) -> u64 { + (self.time.get_or_init(host_clock))() + } + /// Deliver every frame boundary the clock has passed. Every register + /// access calls this, so a read sees the frame count the guest's own + /// clock implies, however late the machine's poll or timer ran. + pub fn sync_clock(&self) { + // Held through delivery: a second caller must not find the boundary + // consumed and read the frame count before the first has applied it. + let mut clock = self.clock.lock(); + let Some(at) = clock.as_mut() else { return }; + let now = self.now(); + let elapsed = now.saturating_sub(*at) / 1_000_000; + *at += elapsed * 1_000_000; + self.advance_frames(elapsed); + } + fn drive_irq(&self) { + loop { + let level = self.state.lock().irq(); + let line = self.irq.lock().clone(); + line.set(level); + // A callback may acknowledge the interrupt or another writer may + // change it while delivery is outside the state lock. + if self.state.lock().irq() == level { + break; + } + } + } + pub fn attach(&self, port: usize, function: Arc) -> Result<(), String> { + if port >= 2 { + return Err("OHCI has two ports".into()); + } + if self.state.lock().ports[port].is_some() { + return Err("OHCI port occupied".into()); + } + let low_speed = function.low_speed(); + function.reset(); + { + let _pass = self.pass.lock(); + let mut s = self.state.lock(); + if s.ports[port].is_some() { + return Err("OHCI port occupied".into()); + } + if s.powered(port) { + s.r[21 + port] |= 1 | 0x10000; + } + s.low_speed[port] = low_speed; + if low_speed { + s.r[21 + port] |= 0x200; + } + s.ports[port] = Some(function); + s.r[3] |= RHSC; + } + self.drive_irq(); + Ok(()) + } + #[cfg(test)] + pub fn detach(&self, port: usize) -> Result<(), String> { + if port >= 2 { + return Err("OHCI has two ports".into()); + } + { + let _pass = self.pass.lock(); + let mut s = self.state.lock(); + let enabled = s.r[21 + port] & 2; + s.ports[port] = None; + s.low_speed[port] = false; + s.reset_ms[port] = 0; + s.r[21 + port] = + (s.r[21 + port] & PPS) | 0x10000 | if enabled != 0 { 0x20000 } else { 0 }; + s.r[3] |= RHSC; + } + self.drive_irq(); + Ok(()) + } + /// Deterministic millisecond ticks; `sync_clock` delivers the same ticks from the clock. + pub fn advance_frames(&self, count: u64) { + for _ in 0..count { + if !self.running.load(Ordering::Acquire) { + break; + } + { + let _pass = self.pass.lock(); + if !self.running.load(Ordering::Acquire) { + break; + } + let mut s = self.state.lock().clone(); + self.frame(&mut s); + *self.state.lock() = s; + } + self.drive_irq(); + } + } + fn frame(&self, s: &mut State) { + s.now_ms = s.now_ms.wrapping_add(1); + for p in 0..2 { + if s.reset_ms[p] != 0 { + s.reset_ms[p] -= 1; + if s.reset_ms[p] == 0 { + s.r[21 + p] = (s.r[21 + p] & !16) | 2 | (1 << 20); + s.r[3] |= RHSC; + } + } + } + if s.r[1] & 0xc0 != 0x80 || s.r[3] & UE != 0 { + return; + } + s.bit_credit += 12000; + while s.bit_credit >= u64::from((s.active_fi & 0x3fff) + 1) { + s.bit_credit -= u64::from((s.active_fi & 0x3fff) + 1); + s.active_fi = s.r[13]; + if self.schedule(s).is_err() { + s.r[3] |= UE; + s.bit_credit = 0; + self.config.set_status_bits(0x8000); + break; + } + } + } +} +impl PciBusDevice for Ohci { + fn config(&self) -> &PciConfig { + &self.config + } + fn connect_dma(&self, dma: Arc) { + *self.dma.lock() = Some(dma); + } + fn connect_interrupt(&self, line: IrqLine) { + *self.irq.lock() = line; + self.drive_irq(); + } + fn bar_read(&self, bar: usize, off: u32, width: usize) -> u32 { + if bar != 0 || width != 4 || off & 3 != 0 || off > 0x58 { + crate::dlog!( + crate::devlog::LogModule::Usb, + "ohci unsupported read at register offset {off:#x}, width {width}" + ); + return 0; + } + self.sync_clock(); + let s = self.state.lock(); + let i = (off / 4) as usize; + if i == 5 { + s.r[4] + } else if i == 21 || i == 22 { + s.r[i] | if s.r[18] & 0x200 != 0 { PPS } else { 0 } + } else { + s.r[i] + } + } + fn bar_write(&self, bar: usize, off: u32, val: u32, width: usize) { + if bar != 0 || width != 4 || off & 3 != 0 || off > 0x58 { + crate::dlog!( + crate::devlog::LogModule::Usb, + "ohci unsupported write at register offset {off:#x}, width {width}" + ); + return; + } + self.sync_clock(); + // Taken before `pass`, as `sync_clock` orders them. Only a control + // write can restart the frame phase. + let mut anchor = (off == 4).then(|| self.clock.lock()); + let mut reset = None; + { + let _pass = self.pass.lock(); + let mut s = self.state.lock(); + match off / 4 { + 1 => { + let old = s.r[1]; + s.r[1] = val & 0x7ff; + if val & 0xc0 == 0x80 && old & 0xc0 != 0x80 { + // OpenHCI 1.0a 6.2.1, 7.3.2, 7.3.3: entering + // UsbOperational loads HcFmRemaining from the frame + // interval and increments HcFmNumber, with no SOF at + // the load; the first follows one frame later. The + // free-running clock's phase therefore restarts here. + s.bit_credit = 0; + s.active_fi = s.r[13]; + s.r[14] = (s.r[13] & 0x8000_0000) | (s.r[13] & 0x3fff); + let old_frame = s.r[15] as u16; + let frame = old_frame.wrapping_add(1); + s.r[15] = u32::from(frame); + if (old_frame ^ frame) & 0x8000 != 0 { + s.r[3] |= 32; + } + if let Some(Some(at)) = anchor.as_deref_mut() { + *at = self.now(); + } + } + if val & 0xc0 == 0 && old & 0xc0 != 0 { + s.r[18] = 0x10000902; + for p in 0..2 { + s.power(p, false); + } + } + if val & 0xc0 == 0x40 || val & 0xc0 == 0xc0 { + crate::dlog!( + crate::devlog::LogModule::Usb, + "ohci suspend/resume timing unsupported at control register offset 0x04" + ); + } + } + 2 => { + if val & 1 != 0 { + s.soft_reset(); + } else { + s.r[2] |= val & 6; + if val & 8 != 0 { + s.r[3] |= 1 << 30; + } + } + } + 3 => s.r[3] &= !(val & EVENTS), + 4 => s.r[4] |= val & (EVENTS | 0x80000000), + 5 => s.r[4] &= !(val & (EVENTS | 0x80000000)), + 6 => s.r[6] = val & !255, + 7..=11 => s.r[(off / 4) as usize] = val & !15, + 13 => s.r[13] = val & 0xffff3fff, + 16 => s.r[16] = val & 0x3fff, + 17 => s.r[17] = val & 0xfff, + 18 => { + s.r[18] = (s.r[18] & !0xff001b00) | (val & 0xff001b00); + if val & 0x200 != 0 { + for p in 0..2 { + s.power(p, true); + } + } + } + 19 => s.r[19] = val & 0x60006, + 20 => { + if val & (1 << 15) != 0 { + s.r[20] |= 1 << 15; + } + if val & (1 << 31) != 0 { + s.r[20] &= !(1 << 15); + } + if val & (1 << 17) != 0 { + s.r[20] &= !(1 << 17); + } + if s.r[18] & 0x200 == 0 { + for p in 0..2 { + if !s.individual(p) { + if val & 1 != 0 { + s.power(p, false); + } + if val & (1 << 16) != 0 { + s.power(p, true); + } + } + } + } + } + 21 | 22 => { + let p = (off / 4 - 21) as usize; + let i = 21 + p; + s.r[i] &= !(val & CHANGES); + if val & 1 != 0 { + s.r[i] &= !2; + } + if val & 2 != 0 { + if s.r[i] & 1 != 0 && s.powered(p) { + s.r[i] |= 2; + } else { + s.r[i] |= 1 << 16; + } + } + if val & 12 != 0 { + crate::dlog!( + crate::devlog::LogModule::Usb, + "ohci port suspend/resume unsupported at port register offset {:#x}", + 0x54 + p * 4 + ); + } + if val & 16 != 0 { + if s.r[i] & 1 != 0 && s.powered(p) { + s.r[i] |= 16; + s.r[i] &= !2; + s.reset_ms[p] = 10; + reset = s.ports[p].clone(); + } else { + s.r[i] |= 1 << 16; + } + } + if s.individual(p) && s.r[18] & 0x200 == 0 { + if val & PPS != 0 { + s.power(p, true); + } + if val & 0x200 != 0 { + s.power(p, false); + } + } + if s.r[i] & CHANGES != 0 { + s.r[3] |= RHSC; + } + } + _ => { + crate::dlog!( + crate::devlog::LogModule::Usb, + "ohci write to read-only register offset {off:#x}" + ); + } + } + } + drop(anchor); + if let Some(f) = reset { + f.reset(); + } + self.drive_irq(); + } +} +impl Resettable for Ohci { + fn power_on(&self) { + let ports = { + let _pass = self.pass.lock(); + let mut s = self.state.lock(); + let ports = s.ports.clone(); + let low_speed = s.low_speed; + *s = State::new(); + s.ports = ports.clone(); + s.low_speed = low_speed; + self.config.reset(); + ports + }; + for f in ports.into_iter().flatten() { + f.reset(); + } + self.drive_irq(); + } +} +impl Device for Ohci { + fn start(&self) { + if self.running.swap(true, Ordering::AcqRel) { + return; + } + let now = self.now(); + *self.clock.lock() = Some(now); + } + fn stop(&self) { + self.sync_clock(); + self.running.store(false, Ordering::Release); + *self.clock.lock() = None; + drop(self.pass.lock()); + } + fn is_running(&self) -> bool { + self.running.load(Ordering::Acquire) + } + fn step(&self, _: u64) {} + fn get_clock(&self) -> u64 { + 0 + } +} + +#[cfg(test)] +mod register_tests { + use super::*; + use crate::pci::PciBusDevice; + use crate::traits::Resettable; + + #[test] + fn lucent_identity_and_reset_contract() { + let hc = Ohci::new(); + assert_eq!(hc.config().read(0), 0x5802_11c1); + assert_eq!(hc.config().read(2), 0x0c03_1010); + assert_eq!(hc.config().read(11), 0xffff_ffff); + assert_eq!(hc.config().read(1), 0x0200_0000); + assert_eq!(hc.bar_read(0, 0, 4), 0x110); + assert_eq!(hc.bar_read(0, 0x10, 4), 0); + assert_eq!(hc.bar_read(0, 0x34, 4), 0x2edf); + assert_eq!(hc.bar_read(0, 0x48, 4), 0x1000_0902); + assert_eq!(hc.bar_read(0, 0x4c, 4), 0x0006_0000); + for p in [0x54, 0x58] { + assert_eq!(hc.bar_read(0, p, 4), 0); + } + hc.bar_write(0, 0x10, 0xffff_ffff, 4); + hc.bar_write(0, 0x08, 1, 4); + assert_eq!(hc.bar_read(0, 0x10, 4), 0); + assert_eq!(hc.bar_read(0, 0x04, 4) & 0xc0, 0xc0); + hc.power_on(); + assert_eq!(hc.bar_read(0, 0x04, 4), 0); + } +} + +#[cfg(test)] +#[path = "ohci/tests.rs"] +mod tests; diff --git a/src/dev/ohci/engine.rs b/src/dev/ohci/engine.rs new file mode 100644 index 000000000..7b76cc646 --- /dev/null +++ b/src/dev/ohci/engine.rs @@ -0,0 +1,223 @@ +//! General TD packet processing; guest DMA byte slices are the endianness edge. +use super::*; + +fn read_words(dma: &dyn PciDma, at: u32) -> Result<[u32; 4], ()> { + let mut bytes = [0; 16]; + if !dma.dma_read(at as u64, &mut bytes).is_ok() { + return Err(()); + } + Ok(std::array::from_fn(|i| { + u32::from_le_bytes(bytes[i * 4..i * 4 + 4].try_into().unwrap()) + })) +} +fn write_words(dma: &dyn PciDma, at: u32, words: &[u32]) -> Result<(), ()> { + let bytes: Vec = words.iter().flat_map(|x| x.to_le_bytes()).collect(); + if dma.dma_write(at as u64, &bytes).is_ok() { + Ok(()) + } else { + Err(()) + } +} +fn remaining(cbp: u32, be: u32) -> Result { + if cbp == 0 { + return Ok(0); + } + if cbp & !0xfff == be & !0xfff { + be.checked_sub(cbp).map(|n| n as usize + 1).ok_or(()) + } else { + Ok(0x1000 - (cbp as usize & 0xfff) + (be as usize & 0xfff) + 1) + } +} +fn advance(cbp: u32, be: u32, n: usize) -> u32 { + let first = 0x1000 - (cbp as usize & 0xfff); + if n < first { + cbp.wrapping_add(n as u32) + } else { + (be & !0xfff).wrapping_add((n - first) as u32) + } +} +fn payload_read(dma: &dyn PciDma, cbp: u32, be: u32, len: usize) -> Result, ()> { + let mut bytes = vec![0; len]; + let first = len.min(0x1000 - (cbp as usize & 0xfff)); + if first != 0 && !dma.dma_read(cbp as u64, &mut bytes[..first]).is_ok() { + return Err(()); + } + if len > first + && !dma + .dma_read((be & !0xfff) as u64, &mut bytes[first..]) + .is_ok() + { + return Err(()); + } + Ok(bytes) +} +fn payload_write(dma: &dyn PciDma, cbp: u32, be: u32, bytes: &[u8]) -> Result<(), ()> { + let first = bytes.len().min(0x1000 - (cbp as usize & 0xfff)); + if first != 0 && !dma.dma_write(cbp as u64, &bytes[..first]).is_ok() { + return Err(()); + } + if bytes.len() > first && !dma.dma_write((be & !0xfff) as u64, &bytes[first..]).is_ok() { + return Err(()); + } + Ok(()) +} +impl Ohci { + /// Returns the next ED and whether this ED still holds queued general work. + pub(super) fn endpoint( + &self, + s: &mut State, + dma: &dyn PciDma, + at: u32, + ) -> Result<(u32, bool, bool), ()> { + let mut ed = read_words(dma, at)?; + let next = ed[3] & !15; + if ed[0] & (1 << 15) != 0 { + crate::dlog!(crate::devlog::LogModule::Usb, "ohci isochronous ED unsupported at guest address {at:#x}"); + return Ok((next, false, false)); + } + if ed[0] & (1 << 14) != 0 || ed[2] & 1 != 0 || ed[2] & !15 == ed[1] & !15 { + return Ok((next, false, false)); + } + let td_at = ed[2] & !15; + let mut td = read_words(dma, td_at)?; + let mps = ((ed[0] >> 16) & 0x7ff) as usize; + if mps == 0 || mps > 1023 { + crate::dlog!(crate::devlog::LogModule::Usb, "ohci invalid maximum packet size {mps} at ED address {at:#x}"); + return Ok((next, false, false)); + } + let Ok(n) = remaining(td[1], td[3]) else { + crate::dlog!(crate::devlog::LogModule::Usb, "ohci malformed TD buffer range at ED address {at:#x}"); + return Ok((next, false, false)); + }; + let direction = match (ed[0] >> 11) & 3 { + 1 => 1, + 2 => 2, + _ => (td[0] >> 19) & 3, + }; + let pid = match direction { + 0 => Pid::Setup, + 1 => Pid::Out, + 2 => Pid::In, + _ => { + crate::dlog!(crate::devlog::LogModule::Usb, "ohci reserved PID at ED address {at:#x}"); + return Ok((next, false, false)); + } + }; + let length = n.min(mps); + let toggle = if td[0] & (1 << 25) != 0 { + td[0] & (1 << 24) != 0 + } else { + ed[2] & 2 != 0 + }; + let outgoing = if pid != Pid::In { + payload_read(dma, td[1], td[3], length)? + } else { + vec![] + }; + let address = (ed[0] & 127) as u8; + let mut functions = (0..2) + .filter(|&p| s.powered(p) && s.r[21 + p] & 0x16 == 2) + .filter_map(|p| s.ports[p].clone()) + .filter(|f| f.address() == address); + // Multiple matching functions are an unsupported topology. Do not choose + // a recipient by port order or claim to model electrical collisions. + let first = functions.next(); + let function = if functions.next().is_none() { + first + } else { + None + }; + let reply = if let Some(f) = function { + f.transact(UsbPacket { + endpoint: ((ed[0] >> 7) & 15) as u8, + pid, + toggle, + capacity: if pid == Pid::In { mps } else { length }, + data: outgoing, + time_ms: s.now_ms, + }) + } else { + UsbOutcome::NoResponse + }; + let mut retire = false; + let mut cc = 0; + let mut halt = false; + match reply { + UsbOutcome::Nak => return Ok((next, true, true)), + UsbOutcome::Stall => { + cc = 4; + halt = true; + retire = true; + } + UsbOutcome::NoResponse | UsbOutcome::CrcError | UsbOutcome::BitStuffError => { + cc = match reply { + UsbOutcome::CrcError => 1, + UsbOutcome::BitStuffError => 2, + _ => 5, + }; + let ec = ((td[0] >> 26) & 3) + 1; + td[0] = (td[0] & !(3 << 26)) | ((ec.min(3)) << 26); + if ec >= 3 { + retire = true; + halt = true; + } + } + UsbOutcome::Ack(bytes) => { + let actual = if pid == Pid::In { bytes.len() } else { length }; + if pid == Pid::In { + payload_write(dma, td[1], td[3], &bytes[..actual.min(n).min(mps)])?; + } + td[0] = (td[0] & !((3 << 24) | (3 << 26))) | ((2 | u32::from(!toggle)) << 24); + if actual > length { + cc = 8; + halt = true; + retire = true; + } else { + td[1] = if actual == n { + 0 + } else { + advance(td[1], td[3], actual) + }; + if actual < length { + retire = true; + if td[0] & (1 << 18) == 0 { + cc = 9; + halt = true; + } + } else if actual == n { + retire = true; + } + } + } + } + td[0] = (td[0] & 0x0fff_ffff) | (cc << 28); + if retire { + let next_td = td[2] & !15; + td[2] = s.r[12]; + // Failure stops immediately. Earlier successful writes and USB side + // effects remain visible; descriptor writeback is not a transaction. + ed[2] = next_td | ((td[0] >> 23) & 2) | u32::from(halt); + write_words(dma, at + 8, &[ed[2]])?; + write_words(dma, td_at, &td)?; + s.r[12] = td_at; + let di = ((td[0] >> 21) & 7) as u8; + s.done_delay = s.done_delay.min(if cc != 0 { 0 } else { di }); + Ok((next, next_td != ed[1] & !15 && !halt, true)) + } else { + write_words(dma, td_at, &td)?; + Ok((next, true, true)) + } + } + pub(super) fn publish(&self, s: &mut State, dma: &dyn PciDma) -> Result<(), ()> { + if s.done_delay == 0 && s.r[3] & WDH == 0 && s.r[12] != 0 { + let other = u32::from(s.r[3] & s.r[4] & EVENTS & !WDH != 0); + write_words(dma, s.r[6] + 0x84, &[s.r[12] | other])?; + s.r[12] = 0; + s.done_delay = 7; + s.r[3] |= WDH; + } else if s.done_delay > 0 && s.done_delay < 7 { + s.done_delay -= 1; + } + Ok(()) + } +} diff --git a/src/dev/ohci/schedule.rs b/src/dev/ohci/schedule.rs new file mode 100644 index 000000000..b6c8c6255 --- /dev/null +++ b/src/dev/ohci/schedule.rs @@ -0,0 +1,98 @@ +//! Functional scheduling approximation: 256 synthetic units per ED visit. +//! Costs deliberately do not represent wire time. See the module's limitations. +use super::*; +const VISIT: u32 = 256; +impl Ohci { + fn list_ready(s: &mut State, head: usize, current: usize, filled: u32) -> bool { + if s.r[current] == 0 && s.r[2] & filled != 0 { + s.r[current] = s.r[head]; + s.r[2] &= !filled; + } + s.r[current] != 0 + } + fn nonperiodic( + &self, + s: &mut State, + dma: &dyn PciDma, + budget: &mut u32, + cutoff: u32, + credit: &mut u32, + ) -> Result<(), ()> { + let mut yielded: Option = None; + while *budget >= VISIT && *budget > cutoff { + let control = s.r[1] & 16 != 0 && Self::list_ready(s, 8, 9, 2); + let bulk = s.r[1] & 32 != 0 && Self::list_ready(s, 10, 11, 4); + if !control && !bulk { + break; + } + let choose_control = control + && (!bulk + || match yielded { + Some(was_control) => !was_control, + None => *credit > 0, + }); + let (current, filled) = if choose_control { (9, 2) } else { (11, 4) }; + let (next, queued, attempt) = self.endpoint(s, dma, s.r[current])?; + *budget -= VISIT; + s.r[current] = next; + if queued { + s.r[2] |= filled; + } + yielded = if attempt { None } else { Some(choose_control) }; + if attempt { + if choose_control { + *credit = credit.saturating_sub(1); + } else { + *credit = (s.r[1] & 3) + 1; + } + } + } + Ok(()) + } + pub(super) fn schedule(&self, s: &mut State) -> Result<(), ()> { + let old = s.r[15] as u16; + let frame = old.wrapping_add(1); + s.r[15] = u32::from(frame); + if (old ^ frame) & 0x8000 != 0 { + s.r[3] |= 32; + } + s.r[3] |= 4; + let mut budget = (s.active_fi & 0x3fff) + 1; + s.r[14] = (s.active_fi & 0x80000000) | budget.min(0x3fff); + if self.config.command() & 4 == 0 { + return Ok(()); + } + let Some(dma) = self.dma.lock().clone() else { + return Err(()); + }; + let frame_bytes = [frame as u8, (frame >> 8) as u8, 0, 0]; + if !dma.dma_write((s.r[6] + 0x80) as u64, &frame_bytes).is_ok() { + return Err(()); + } + let mut credit = (s.r[1] & 3) + 1; + let cutoff = s.r[16].min(budget); + self.nonperiodic(s, &*dma, &mut budget, cutoff, &mut credit)?; + if s.r[1] & 4 != 0 { + let mut bytes = [0; 4]; + if !dma + .dma_read((s.r[6] + u32::from(frame & 31) * 4) as u64, &mut bytes) + .is_ok() + { + return Err(()); + } + s.r[7] = u32::from_le_bytes(bytes) & !15; + while s.r[7] != 0 && budget >= VISIT { + let (next, _, _) = self.endpoint(s, &*dma, s.r[7])?; + s.r[7] = next; + budget -= VISIT; + } + if s.r[7] != 0 { + s.r[3] |= 1; + s.r[2] = (s.r[2] & !0x30000) | ((s.r[2].wrapping_add(0x10000)) & 0x30000); + } + } + self.nonperiodic(s, &*dma, &mut budget, 0, &mut credit)?; + s.r[14] = (s.active_fi & 0x80000000) | budget.min(0x3fff); + self.publish(s, &*dma) + } +} diff --git a/src/dev/ohci/tests.rs b/src/dev/ohci/tests.rs new file mode 100644 index 000000000..1d0733cd3 --- /dev/null +++ b/src/dev/ohci/tests.rs @@ -0,0 +1,554 @@ +// The register and clock cases follow tests/qtest/usb-lucent-ohci-test.c in +// jamesbraid/qemu-sgi-origami: root-hub-descriptor-a-is-writable and +// frame-number-tracks-clock-while-timer-starved. The descriptor engine cases +// drive a test-only function and have no qtest counterpart. +use super::*; +use crate::pci::{PciBusDevice, PciDmaStatus}; +use crate::traits::{Device, Resettable}; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::Weak; + +struct Memory { + bytes: Mutex>, + refuse_read: Mutex>, + refuse_write: Mutex>, + writes: Mutex>, +} +impl Memory { + fn new() -> Arc { + Arc::new(Self { + bytes: Mutex::new(vec![0xa5; 0x10000]), + refuse_read: Mutex::new(None), + refuse_write: Mutex::new(None), + writes: Mutex::new(vec![]), + }) + } + fn words(&self, at: u32, words: &[u32]) { + for (i, w) in words.iter().enumerate() { + self.bytes.lock()[at as usize + i * 4..at as usize + i * 4 + 4] + .copy_from_slice(&w.to_le_bytes()); + } + } + fn word(&self, at: u32) -> u32 { + u32::from_le_bytes( + self.bytes.lock()[at as usize..at as usize + 4] + .try_into() + .unwrap(), + ) + } +} +impl PciDma for Memory { + fn dma_read(&self, addr: u64, dst: &mut [u8]) -> PciDmaStatus { + if self + .refuse_read + .lock() + .is_some_and(|x| (addr..addr + dst.len() as u64).contains(&(x as u64))) + { + return PciDmaStatus::MasterAbort; + } + let b = self.bytes.lock(); + let Some(s) = b.get(addr as usize..addr as usize + dst.len()) else { + return PciDmaStatus::MasterAbort; + }; + dst.copy_from_slice(s); + PciDmaStatus::Ok + } + fn dma_write(&self, addr: u64, src: &[u8]) -> PciDmaStatus { + if self + .refuse_write + .lock() + .is_some_and(|x| (addr..addr + src.len() as u64).contains(&(x as u64))) + { + return PciDmaStatus::MasterAbort; + } + let mut b = self.bytes.lock(); + let Some(s) = b.get_mut(addr as usize..addr as usize + src.len()) else { + return PciDmaStatus::MasterAbort; + }; + s.copy_from_slice(src); + self.writes.lock().push(addr); + PciDmaStatus::Ok + } +} +struct Function { + address: AtomicUsize, + calls: AtomicUsize, + reply: Mutex, + packets: Mutex>, +} +impl Function { + fn new(reply: UsbOutcome) -> Arc { + Arc::new(Self { + address: AtomicUsize::new(0), + calls: AtomicUsize::new(0), + reply: Mutex::new(reply), + packets: Mutex::new(vec![]), + }) + } +} +impl UsbFunction for Function { + fn reset(&self) {} + fn address(&self) -> u8 { + self.address.load(Ordering::SeqCst) as u8 + } + fn transact(&self, packet: UsbPacket) -> UsbOutcome { + self.calls.fetch_add(1, Ordering::SeqCst); + self.packets.lock().push(packet); + self.reply.lock().clone() + } +} +fn start_port_reset(h: &Ohci, port: u32) { + let status = 0x54 + port * 4; + h.bar_write(0, status, 0x100, 4); + h.bar_write(0, status, 0x10, 4); +} + +fn setup( + pid: Pid, + reply: UsbOutcome, + cbp: u32, + be: u32, + flags: u32, +) -> (Arc, Arc, Arc) { + let h = Ohci::new(); + let m = Memory::new(); + let f = Function::new(reply); + h.connect_dma(m.clone()); + h.attach(0, f.clone()).unwrap(); + h.start(); + start_port_reset(&h, 0); + h.advance_frames(10); + m.words(0x1000, &[0; 64]); + m.words(0x2000, &[(8 << 16), 0x3020, 0x3000, 0]); + m.words( + 0x3000, + &[0xf000_0000 | flags | ((pid as u32) << 19), cbp, 0x3020, be], + ); + h.config().write(1, 6); + h.bar_write(0, 0x18, 0x1000, 4); + h.bar_write(0, 0x20, 0x2000, 4); + h.bar_write(0, 0x08, 2, 4); + h.bar_write(0, 0x04, 0x90, 4); + (h, m, f) +} +/// The guest receives fatal status if the controller cannot access a descriptor. +#[test] +fn dma_refusal_preserves_descriptors_and_reports_a_fatal_error() { + for periodic in [false, true] { + for pid in [Pid::Out, Pid::In] { + let (h, m, f) = setup(pid, UsbOutcome::Ack(vec![5; 8]), 0x4000, 0x4007, 0); + if periodic { + m.words(0x1000, &[0x2000; 32]); + h.bar_write(0, 4, 0x84, 4); + } + if pid == Pid::In { + *m.refuse_write.lock() = Some(0x4000); + } else { + *m.refuse_read.lock() = Some(0x4000); + } + let before = m.bytes.lock()[0x3000..0x3010].to_vec(); + h.advance_frames(1); + let frame = h.bar_read(0, 0x3c, 4); + h.advance_frames(3); + assert_eq!(f.calls.load(Ordering::SeqCst), usize::from(pid == Pid::In)); + assert_eq!(&m.bytes.lock()[0x3000..0x3010], &before); + assert_eq!(m.word(0x2008), 0x3000); + assert_eq!(m.word(0x1084), 0); + assert_eq!(h.bar_read(0, 0x3c, 4), frame); + assert_ne!(h.bar_read(0, 0xc, 4) & UE, 0); + assert_eq!(h.config().read(1) >> 16, 0x8200); + h.config().write(1, 0x8000_0006); + assert_eq!(h.config().read(1) >> 16, 0x0200); + assert_ne!(h.bar_read(0, 0xc, 4) & UE, 0); + } + } +} + +/// Short and overlong USB replies leave bytes outside the packet untouched. +#[test] +fn short_or_overlong_in_packets_preserve_unwritten_bytes() { + for (name, payload, end, flags, bytes, current, condition) in [ + ("short", vec![1, 2, 3], 0x401f, (1 << 18) | (3 << 24), [1, 2, 3, 0xa5, 0xa5], 0x4003, 0), + ("overrun", vec![1; 8], 0x4003, 3 << 24, [1, 1, 1, 1, 0xa5], 0x4000, 8), + ] { + let (h, m, _) = setup(Pid::In, UsbOutcome::Ack(payload), 0x4000, end, flags); + h.advance_frames(1); + assert_eq!(&m.bytes.lock()[0x4000..0x4005], &bytes, "{name}"); + assert_eq!(m.word(0x3004), current, "{name}"); + assert_eq!(m.word(0x3000) >> 28, condition, "{name}"); + assert_eq!((m.word(0x3000) >> 24) & 3, 2, "{name}"); + if condition == 0 { + assert_eq!(m.word(0x2008) & !15, 0x3020); + assert_eq!(m.word(0x1084) & !15, 0x3000); + } + } +} + +/// The guest retries NAKs, stops on STALL, and retains consistent short-transfer state. +#[test] +fn nak_stall_and_short_in_completions_keep_the_endpoint_consistent() { + for (name, reply, flags, condition) in [ + ("NAK", UsbOutcome::Nak, 0, 15), + ("STALL", UsbOutcome::Stall, 3 << 24, 4), + ("no response", UsbOutcome::NoResponse, 0, 5), + ("short without rounding", UsbOutcome::Ack(vec![2]), 0, 9), + ] { + let (h, m, f) = setup(Pid::In, reply, 0x4000, 0x4007, flags); + h.bar_write(0, 0x34, 255, 4); + let before = m.bytes.lock()[0x3000..0x3010].to_vec(); + h.advance_frames(1); + if condition == 5 { + assert_eq!(f.calls.load(Ordering::SeqCst), 1); + assert_eq!((m.word(0x3000) >> 26) & 3, 1); + assert_eq!(m.word(0x2008), 0x3000); + h.advance_frames(2); + } + assert_eq!(m.word(0x3000) >> 28, condition, "{name}"); + if condition == 15 { + assert_eq!(&m.bytes.lock()[0x3000..0x3010], &before); + assert_eq!(m.word(0x2008), 0x3000); + } else { + assert_ne!(m.word(0x2008) & 1, 0, "{name}"); + } + if condition == 4 { + assert_eq!(m.word(0x3004), 0x4000); + assert_eq!((m.word(0x3000) >> 24) & 3, 3); + assert_eq!(m.word(0x2008) & 3, 3); + } + } +} + +/// Port and controller reset paths retain attached USB functions. +#[test] +fn attached_functions_survive_port_and_controller_resets() { + let h = Ohci::new(); + h.start(); + h.attach(1, Function::new(UsbOutcome::Nak)).unwrap(); + assert_eq!(h.bar_read(0, 0x58, 4) & 1, 0); + start_port_reset(&h, 1); + h.advance_frames(9); + assert_ne!(h.bar_read(0, 0x58, 4) & 16, 0); + h.advance_frames(1); + assert_eq!(h.bar_read(0, 0x58, 4) & 16, 0); + assert_ne!(h.bar_read(0, 0x58, 4) & (1 << 20), 0); + h.bar_write(0, 8, 1, 4); + assert_ne!(h.bar_read(0, 0x58, 4) & 1, 0); + h.bar_write(0, 0x58, 0x200, 4); + assert_eq!(h.bar_read(0, 0x58, 4) & 0x117, 0); + h.bar_write(0, 0x58, PPS, 4); + assert_ne!(h.bar_read(0, 0x58, 4) & 1, 0); + h.power_on(); + assert_eq!(h.bar_read(0, 0x58, 4), 0); + h.bar_write(0, 0x58, PPS, 4); + assert_ne!(h.bar_read(0, 0x58, 4) & 1, 0); +} + +/// The guest observes descriptor updates before the done bit and acknowledges them later. +#[test] +fn done_publication_orders_writes_and_waits_for_acknowledgement() { + let (h, m, _) = setup(Pid::In, UsbOutcome::Ack(vec![8; 8]), 0x4000, 0x4007, 0); + h.advance_frames(1); + { + let writes = m.writes.lock(); + assert!(writes.iter().position(|a| *a == 0x2008).unwrap() < writes.iter().position(|a| *a == 0x3000).unwrap()); + } + assert_eq!(m.word(0x1084) & !15, 0x3000); + m.words(0x3020, &[0xf0000000 | (2 << 19), 0x4100, 0x3040, 0x4107]); + m.words(0x2004, &[0x3040]); + h.bar_write(0, 8, 2, 4); + h.advance_frames(1); + assert_eq!(m.word(0x1084) & !15, 0x3000); + assert_eq!(h.bar_read(0, 0x30, 4), 0x3020); + h.bar_write(0, 0xc, 2, 4); + h.advance_frames(1); + assert_eq!(m.word(0x1084) & !15, 0x3020); + for delay in [0, 2, 7] { + let (h, m, _) = setup(Pid::In, UsbOutcome::Ack(vec![5; 8]), 0x4000, 0x4007, delay << 21); + h.advance_frames(1); + if delay != 0 { assert_eq!(m.word(0x1084), 0); } + h.advance_frames(u64::from(delay)); + assert_eq!(m.word(0x1084) & !15, if delay == 7 { 0 } else { 0x3000 }); + } + let (h, m, _) = setup(Pid::In, UsbOutcome::Ack(vec![1; 8]), 0x4000, 0x4007, 0); + *m.refuse_write.lock() = Some(0x1084); + h.advance_frames(1); + assert_ne!(h.bar_read(0, 0xc, 4) & UE, 0); + assert_eq!(h.bar_read(0, 0xc, 4) & WDH, 0); + assert_eq!(m.word(0x1084), 0); +} + +/// OHCI must yield to the other transfer list when a guest marks one list skipped. +#[test] +fn scheduler_shares_service_and_yields_past_skipped_lists() { + for (cbsr, skip_control, skip_bulk) in [(0, false, false), (3, false, false), (3, true, false), (0, false, true)] { + let (h, m, f) = setup(Pid::In, UsbOutcome::Nak, 0x4000, 0x4007, 0); + m.words(0x2000, &[(8 << 16) | (1 << 7) | (u32::from(skip_control) << 14), 0x3020, 0x3000, if skip_control { 0x2000 } else { 0 }]); + m.words(0x2100, &[(8 << 16) | (2 << 7) | (u32::from(skip_bulk) << 14), 0x3120, 0x3100, if skip_bulk { 0x2100 } else { 0 }]); + m.words(0x3100, &[0xf0100000, 0x4100, 0x3120, 0x4107]); + h.bar_write(0, 0x28, 0x2100, 4); + h.bar_write(0, 8, 6, 4); + h.bar_write(0, 4, 0xb0 | cbsr, 4); + h.advance_frames(1); + let packets = f.packets.lock(); + let control = packets.iter().filter(|p| p.endpoint == 1).count(); + let bulk = packets.iter().filter(|p| p.endpoint == 2).count(); + assert_eq!(control > 0, !skip_control); + assert_eq!(bulk > 0, !skip_bulk); + if skip_bulk { + assert!(control > 1, "control resumes after its credit is exhausted"); + } + if !skip_control && !skip_bulk { + assert!(control >= bulk * (cbsr as usize + 1)); + assert!(control <= bulk * (cbsr as usize + 1) + cbsr as usize + 1); + } + assert_eq!(h.bar_read(0, 0xc, 4) & UE, 0); + } +} + +/// The controller routes address zero only while one attached function can claim it. +#[test] +fn address_zero_is_routed_only_to_a_unique_eligible_function() { + for (name, address, peer_status, first_selected, peer_selected) in [ + ("ambiguous", 0, PPS | 1 | 2, false, false), + ("disabled peer", 0, PPS | 1, true, false), + ("assigned first", 5, PPS | 1 | 2, false, true), + ] { + let (h, m, f) = setup(Pid::In, UsbOutcome::Nak, 0x4000, 0x4007, 0); + let peer = Function::new(UsbOutcome::Nak); + h.attach(1, peer.clone()).unwrap(); + h.state.lock().r[22] = peer_status; + f.address.store(address, Ordering::SeqCst); + h.advance_frames(1); + assert_eq!(f.calls.load(Ordering::SeqCst) > 0, first_selected, "{name}"); + assert_eq!(peer.calls.load(Ordering::SeqCst) > 0, peer_selected, "{name}"); + if address != 0 { + let peer_calls = peer.calls.load(Ordering::SeqCst); + m.words(0x2000, &[(8 << 16) | 5]); + h.advance_frames(1); + assert!(f.calls.load(Ordering::SeqCst) > 0); + assert_eq!(peer.calls.load(Ordering::SeqCst), peer_calls); + } else if !first_selected { + assert_eq!(m.word(0x3000) >> 28, 5); + assert_eq!((m.word(0x3000) >> 26) & 3, 3); + } + } +} + + +/// A guest USB packet can cross physically nonadjacent DMA pages. +#[test] +fn packet_crosses_nonadjacent_pages() { + let (h, m, _) = setup( + Pid::In, + UsbOutcome::Ack(vec![1, 2, 3, 4, 5, 6, 7, 8]), + 0x4ffc, + 0x9003, + 0, + ); + h.advance_frames(1); + assert_eq!(&m.bytes.lock()[0x4ffc..0x5000], &[1, 2, 3, 4]); + assert_eq!(&m.bytes.lock()[0x9000..0x9004], &[5, 6, 7, 8]); + assert_eq!(m.bytes.lock()[0x5000], 0xa5); + assert_eq!(m.word(0x3004), 0); +} + +/// The frame counter catches up after its host timer was delayed. +#[test] +fn clock_read_catches_up_after_timer_starvation() { + let h = Ohci::new(); + let now = Arc::new(AtomicUsize::new(0)); + let t = now.clone(); + h.set_clock(Arc::new(move || t.load(Ordering::Relaxed) as u64)); + h.start(); + h.bar_write(0, 4, 0x80, 4); + now.store(5_000_000, Ordering::Relaxed); + assert!(h.bar_read(0, 0x3c, 4) >= 5); + h.stop(); + let frame = h.bar_read(0, 0x3c, 4); + h.advance_frames(10); + assert_eq!(h.bar_read(0, 0x3c, 4), frame); +} + +/// The frame count follows the guest's clock, not when the machine's poll or +/// a timer thread last ran: a guest that waits 1 ms after going operational +/// and reads HcFmNumber sees the frame, even if nothing else advanced the +/// controller in between (a slow or descheduled host thread). +#[test] +fn frame_number_follows_the_clock_with_no_poll_or_timer() { + let h = Ohci::new(); + let now = Arc::new(AtomicUsize::new(10_000_000)); + let t = now.clone(); + h.set_clock(Arc::new(move || t.load(Ordering::Relaxed) as u64)); + h.start(); + // Part-way into a frame period when the guest sets operational. + now.store(10_400_000, Ordering::Relaxed); + h.bar_write(0, 4, 0x80, 4); + // Entering Operational increments the frame number at once. + assert_eq!(h.bar_read(0, 0x3c, 4), 1); + // 1.1 ms on the guest's clock: the first boundary, 1 ms after the write, has passed. + now.store(11_500_000, Ordering::Relaxed); + assert_eq!(h.bar_read(0, 0x3c, 4), 2); + // A long guest-time gap is delivered in full at the next access. + now.store(31_500_000, Ordering::Relaxed); + assert_eq!(h.bar_read(0, 0x3c, 4), 22); + // Time that has not passed on the guest's clock delivers nothing. + assert_eq!(h.bar_read(0, 0x3c, 4), 22); + h.stop(); +} + +fn fake_clock_ohci(start_ns: usize) -> (Arc, Arc) { + let h = Ohci::new(); + let now = Arc::new(AtomicUsize::new(start_ns)); + let t = now.clone(); + h.set_clock(Arc::new(move || t.load(Ordering::Relaxed) as u64)); + h.start(); + (h, now) +} + +/// OpenHCI 1.0a sections 6.2.1 and 7.3.3: entering UsbOperational loads +/// HcFmRemaining from the frame interval, increments HcFmNumber, and sends the +/// first SOF one frame later, whatever the phase of the free-running clock. +fn operational_entry_restarts_the_frame(from_hcfs: Option) { + let (h, now) = fake_clock_ohci(10_000_000); + let m = Memory::new(); + h.connect_dma(m.clone()); + h.config().write(1, 6); + h.bar_write(0, 0x18, 0x1000, 4); + // Not operational: no frames count however much time passes. + if let Some(hcfs) = from_hcfs { + h.bar_write(0, 4, hcfs, 4); + } + now.store(15_000_000, Ordering::Relaxed); + assert_eq!(h.bar_read(0, 0x3c, 4), 0); + now.store(15_400_000, Ordering::Relaxed); + h.bar_write(0, 4, 0x80, 4); + assert_eq!(h.bar_read(0, 0x3c, 4), 1); + assert_eq!(h.bar_read(0, 0x38, 4), 0x2edf); + assert_eq!(h.bar_read(0, 0x0c, 4) & 4, 0, "no SOF at the load"); + assert_eq!(m.bytes.lock()[0x1080], 0xa5, "no HCCA write at the load"); + // Just short of one frame after the write: still nothing. + now.store(16_399_999, Ordering::Relaxed); + assert_eq!(h.bar_read(0, 0x3c, 4), 1); + assert_eq!(h.bar_read(0, 0x0c, 4) & 4, 0); + assert_eq!(m.bytes.lock()[0x1080], 0xa5); + // Exactly one frame after: the first SOF and its HCCA frame number. + now.store(16_400_000, Ordering::Relaxed); + assert_eq!(h.bar_read(0, 0x0c, 4) & 4, 4); + assert_eq!(h.bar_read(0, 0x3c, 4), 2); + assert_eq!(m.bytes.lock()[0x1080], 2); + h.stop(); +} + +#[test] +fn operational_entry_from_reset_restarts_the_frame() { + operational_entry_restarts_the_frame(None); +} + +#[test] +fn operational_entry_from_suspend_restarts_the_frame() { + // HCFS 11b. + operational_entry_restarts_the_frame(Some(0xc0)); +} + +#[test] +fn operational_entry_from_resume_restarts_the_frame() { + // HCFS 01b. + operational_entry_restarts_the_frame(Some(0x40)); +} + +/// The increment on entry carries into the frame-number overflow flag like a +/// frame's own, and HcFmRemaining takes the frame-interval toggle bit too. +#[test] +fn operational_entry_sets_overflow_and_loads_the_interval_toggle() { + for (before, after, overflow) in [(0x7fff, 0x8000, true), (0xffff, 0, true), (0x1234, 0x1235, false)] { + let (h, _now) = fake_clock_ohci(10_000_000); + h.state.lock().r[15] = before; + h.bar_write(0, 0x34, 0x8000_2edf, 4); + h.bar_write(0, 4, 0x80, 4); + assert_eq!(h.bar_read(0, 0x3c, 4), after, "from {before:#x}"); + assert_eq!(h.bar_read(0, 0x0c, 4) & 32 != 0, overflow, "from {before:#x}"); + assert_eq!(h.bar_read(0, 0x38, 4), 0x8000_2edf); + h.stop(); + } +} + +/// An interrupt handler can acknowledge a line while the controller raises it. +#[test] +fn reentrant_irq_acknowledgement_leaves_final_line_clear() { + struct Ack { + hc: Weak, + level: AtomicBool, + once: AtomicBool, + } + impl crate::irq::InterruptController for Ack { + fn set_line(&self, _: u32, level: bool) { + if level && !self.once.swap(true, Ordering::SeqCst) { + self.hc.upgrade().unwrap().bar_write(0, 0xc, RHSC, 4); + } + self.level.store(level, Ordering::SeqCst); + } + } + let h = Ohci::new(); + let ack = Arc::new(Ack { + hc: Arc::downgrade(&h), + level: AtomicBool::new(false), + once: AtomicBool::new(false), + }); + h.connect_interrupt(IrqLine::new(ack.clone(), 0)); + h.bar_write(0, 0x10, 0x80000040, 4); + h.attach(0, Function::new(UsbOutcome::Nak)).unwrap(); + assert!(!ack.level.load(Ordering::SeqCst)); +} + +/// The guest sees the documented write masks, HCR exemptions, and ownership bit. +#[test] +fn register_masks_preserve_hcr_exempt_fields_and_ownership_status() { + let h = Ohci::new(); + h.bar_write(0, 0x18, 0x123456ff, 4); + assert_eq!(h.bar_read(0, 0x18, 4), 0x12345600); + h.bar_write(0, 0x20, 0x123456ff, 4); + assert_eq!(h.bar_read(0, 0x20, 4), 0x123456f0); + h.bar_write(0, 0x10, u32::MAX, 4); + assert_eq!(h.bar_read(0, 0x10, 4), 0xc000007f); + h.bar_write(0, 8, 8, 4); + assert_eq!(h.bar_read(0, 0xc, 4), 1 << 30); + h.bar_write(0, 0xc, 1 << 31, 4); + assert_eq!(h.bar_read(0, 0xc, 4), 1 << 30); + h.bar_write(0, 0xc, 1 << 30, 4); + assert_eq!(h.bar_read(0, 0xc, 4), 0); + h.bar_write(0, 0x48, u32::MAX, 4); + assert_eq!(h.bar_read(0, 0x48, 4), 0xff001b02); + h.bar_write(0, 4, 0x700, 4); + h.bar_write(0, 8, 1, 4); + assert_eq!(h.bar_read(0, 4, 4), 0x3c0); + assert_eq!(h.bar_read(0, 0x48, 4), 0xff001b02); + assert_eq!(h.bar_read(0, 0x10, 4), 0); + h.bar_write(0, 0x3c, 0xffff, 4); + assert_eq!(h.bar_read(0, 0x3c, 4), 0); + h.bar_write(0, 0x18, 1, 1); + assert_eq!(h.bar_read(0, 0x18, 4), 0); +} + +/// The periodic phase controls how the guest control and bulk lists share service. +#[test] +fn periodic_phase_preserves_control_and_bulk_credit() { + let (h, m, f) = setup(Pid::In, UsbOutcome::Nak, 0x4000, 0x4007, 0); + m.words(0x2000, &[(8 << 16) | (1 << 7), 0x3020, 0x3000, 0]); + m.words(0x2100, &[(8 << 16) | (2 << 7), 0x3120, 0x3100, 0]); + m.words(0x3100, &[0xf0100000, 0x4100, 0x3120, 0x4107]); + m.words(0x2200, &[(8 << 16) | (3 << 7), 0x3220, 0x3200, 0]); + m.words(0x3200, &[0xf0100000, 0x4200, 0x3220, 0x4207]); + for i in 0..32 { + m.words(0x1000 + i * 4, &[0x2200]); + } + let mut s = h.state.lock().clone(); + s.active_fi = 5 * 256 - 1; + s.r[16] = 3 * 256; + s.r[1] = 0xb5; + s.r[10] = 0x2100; + s.r[2] |= 4; + h.schedule(&mut s).unwrap(); + let order: Vec<_> = f.packets.lock().iter().map(|p| p.endpoint).collect(); + assert_eq!(order, [1, 1, 3, 2, 1]); + assert_eq!(s.r[14], 0); +} diff --git a/src/dev/ps2.rs b/src/dev/ps2.rs index fa24f7b6e..068e70084 100644 --- a/src/dev/ps2.rs +++ b/src/dev/ps2.rs @@ -57,8 +57,25 @@ struct Ps2State { const MOUSE_MAX: i32 = 127; /// Combined PS/2 Keyboard and Mouse Controller +/// Another input device on the same host keyboard and mouse: a machine whose +/// guest may use USB instead of PS/2 (the Fuel) fits one beside the 8042 or +/// IOC3 controller, and host input reaches both. Each device drops what its +/// guest has not enabled, as the PS/2 side does. +pub trait HostInputSink: Send + Sync { + fn key(&self, key: KeyCode, pressed: bool); + /// Buttons in PS/2 order (left 1, right 2, middle 4); `dy` is positive + /// down the screen. + fn mouse(&self, buttons: u8, dx: i32, dy: i32, dz: i32); + /// Whether the guest has enabled this mouse. + fn mouse_enabled(&self) -> bool; + /// Mouse movement queued that the guest has not read yet. + fn mouse_pending(&self) -> usize; +} + pub struct Ps2Controller { state: Mutex, + /// Other devices fed from the same host input. + sinks: Mutex>>, /// The controller's interrupt: up while a byte waits to be read. irq: Option, running: AtomicBool, @@ -91,6 +108,7 @@ impl Ps2Controller { held_keys: [false; 256], }), irq, + sinks: Mutex::new(Vec::new()), running: AtomicBool::new(false), layout: AtomicU8::new(KeyboardLayout::Ansi as u8), } @@ -526,17 +544,27 @@ impl Ps2Controller { /// clicks are dropped. pub fn mouse_enabled(&self) -> bool { let state = self.state.lock(); - state.mouse_enabled && state.config & 0x20 == 0 + (state.mouse_enabled && state.config & 0x20 == 0) + || self.sinks.lock().iter().any(|s| s.mouse_enabled()) + } + + /// Feed host keyboard and mouse input to `sink` as well. + pub fn add_sink(&self, sink: Arc) { + self.sinks.lock().push(sink); } /// Mouse bytes queued that the guest has not read yet. pub fn mouse_pending(&self) -> usize { - self.state.lock().mouse_queue_bytes + let own = self.state.lock().mouse_queue_bytes; + self.sinks.lock().iter().fold(own, |n, s| n.max(s.mouse_pending())) } /// Push a scancode byte to the keyboard queue (called from UI) pub fn push_kb(&self, key: KeyCode, pressed: bool) { if !self.running.load(Ordering::Relaxed) { return; } + for sink in self.sinks.lock().iter() { + sink.key(key, pressed); + } let mut state = self.state.lock(); if !state.scanning_enabled || state.config & 0x10 != 0 { return; @@ -968,6 +996,9 @@ impl Ps2Controller { /// - `dz`: scroll wheel delta (positive = scroll up/away from user) pub fn push_mouse_input(&self, buttons: u8, dx: i32, dy: i32, dz: i32) { if !self.running.load(Ordering::Relaxed) { return; } + for sink in self.sinks.lock().iter() { + sink.mouse(buttons, dx, dy, dz); + } let mut state = self.state.lock(); if !state.mouse_enabled || state.config & 0x20 != 0 { return; } @@ -1348,6 +1379,41 @@ impl Saveable for Ps2Controller { mod tests { use super::*; + /// A sink sees host keys and mouse moves even while the guest has not + /// enabled the PS/2 devices, and its mouse counts toward `mouse_enabled` + /// and `mouse_pending`. + #[test] + fn a_sink_shares_the_host_input() { + #[derive(Default)] + struct Sink(Mutex>); + impl HostInputSink for Sink { + fn key(&self, key: KeyCode, pressed: bool) { + self.0.lock().push(format!("{key:?} {pressed}")); + } + fn mouse(&self, buttons: u8, dx: i32, dy: i32, dz: i32) { + self.0.lock().push(format!("mouse {buttons} {dx} {dy} {dz}")); + } + fn mouse_enabled(&self) -> bool { + true + } + fn mouse_pending(&self) -> usize { + 5 + } + } + let p = Ps2Controller::new(None); + let sink = Arc::new(Sink::default()); + p.add_sink(sink.clone()); + p.push_kb(KeyCode::KeyA, true); + assert!(sink.0.lock().is_empty(), "a stopped controller passes nothing on"); + p.start(); + assert!(p.mouse_enabled(), "the sink's mouse is on though the PS/2 mouse is not"); + assert_eq!(p.mouse_pending(), 5); + p.push_kb(KeyCode::KeyA, true); + p.push_mouse_input(2, 3, -4, 1); + assert_eq!(*sink.0.lock(), ["KeyA true", "mouse 2 3 -4 1"]); + assert_eq!(p.pop_from(false), None, "and the PS/2 keyboard, not enabled, took nothing"); + } + /// `type_char` sends what the keys themselves would: Shift around a /// shifted character, Enter for a newline, and nothing for a character /// with no key. diff --git a/src/dev/usb_hid.rs b/src/dev/usb_hid.rs new file mode 100644 index 000000000..8f417c05b --- /dev/null +++ b/src/dev/usb_hid.rs @@ -0,0 +1,609 @@ +//! USB HID boot-protocol keyboard and mouse: low-speed functions that plug +//! into an OHCI root port and are fed from the host's keyboard and mouse. +//! +//! Written from the public specifications: USB 1.1 (chapter 9 requests and +//! descriptors), Device Class Definition for HID 1.11 (class requests and +//! Appendix B, the boot interface descriptors and report formats) and the HID +//! Usage Tables (keyboard/keypad page). Each device has one configuration, +//! one boot interface and one interrupt-IN endpoint. +//! +//! The control pipe follows USB 1.1 section 8.5.2: a SETUP packet, an optional +//! data stage and a status stage. SET_ADDRESS takes effect once its status +//! stage completes. Interrupt reports queue one per host-side change so a key +//! tapped and released between two polls is still seen twice. Input that +//! arrives before the guest has selected a configuration is dropped, as an +//! unconfigured device would not report it. +//! +//! The data toggle on the wire is not tracked: the controller model keeps +//! each endpoint's toggle and a lost transfer cannot be replayed here. +//! Remote wakeup, the wheel, and the host keyboard layout are not modelled. + +use crate::dev::ohci::{Pid, UsbFunction, UsbOutcome, UsbPacket}; +use crate::dev::ps2::HostInputSink; +use parking_lot::Mutex; +use std::collections::VecDeque; +use std::sync::Arc; +use winit::keyboard::KeyCode; + +/// A vendor and product pair from the pid.codes open test allocation, since +/// the devices are not products of any vendor. +const VENDOR_ID: u16 = 0x1209; +const PRODUCT_ID: u16 = 0x0001; + +// Standard request codes (USB 1.1 table 9-4) and descriptor types (table 9-5). +const GET_STATUS: u8 = 0; +const CLEAR_FEATURE: u8 = 1; +const SET_FEATURE: u8 = 3; +const SET_ADDRESS: u8 = 5; +const GET_DESCRIPTOR: u8 = 6; +const GET_CONFIGURATION: u8 = 8; +const SET_CONFIGURATION: u8 = 9; +const GET_INTERFACE: u8 = 10; +const SET_INTERFACE: u8 = 11; +const DESC_DEVICE: u8 = 1; +const DESC_CONFIGURATION: u8 = 2; +const DESC_STRING: u8 = 3; +const DESC_HID: u8 = 0x21; +const DESC_REPORT: u8 = 0x22; +// HID class requests (HID 1.11 section 7.2). +const GET_REPORT: u8 = 1; +const GET_IDLE: u8 = 2; +const GET_PROTOCOL: u8 = 3; +const SET_REPORT: u8 = 9; +const SET_IDLE: u8 = 10; +const SET_PROTOCOL: u8 = 11; + +/// HID 1.11 appendix B.1: the boot keyboard's report descriptor. Input: a +/// modifier bit per key and six key usages; output: five LED bits. +pub const KEYBOARD_REPORT_DESCRIPTOR: [u8; 63] = [ + 0x05, 0x01, 0x09, 0x06, 0xa1, 0x01, 0x05, 0x07, 0x19, 0xe0, 0x29, 0xe7, 0x15, 0x00, 0x25, 0x01, + 0x75, 0x01, 0x95, 0x08, 0x81, 0x02, 0x95, 0x01, 0x75, 0x08, 0x81, 0x01, 0x95, 0x05, 0x75, 0x01, + 0x05, 0x08, 0x19, 0x01, 0x29, 0x05, 0x91, 0x02, 0x95, 0x01, 0x75, 0x03, 0x91, 0x01, 0x95, 0x06, + 0x75, 0x08, 0x15, 0x00, 0x25, 0x65, 0x05, 0x07, 0x19, 0x00, 0x29, 0x65, 0x81, 0x00, 0xc0, +]; +/// HID 1.11 appendix B.2: the boot mouse's report descriptor. Three buttons +/// and relative X and Y. +pub const MOUSE_REPORT_DESCRIPTOR: [u8; 50] = [ + 0x05, 0x01, 0x09, 0x02, 0xa1, 0x01, 0x09, 0x01, 0xa1, 0x00, 0x05, 0x09, 0x19, 0x01, 0x29, 0x03, + 0x15, 0x00, 0x25, 0x01, 0x95, 0x03, 0x75, 0x01, 0x81, 0x02, 0x95, 0x01, 0x75, 0x05, 0x81, 0x01, + 0x05, 0x01, 0x09, 0x30, 0x09, 0x31, 0x15, 0x81, 0x25, 0x7f, 0x75, 0x08, 0x95, 0x02, 0x81, 0x06, + 0xc0, 0xc0, +]; + +/// Reports held for the guest before the oldest is dropped. +const QUEUE_LIMIT: usize = 64; + +/// What differs between the two devices: the report they send and how host +/// input becomes one. +pub trait Boot: Send { + const PRODUCT: &'static str; + /// bInterfaceProtocol: 1 keyboard, 2 mouse. + const PROTOCOL: u8; + const REPORT_DESCRIPTOR: &'static [u8]; + const MAX_PACKET: u8; + /// The idle rate at reset, in 4 ms units (HID 1.11 section 7.2.4). + const DEFAULT_IDLE: u8; + fn reset(&mut self); + /// The report GET_REPORT returns now. + fn current(&self) -> Vec; + /// The next queued report, if any. + fn next_report(&mut self) -> Option>; + /// An output report; false when the device has none. + fn set_output(&mut self, _data: &[u8]) -> bool { + false + } + /// Host input waiting for the guest, in reports. + fn backlog(&self) -> usize; +} + +enum Stage { + Idle, + /// Data stage toward the host, then an OUT status. + In { data: Vec, sent: usize }, + /// Data stage from the host for a SET_REPORT of `length` bytes. + Out { length: usize, data: Vec }, + /// Status stage of a request with no data toward the host: the next IN + /// answers with an empty packet and the request takes effect. + StatusIn(Effect), +} + +#[derive(Clone, Copy)] +enum Effect { + None, + Address(u8), + Configuration(u8), + Idle(u8), + Protocol(u8), + Output, +} + +struct Inner { + dev: B, + address: u8, + configured: bool, + /// 0 boot, 1 report; a device starts in report protocol. + protocol: u8, + idle: u8, + remote_wakeup: bool, + stage: Stage, + last_report_ms: u64, +} + +pub struct UsbHid { + inner: Mutex>, +} + +impl UsbHid { + fn with(dev: B) -> Arc { + Arc::new(Self { + inner: Mutex::new(Inner { + dev, + address: 0, + configured: false, + protocol: 1, + idle: B::DEFAULT_IDLE, + remote_wakeup: false, + stage: Stage::Idle, + last_report_ms: 0, + }), + }) + } +} + +fn string_descriptor(s: &str) -> Vec { + let units: Vec = s.encode_utf16().flat_map(u16::to_le_bytes).collect(); + let mut d = vec![2 + units.len() as u8, DESC_STRING]; + d.extend(units); + d +} + +impl Inner { + fn device_descriptor() -> Vec { + let mut d = vec![18, DESC_DEVICE, 0x10, 0x01, 0, 0, 0, 8]; + d.extend(VENDOR_ID.to_le_bytes()); + d.extend(PRODUCT_ID.to_le_bytes()); + // bcdDevice, manufacturer and product strings, no serial, one configuration. + d.extend([0x00, 0x01, 1, 2, 0, 1]); + d + } + + fn hid_descriptor() -> Vec { + let n = B::REPORT_DESCRIPTOR.len() as u16; + let mut d = vec![9, DESC_HID, 0x11, 0x01, 0, 1, DESC_REPORT]; + d.extend(n.to_le_bytes()); + d + } + + /// The configuration descriptor followed by its interface, HID and + /// endpoint descriptors, as GET_DESCRIPTOR(configuration) returns them. + fn configuration_descriptor() -> Vec { + let interface = [9, 4, 0, 0, 1, 3, 1, B::PROTOCOL, 0]; + let hid = Self::hid_descriptor(); + // Interrupt IN endpoint 1, polled every 10 ms. + let endpoint = [7, 5, 0x81, 3, B::MAX_PACKET, 0, 10]; + let total = (9 + interface.len() + hid.len() + endpoint.len()) as u16; + let mut d = vec![9, DESC_CONFIGURATION]; + d.extend(total.to_le_bytes()); + // One interface, configuration 1, no string, bus powered with remote + // wakeup, 100 mA. + d.extend([1, 1, 0, 0xa0, 50]); + d.extend(interface); + d.extend(hid); + d.extend(endpoint); + d + } + + fn string(index: u8) -> Option> { + match index { + 0 => Some(vec![4, DESC_STRING, 0x09, 0x04]), + 1 => Some(string_descriptor("iris")), + 2 => Some(string_descriptor(B::PRODUCT)), + _ => None, + } + } + + fn reset(&mut self) { + self.address = 0; + self.configured = false; + self.protocol = 1; + self.idle = B::DEFAULT_IDLE; + self.remote_wakeup = false; + self.stage = Stage::Idle; + self.dev.reset(); + } + + /// Decode a SETUP packet (USB 1.1 table 9-2). None is a request this + /// device does not have, answered with a STALL. + fn setup(&mut self, p: &[u8]) -> Option<()> { + let [rt, req, vl, vh, il, _ih, ll, lh] = <[u8; 8]>::try_from(p).ok()?; + let value = u16::from_le_bytes([vl, vh]); + let length = u16::from_le_bytes([ll, lh]) as usize; + let reply = |data: Vec| Stage::In { sent: 0, data: data[..data.len().min(length)].to_vec() }; + let (kind, recipient) = ((rt >> 5) & 3, rt & 0x1f); + let to_host = rt & 0x80 != 0; + self.stage = match (kind, recipient, to_host, req) { + // Standard requests (USB 1.1 section 9.4). + (0, 0, true, GET_STATUS) => reply(vec![u8::from(self.remote_wakeup) << 1, 0]), + (0, 1 | 2, true, GET_STATUS) if value == 0 => reply(vec![0, 0]), + (0, 0, false, CLEAR_FEATURE) if value == 1 => { + self.remote_wakeup = false; + Stage::StatusIn(Effect::None) + } + (0, 0, false, SET_FEATURE) if value == 1 => { + self.remote_wakeup = true; + Stage::StatusIn(Effect::None) + } + // Clearing a halt on the interrupt endpoint is accepted; nothing halts it. + (0, 2, false, CLEAR_FEATURE) if value == 0 && il & 0x8f == 0x81 => Stage::StatusIn(Effect::None), + (0, 0, false, SET_ADDRESS) if value <= 127 => Stage::StatusIn(Effect::Address(value as u8)), + (0, 0, true, GET_DESCRIPTOR) => match (vh, vl) { + (DESC_DEVICE, 0) => reply(Self::device_descriptor()), + (DESC_CONFIGURATION, 0) => reply(Self::configuration_descriptor()), + (DESC_STRING, n) => reply(Self::string(n)?), + _ => return None, + }, + (0, 0, true, GET_CONFIGURATION) => reply(vec![u8::from(self.configured)]), + (0, 0, false, SET_CONFIGURATION) if value <= 1 => Stage::StatusIn(Effect::Configuration(value as u8)), + (0, 1, true, GET_INTERFACE) if il == 0 => reply(vec![0]), + (0, 1, false, SET_INTERFACE) if il == 0 && value == 0 => Stage::StatusIn(Effect::None), + // The HID descriptor and report descriptor belong to the interface. + (0, 1, true, GET_DESCRIPTOR) if il == 0 => match (vh, vl) { + (DESC_HID, 0) => reply(Self::hid_descriptor()), + (DESC_REPORT, 0) => reply(B::REPORT_DESCRIPTOR.to_vec()), + _ => return None, + }, + // HID class requests (HID 1.11 section 7.2), interface 0. + (1, 1, true, GET_REPORT) if il == 0 && vh == 1 && vl == 0 => reply(self.dev.current()), + (1, 1, false, SET_REPORT) if il == 0 && vh == 2 && vl == 0 && length > 0 => { + Stage::Out { length, data: Vec::new() } + } + (1, 1, true, GET_IDLE) if il == 0 => reply(vec![self.idle]), + (1, 1, false, SET_IDLE) if il == 0 => Stage::StatusIn(Effect::Idle(vh)), + (1, 1, true, GET_PROTOCOL) if il == 0 => reply(vec![self.protocol]), + (1, 1, false, SET_PROTOCOL) if il == 0 && value <= 1 => Stage::StatusIn(Effect::Protocol(value as u8)), + _ => return None, + }; + Some(()) + } + + fn apply(&mut self, effect: Effect, data: &[u8], now_ms: u64) -> bool { + match effect { + Effect::None => {} + Effect::Address(a) => self.address = a, + Effect::Configuration(c) => { + self.configured = c != 0; + self.dev.reset(); + self.last_report_ms = now_ms; + } + Effect::Idle(rate) => self.idle = rate, + Effect::Protocol(p) => self.protocol = p, + Effect::Output => return self.dev.set_output(data), + } + true + } + + fn control(&mut self, p: UsbPacket) -> UsbOutcome { + match p.pid { + Pid::Setup => match self.setup(&p.data) { + Some(()) => UsbOutcome::Ack(Vec::new()), + None => { + self.stage = Stage::Idle; + UsbOutcome::Stall + } + }, + Pid::In => match &mut self.stage { + Stage::In { data, sent } => { + let n = p.capacity.min(data.len() - *sent); + let chunk = data[*sent..*sent + n].to_vec(); + *sent += n; + UsbOutcome::Ack(chunk) + } + Stage::StatusIn(effect) => { + let effect = *effect; + self.stage = Stage::Idle; + if self.apply(effect, &[], p.time_ms) { UsbOutcome::Ack(Vec::new()) } else { UsbOutcome::Stall } + } + _ => UsbOutcome::Stall, + }, + Pid::Out => match &mut self.stage { + // The status stage of a request that sent data to the host. + Stage::In { .. } if p.data.is_empty() => { + self.stage = Stage::Idle; + UsbOutcome::Ack(Vec::new()) + } + Stage::Out { length, data } => { + data.extend(&p.data); + if data.len() >= *length { + let data = std::mem::take(data); + let ok = self.apply(Effect::Output, &data, p.time_ms); + self.stage = if ok { Stage::StatusIn(Effect::None) } else { Stage::Idle }; + if !ok { + return UsbOutcome::Stall; + } + } + UsbOutcome::Ack(Vec::new()) + } + _ => UsbOutcome::Stall, + }, + } + } + + /// Interrupt IN endpoint 1 (USB 1.1 section 5.7): a report when there is + /// one, otherwise NAK. A nonzero idle rate repeats the last state once the + /// period passes without a change. + fn interrupt(&mut self, p: UsbPacket) -> UsbOutcome { + if !self.configured { + return UsbOutcome::Stall; + } + if let Some(r) = self.dev.next_report() { + self.last_report_ms = p.time_ms; + return UsbOutcome::Ack(r); + } + if self.idle != 0 && p.time_ms.saturating_sub(self.last_report_ms) >= u64::from(self.idle) * 4 { + self.last_report_ms = p.time_ms; + return UsbOutcome::Ack(self.dev.current()); + } + UsbOutcome::Nak + } +} + +impl UsbFunction for UsbHid { + fn reset(&self) { + self.inner.lock().reset(); + } + fn address(&self) -> u8 { + self.inner.lock().address + } + fn low_speed(&self) -> bool { + true + } + fn transact(&self, packet: UsbPacket) -> UsbOutcome { + let mut s = self.inner.lock(); + match packet.endpoint { + 0 => s.control(packet), + 1 if packet.pid == Pid::In => s.interrupt(packet), + _ => UsbOutcome::Stall, + } + } +} + +// --------------------------------------------------------------------------- +// Keyboard +// --------------------------------------------------------------------------- + +/// The HID usage (keyboard/keypad page 0x07) of a host key, or None for a +/// key the page has no entry for. +pub fn usage(key: KeyCode) -> Option { + use KeyCode::*; + Some(match key { + KeyA => 0x04, KeyB => 0x05, KeyC => 0x06, KeyD => 0x07, KeyE => 0x08, KeyF => 0x09, + KeyG => 0x0a, KeyH => 0x0b, KeyI => 0x0c, KeyJ => 0x0d, KeyK => 0x0e, KeyL => 0x0f, + KeyM => 0x10, KeyN => 0x11, KeyO => 0x12, KeyP => 0x13, KeyQ => 0x14, KeyR => 0x15, + KeyS => 0x16, KeyT => 0x17, KeyU => 0x18, KeyV => 0x19, KeyW => 0x1a, KeyX => 0x1b, + KeyY => 0x1c, KeyZ => 0x1d, + Digit1 => 0x1e, Digit2 => 0x1f, Digit3 => 0x20, Digit4 => 0x21, Digit5 => 0x22, + Digit6 => 0x23, Digit7 => 0x24, Digit8 => 0x25, Digit9 => 0x26, Digit0 => 0x27, + Enter => 0x28, Escape => 0x29, Backspace => 0x2a, Tab => 0x2b, Space => 0x2c, + Minus => 0x2d, Equal => 0x2e, BracketLeft => 0x2f, BracketRight => 0x30, + Backslash => 0x31, Semicolon => 0x33, Quote => 0x34, Backquote => 0x35, + Comma => 0x36, Period => 0x37, Slash => 0x38, CapsLock => 0x39, + F1 => 0x3a, F2 => 0x3b, F3 => 0x3c, F4 => 0x3d, F5 => 0x3e, F6 => 0x3f, + F7 => 0x40, F8 => 0x41, F9 => 0x42, F10 => 0x43, F11 => 0x44, F12 => 0x45, + PrintScreen => 0x46, ScrollLock => 0x47, Pause => 0x48, Insert => 0x49, + Home => 0x4a, PageUp => 0x4b, Delete => 0x4c, End => 0x4d, PageDown => 0x4e, + ArrowRight => 0x4f, ArrowLeft => 0x50, ArrowDown => 0x51, ArrowUp => 0x52, + NumLock => 0x53, NumpadDivide => 0x54, NumpadMultiply => 0x55, + NumpadSubtract => 0x56, NumpadAdd => 0x57, NumpadEnter => 0x58, + Numpad1 => 0x59, Numpad2 => 0x5a, Numpad3 => 0x5b, Numpad4 => 0x5c, + Numpad5 => 0x5d, Numpad6 => 0x5e, Numpad7 => 0x5f, Numpad8 => 0x60, + Numpad9 => 0x61, Numpad0 => 0x62, NumpadDecimal => 0x63, + IntlBackslash => 0x64, ContextMenu => 0x65, + _ => return None, + }) +} + +/// The bit of a modifier key in the report's first byte (usages 0xe0 to 0xe7). +fn modifier(key: KeyCode) -> Option { + use KeyCode::*; + Some(match key { + ControlLeft => 0x01, ShiftLeft => 0x02, AltLeft => 0x04, SuperLeft => 0x08, + ControlRight => 0x10, ShiftRight => 0x20, AltRight => 0x40, SuperRight => 0x80, + _ => return None, + }) +} + +#[derive(Default)] +pub struct Keyboard { + modifiers: u8, + /// Pressed keys in the order they went down. + keys: Vec, + queue: VecDeque>, + leds: u8, +} + +impl Keyboard { + /// The 8-byte boot report (HID 1.11 B.1): modifiers, reserved, six keys. + /// More than six keys at once is the phantom state, ErrorRollOver in every slot. + fn report(&self) -> Vec { + let mut r = vec![self.modifiers, 0, 0, 0, 0, 0, 0, 0]; + if self.keys.len() > 6 { + r[2..].fill(0x01); + } else { + r[2..2 + self.keys.len()].copy_from_slice(&self.keys); + } + r + } + + fn push(&mut self) { + if self.queue.len() >= QUEUE_LIMIT { + self.queue.pop_front(); + } + let report = self.report(); + self.queue.push_back(report); + } +} + +impl Boot for Keyboard { + const PRODUCT: &'static str = "USB Keyboard"; + const PROTOCOL: u8 = 1; + const REPORT_DESCRIPTOR: &'static [u8] = &KEYBOARD_REPORT_DESCRIPTOR; + const MAX_PACKET: u8 = 8; + // 500 ms, the recommended keyboard default. + const DEFAULT_IDLE: u8 = 125; + fn reset(&mut self) { + *self = Keyboard::default(); + } + fn current(&self) -> Vec { + self.report() + } + fn next_report(&mut self) -> Option> { + self.queue.pop_front() + } + /// The LED output report (HID 1.11 B.1): num lock, caps lock, scroll + /// lock, compose, kana in bits 0 to 4. + fn set_output(&mut self, data: &[u8]) -> bool { + self.leds = data.first().copied().unwrap_or(0) & 0x1f; + true + } + fn backlog(&self) -> usize { + self.queue.len() + } +} + +pub type UsbKeyboard = UsbHid; + +impl UsbKeyboard { + pub fn new() -> Arc { + Self::with(Keyboard::default()) + } + + /// The LEDs the guest last asked for. + pub fn leds(&self) -> u8 { + self.inner.lock().dev.leds + } +} + +impl HostInputSink for UsbKeyboard { + fn key(&self, key: KeyCode, pressed: bool) { + let mut s = self.inner.lock(); + if !s.configured { + return; + } + let k = &mut s.dev; + if let Some(bit) = modifier(key) { + let before = k.modifiers; + if pressed { k.modifiers |= bit } else { k.modifiers &= !bit } + if k.modifiers != before { + k.push(); + } + } else if let Some(u) = usage(key) { + let at = k.keys.iter().position(|&x| x == u); + match (pressed, at) { + (true, None) => k.keys.push(u), + (false, Some(i)) => { + k.keys.remove(i); + } + _ => return, + } + k.push(); + } + } + fn mouse(&self, _: u8, _: i32, _: i32, _: i32) {} + fn mouse_enabled(&self) -> bool { + false + } + fn mouse_pending(&self) -> usize { + 0 + } +} + +// --------------------------------------------------------------------------- +// Mouse +// --------------------------------------------------------------------------- + +#[derive(Default)] +pub struct Mouse { + buttons: u8, + /// Movement not yet in a report. + dx: i32, + dy: i32, + queue: VecDeque>, +} + +impl Mouse { + /// Move accumulated motion into reports, at most a signed byte on each + /// axis apiece (HID 1.11 B.2: buttons, X, Y). `force` sends one even + /// with no motion, for a button change. + fn flush(&mut self, mut force: bool) { + while (force || self.dx != 0 || self.dy != 0) && self.queue.len() < QUEUE_LIMIT { + let (x, y) = (self.dx.clamp(-127, 127), self.dy.clamp(-127, 127)); + self.dx -= x; + self.dy -= y; + self.queue.push_back(vec![self.buttons, x as i8 as u8, y as i8 as u8]); + force = false; + } + } +} + +impl Boot for Mouse { + const PRODUCT: &'static str = "USB Mouse"; + const PROTOCOL: u8 = 2; + const REPORT_DESCRIPTOR: &'static [u8] = &MOUSE_REPORT_DESCRIPTOR; + const MAX_PACKET: u8 = 4; + // A mouse reports only on change. + const DEFAULT_IDLE: u8 = 0; + fn reset(&mut self) { + *self = Mouse::default(); + } + fn current(&self) -> Vec { + vec![self.buttons, 0, 0] + } + fn next_report(&mut self) -> Option> { + if self.queue.is_empty() { + self.flush(false); + } + self.queue.pop_front() + } + fn backlog(&self) -> usize { + self.queue.len() + usize::from(self.dx != 0 || self.dy != 0) + } +} + +pub type UsbMouse = UsbHid; + +impl UsbMouse { + pub fn new() -> Arc { + Self::with(Mouse::default()) + } +} + +impl HostInputSink for UsbMouse { + fn key(&self, _: KeyCode, _: bool) {} + fn mouse(&self, buttons: u8, dx: i32, dy: i32, _dz: i32) { + let mut s = self.inner.lock(); + if !s.configured { + return; + } + let m = &mut s.dev; + m.dx = m.dx.saturating_add(dx); + m.dy = m.dy.saturating_add(dy); + let buttons = buttons & 7; + // Motion up to now travels with the old button state. + if buttons != m.buttons { + m.flush(false); + m.buttons = buttons; + m.flush(true); + } else { + m.flush(false); + } + } + fn mouse_enabled(&self) -> bool { + self.inner.lock().configured + } + fn mouse_pending(&self) -> usize { + self.inner.lock().dev.backlog() + } +} + +#[cfg(test)] +mod tests; diff --git a/src/dev/usb_hid/tests.rs b/src/dev/usb_hid/tests.rs new file mode 100644 index 000000000..5efe1a8b2 --- /dev/null +++ b/src/dev/usb_hid/tests.rs @@ -0,0 +1,448 @@ +// The cases follow USB 1.1 chapter 9 and HID 1.11: descriptor layout and +// lengths, the control pipe's stages, and the boot report formats. The last +// ones put a device behind the OHCI model to show an interrupt transfer +// returns a report through guest memory. +use super::*; +use crate::dev::ohci::Ohci; +use crate::pci::{PciBusDevice, PciDma, PciDmaStatus}; +use crate::traits::Device; + +fn packet(endpoint: u8, pid: Pid, capacity: usize, data: &[u8], time_ms: u64) -> UsbPacket { + UsbPacket { endpoint, pid, toggle: false, capacity, data: data.to_vec(), time_ms } +} + +fn ack(o: UsbOutcome) -> Vec { + match o { + UsbOutcome::Ack(b) => b, + other => panic!("expected an ACK, got {other:?}"), + } +} + +/// A control transfer toward the host: SETUP, IN packets of 8 until short, +/// then the OUT status. +fn control_in(f: &dyn UsbFunction, rt: u8, req: u8, value: u16, index: u16, length: u16) -> Result, UsbOutcome> { + let mut setup = vec![rt, req]; + setup.extend(value.to_le_bytes()); + setup.extend(index.to_le_bytes()); + setup.extend(length.to_le_bytes()); + match f.transact(packet(0, Pid::Setup, 8, &setup, 0)) { + UsbOutcome::Ack(_) => {} + other => return Err(other), + } + let mut data = Vec::new(); + loop { + let chunk = ack(f.transact(packet(0, Pid::In, 8, &[], 0))); + let short = chunk.len() < 8; + data.extend(chunk); + if short || data.len() >= length as usize { + break; + } + } + ack(f.transact(packet(0, Pid::Out, 8, &[], 0))); + Ok(data) +} + +/// A control transfer with no data stage: SETUP then the IN status. +fn control_nodata(f: &dyn UsbFunction, rt: u8, req: u8, value: u16, index: u16) -> UsbOutcome { + let mut setup = vec![rt, req]; + setup.extend(value.to_le_bytes()); + setup.extend(index.to_le_bytes()); + setup.extend(0u16.to_le_bytes()); + match f.transact(packet(0, Pid::Setup, 8, &setup, 0)) { + UsbOutcome::Ack(_) => f.transact(packet(0, Pid::In, 8, &[], 0)), + other => other, + } +} + +fn enumerate(f: &dyn UsbFunction, address: u8) { + f.reset(); + assert!(ack(control_nodata(f, 0x00, SET_ADDRESS, address as u16, 0)).is_empty()); + assert_eq!(f.address(), address); + assert!(ack(control_nodata(f, 0x00, SET_CONFIGURATION, 1, 0)).is_empty()); +} + +fn interrupt(f: &dyn UsbFunction, time_ms: u64) -> UsbOutcome { + f.transact(packet(1, Pid::In, 8, &[], time_ms)) +} + +#[test] +fn the_device_descriptor_names_a_low_speed_hid_function() { + for f in [&*UsbKeyboard::new() as &dyn UsbFunction, &*UsbMouse::new()] { + assert!(f.low_speed()); + let d = control_in(f, 0x80, GET_DESCRIPTOR, 0x0100, 0, 64).unwrap(); + assert_eq!(d.len(), 18); + assert_eq!(d[0], 18, "bLength"); + assert_eq!(d[1], 1, "bDescriptorType"); + assert_eq!(u16::from_le_bytes([d[2], d[3]]), 0x0110, "USB 1.1"); + assert_eq!(d[4], 0, "the class is in the interface"); + assert_eq!(d[7], 8, "a low-speed control pipe carries 8 bytes"); + assert_eq!(d[17], 1, "one configuration"); + // The guest may read just the first 8 bytes before addressing. + assert_eq!(control_in(f, 0x80, GET_DESCRIPTOR, 0x0100, 0, 8).unwrap(), d[..8]); + } +} + +/// A configuration, its interface, the HID descriptor and the endpoint come +/// back in one block whose wTotalLength covers them (USB 1.1 9.6.2). +#[test] +fn the_configuration_block_describes_one_boot_interface_and_one_interrupt_endpoint() { + for (f, protocol, report_len, packet_size) in [ + (&*UsbKeyboard::new() as &dyn UsbFunction, 1u8, KEYBOARD_REPORT_DESCRIPTOR.len(), 8u8), + (&*UsbMouse::new(), 2, MOUSE_REPORT_DESCRIPTOR.len(), 4), + ] { + let c = control_in(f, 0x80, GET_DESCRIPTOR, 0x0200, 0, 255).unwrap(); + assert_eq!(c.len(), 34); + assert_eq!(&c[..2], [9, 2]); + assert_eq!(u16::from_le_bytes([c[2], c[3]]) as usize, c.len(), "wTotalLength"); + assert_eq!((c[4], c[5]), (1, 1), "one interface; configuration value 1"); + assert_eq!(&c[9..14], [9, 4, 0, 0, 1], "interface 0, no alternate, one endpoint"); + assert_eq!(&c[14..17], [3, 1, protocol], "HID class, boot subclass, keyboard or mouse"); + assert_eq!(&c[18..20], [9, 0x21], "HID descriptor"); + assert_eq!(u16::from_le_bytes([c[20], c[21]]), 0x0111, "bcdHID 1.11"); + assert_eq!(&c[24..26], [0x22, report_len as u8], "one report descriptor of its length"); + assert_eq!(&c[27..], [7, 5, 0x81, 3, packet_size, 0, 10], "interrupt IN 1, polled at 10 ms"); + // The HID descriptor is also readable on its own through the interface. + assert_eq!(control_in(f, 0x81, GET_DESCRIPTOR, 0x2100, 0, 9).unwrap(), c[18..27]); + } +} + +#[test] +fn the_report_descriptors_are_the_boot_descriptors() { + let k = UsbKeyboard::new(); + let r = control_in(&*k, 0x81, GET_DESCRIPTOR, 0x2200, 0, 255).unwrap(); + assert_eq!(r, KEYBOARD_REPORT_DESCRIPTOR); + // Usage page generic desktop, usage keyboard, application collection. + assert_eq!(&r[..6], [0x05, 0x01, 0x09, 0x06, 0xa1, 0x01]); + assert_eq!(r.last(), Some(&0xc0), "collection ends"); + let m = UsbMouse::new(); + let r = control_in(&*m, 0x81, GET_DESCRIPTOR, 0x2200, 0, 255).unwrap(); + assert_eq!(r, MOUSE_REPORT_DESCRIPTOR); + assert_eq!(&r[..6], [0x05, 0x01, 0x09, 0x02, 0xa1, 0x01]); +} + +#[test] +fn strings_are_utf16_with_a_language_table() { + let k = UsbKeyboard::new(); + assert_eq!(control_in(&*k, 0x80, GET_DESCRIPTOR, 0x0300, 0, 255).unwrap(), [4, 3, 0x09, 0x04]); + let s = control_in(&*k, 0x80, GET_DESCRIPTOR, 0x0302, 0x0409, 255).unwrap(); + assert_eq!(s[0] as usize, s.len()); + let text: Vec = s[2..].chunks(2).map(|c| u16::from_le_bytes([c[0], c[1]])).collect(); + assert_eq!(String::from_utf16(&text).unwrap(), "USB Keyboard"); + assert!(control_in(&*k, 0x80, GET_DESCRIPTOR, 0x0309, 0x0409, 255).is_err(), "no such string"); +} + +/// SET_ADDRESS takes effect after its status stage, not before (USB 1.1 9.4.6). +#[test] +fn set_address_applies_after_the_status_stage() { + let m = UsbMouse::new(); + let mut setup = vec![0x00, SET_ADDRESS, 9, 0, 0, 0, 0, 0]; + assert!(ack(m.transact(packet(0, Pid::Setup, 8, &setup, 0))).is_empty()); + assert_eq!(m.address(), 0, "still the default address during the status stage"); + assert!(ack(m.transact(packet(0, Pid::In, 8, &[], 0))).is_empty()); + assert_eq!(m.address(), 9); + // An address above 127 is not one. + setup[2] = 200; + assert_eq!(m.transact(packet(0, Pid::Setup, 8, &setup, 0)), UsbOutcome::Stall); + // A bus reset returns the device to the default address. + m.reset(); + assert_eq!(m.address(), 0); +} + +#[test] +fn configuration_state_is_readable_and_unknown_requests_stall() { + let k = UsbKeyboard::new(); + assert_eq!(control_in(&*k, 0x80, GET_CONFIGURATION, 0, 0, 1).unwrap(), [0]); + assert!(ack(control_nodata(&*k, 0x00, SET_CONFIGURATION, 1, 0)).is_empty()); + assert_eq!(control_in(&*k, 0x80, GET_CONFIGURATION, 0, 0, 1).unwrap(), [1]); + assert_eq!(control_nodata(&*k, 0x00, SET_CONFIGURATION, 2, 0), UsbOutcome::Stall); + assert_eq!(control_in(&*k, 0x81, GET_INTERFACE, 0, 0, 1).unwrap(), [0]); + assert_eq!(control_in(&*k, 0x80, GET_STATUS, 0, 0, 2).unwrap(), [0, 0]); + // No device qualifier on a low-speed-only device; no such request. + assert!(control_in(&*k, 0x80, GET_DESCRIPTOR, 0x0600, 0, 10).is_err()); + assert_eq!(control_nodata(&*k, 0x00, 0x77, 0, 0), UsbOutcome::Stall); + // A stalled request leaves the pipe usable for the next SETUP. + assert_eq!(control_in(&*k, 0x80, GET_CONFIGURATION, 0, 0, 1).unwrap(), [1]); + // The interrupt endpoint is not there before a configuration is selected. + let fresh = UsbKeyboard::new(); + assert_eq!(interrupt(&*fresh, 0), UsbOutcome::Stall); +} + +#[test] +fn hid_class_requests_set_and_report_idle_and_protocol() { + let k = UsbKeyboard::new(); + // A device starts in report protocol with the keyboard's 500 ms idle. + assert_eq!(control_in(&*k, 0xa1, GET_PROTOCOL, 0, 0, 1).unwrap(), [1]); + assert_eq!(control_in(&*k, 0xa1, GET_IDLE, 0, 0, 1).unwrap(), [125]); + assert!(ack(control_nodata(&*k, 0x21, SET_PROTOCOL, 0, 0)).is_empty()); + assert_eq!(control_in(&*k, 0xa1, GET_PROTOCOL, 0, 0, 1).unwrap(), [0]); + // SET_IDLE carries the duration in the high byte of wValue. + assert!(ack(control_nodata(&*k, 0x21, SET_IDLE, 0, 0)).is_empty()); + assert_eq!(control_in(&*k, 0xa1, GET_IDLE, 0, 0, 1).unwrap(), [0]); + assert_eq!(control_nodata(&*k, 0x21, SET_PROTOCOL, 2, 0), UsbOutcome::Stall); + // A mouse starts with no idle repeat. + let m = UsbMouse::new(); + assert_eq!(control_in(&*m, 0xa1, GET_IDLE, 0, 0, 1).unwrap(), [0]); +} + +#[test] +fn set_report_takes_the_keyboards_leds_and_a_mouse_has_none() { + let k = UsbKeyboard::new(); + let setup = [0x21, SET_REPORT, 0, 2, 0, 0, 1, 0]; + assert!(ack(k.transact(packet(0, Pid::Setup, 8, &setup, 0))).is_empty()); + assert!(ack(k.transact(packet(0, Pid::Out, 8, &[0x1f | 0x80], 0))).is_empty()); + assert!(ack(k.transact(packet(0, Pid::In, 8, &[], 0))).is_empty(), "status stage"); + assert_eq!(k.leds(), 0x1f, "five LED bits"); + let m = UsbMouse::new(); + assert!(ack(m.transact(packet(0, Pid::Setup, 8, &setup, 0))).is_empty()); + assert_eq!(m.transact(packet(0, Pid::Out, 8, &[1], 0)), UsbOutcome::Stall); +} + +#[test] +fn keyboard_reports_carry_modifiers_and_keys_in_press_order() { + let k = UsbKeyboard::new(); + enumerate(&*k, 3); + assert_eq!(interrupt(&*k, 1), UsbOutcome::Nak, "nothing to report yet"); + k.key(KeyCode::ShiftLeft, true); + k.key(KeyCode::KeyA, true); + k.key(KeyCode::KeyB, true); + k.key(KeyCode::KeyA, false); + k.key(KeyCode::ShiftLeft, false); + k.key(KeyCode::KeyB, false); + let reports: Vec> = (0..6).map(|i| ack(interrupt(&*k, i))).collect(); + assert_eq!(reports, [ + vec![0x02, 0, 0, 0, 0, 0, 0, 0], + vec![0x02, 0, 0x04, 0, 0, 0, 0, 0], + vec![0x02, 0, 0x04, 0x05, 0, 0, 0, 0], + vec![0x02, 0, 0x05, 0, 0, 0, 0, 0], + vec![0x00, 0, 0x05, 0, 0, 0, 0, 0], + vec![0x00, 0, 0, 0, 0, 0, 0, 0], + ], "a tap between polls still reports both edges"); + assert_eq!(interrupt(&*k, 7), UsbOutcome::Nak); +} + +#[test] +fn keyboard_usages_follow_the_usage_table() { + assert_eq!(usage(KeyCode::KeyA), Some(0x04)); + assert_eq!(usage(KeyCode::KeyZ), Some(0x1d)); + assert_eq!(usage(KeyCode::Digit1), Some(0x1e)); + assert_eq!(usage(KeyCode::Digit0), Some(0x27)); + assert_eq!(usage(KeyCode::Enter), Some(0x28)); + assert_eq!(usage(KeyCode::Escape), Some(0x29)); + assert_eq!(usage(KeyCode::Space), Some(0x2c)); + assert_eq!(usage(KeyCode::F1), Some(0x3a)); + assert_eq!(usage(KeyCode::F12), Some(0x45)); + assert_eq!(usage(KeyCode::ArrowRight), Some(0x4f)); + assert_eq!(usage(KeyCode::ArrowUp), Some(0x52)); + assert_eq!(usage(KeyCode::Numpad0), Some(0x62)); + assert_eq!(usage(KeyCode::ControlLeft), None, "modifiers have a report bit, not a usage slot"); + assert_eq!(modifier(KeyCode::ControlLeft), Some(0x01)); + assert_eq!(modifier(KeyCode::SuperRight), Some(0x80)); +} + +#[test] +fn a_seventh_key_reports_rollover_and_release_recovers() { + let k = UsbKeyboard::new(); + enumerate(&*k, 1); + let keys = [KeyCode::KeyA, KeyCode::KeyB, KeyCode::KeyC, KeyCode::KeyD, KeyCode::KeyE, KeyCode::KeyF, KeyCode::KeyG]; + for key in keys { + k.key(key, true); + } + let reports: Vec> = (0..7).map(|i| ack(interrupt(&*k, i))).collect(); + assert_eq!(reports[5], [0, 0, 4, 5, 6, 7, 8, 9]); + assert_eq!(reports[6], [0, 0, 1, 1, 1, 1, 1, 1], "ErrorRollOver"); + k.key(KeyCode::KeyG, false); + assert_eq!(ack(interrupt(&*k, 8)), [0, 0, 4, 5, 6, 7, 8, 9]); + // A repeated press of a held key is not a new edge. + k.key(KeyCode::KeyA, true); + assert_eq!(interrupt(&*k, 9), UsbOutcome::Nak); +} + +#[test] +fn input_before_configuration_is_dropped_and_get_report_reads_the_state() { + let k = UsbKeyboard::new(); + k.key(KeyCode::KeyA, true); + enumerate(&*k, 2); + assert_eq!(interrupt(&*k, 1), UsbOutcome::Nak, "the early key never reached the report queue"); + k.key(KeyCode::KeyQ, true); + assert_eq!(control_in(&*k, 0xa1, GET_REPORT, 0x0100, 0, 8).unwrap(), [0, 0, 0x14, 0, 0, 0, 0, 0]); +} + +/// With a nonzero idle rate the keyboard repeats its state when the period +/// passes without a change (HID 1.11 7.2.4), and with zero it stays quiet. +#[test] +fn the_idle_rate_repeats_the_last_report() { + let k = UsbKeyboard::new(); + enumerate(&*k, 2); + k.key(KeyCode::KeyA, true); + assert_eq!(ack(interrupt(&*k, 10)), [0, 0, 4, 0, 0, 0, 0, 0]); + assert_eq!(interrupt(&*k, 509), UsbOutcome::Nak, "500 ms not yet passed"); + assert_eq!(ack(interrupt(&*k, 510)), [0, 0, 4, 0, 0, 0, 0, 0]); + assert!(ack(control_nodata(&*k, 0x21, SET_IDLE, 0, 0)).is_empty()); + assert_eq!(interrupt(&*k, 5000), UsbOutcome::Nak); +} + +#[test] +fn mouse_reports_carry_buttons_and_signed_motion() { + let m = UsbMouse::new(); + enumerate(&*m, 4); + assert!(m.mouse_enabled()); + assert_eq!(interrupt(&*m, 1), UsbOutcome::Nak); + // Host dy is positive down the screen, as HID Y is. + m.mouse(0, 10, -20, 0); + assert_eq!(ack(interrupt(&*m, 2)), [0, 10, (-20i8) as u8]); + // A press travels with its own report, then the release. + m.mouse(1, 0, 0, 0); + m.mouse(0, 0, 0, 0); + assert_eq!(ack(interrupt(&*m, 3)), [1, 0, 0]); + assert_eq!(ack(interrupt(&*m, 4)), [0, 0, 0]); + // Buttons in PS/2 order: right is 2, middle is 4. + m.mouse(2 | 4, 0, 0, 0); + assert_eq!(ack(interrupt(&*m, 5)), [6, 0, 0]); + assert_eq!(control_in(&*m, 0xa1, GET_REPORT, 0x0100, 0, 3).unwrap(), [6, 0, 0]); + // Boot protocol has no wheel byte: the report stays 3 bytes. + m.mouse(6, 0, 0, 3); + assert_eq!(interrupt(&*m, 6), UsbOutcome::Nak); +} + +#[test] +fn large_motion_splits_across_reports_without_loss() { + let m = UsbMouse::new(); + enumerate(&*m, 4); + m.mouse(0, 300, -200, 0); + assert_eq!(m.mouse_pending(), 3); + let mut sum = (0i32, 0i32); + let mut n = 0; + while let UsbOutcome::Ack(r) = interrupt(&*m, n) { + sum.0 += r[1] as i8 as i32; + sum.1 += r[2] as i8 as i32; + assert!(r[1] as i8 != -128 && r[2] as i8 != -128); + n += 1; + } + assert_eq!(sum, (300, -200)); + assert_eq!(m.mouse_pending(), 0); +} + +#[test] +fn a_mouse_before_configuration_is_not_enabled_and_drops_motion() { + let m = UsbMouse::new(); + assert!(!m.mouse_enabled()); + m.mouse(1, 5, 5, 0); + assert_eq!(m.mouse_pending(), 0); + enumerate(&*m, 1); + assert_eq!(interrupt(&*m, 1), UsbOutcome::Nak); +} + +/// Host input arrives through the PS/2 controller's sinks, so the one +/// interface the GUI, VNC and CI use feeds both kinds of device. +#[test] +fn host_input_reaches_usb_devices_through_the_ps2_controller() { + use crate::dev::ps2::Ps2Controller; + let ps2 = Ps2Controller::new(None); + Device::start(&ps2); + let (k, m) = (UsbKeyboard::new(), UsbMouse::new()); + ps2.add_sink(k.clone()); + ps2.add_sink(m.clone()); + enumerate(&*k, 1); + enumerate(&*m, 2); + assert!(ps2.mouse_enabled(), "the USB mouse counts: the PS/2 mouse is still off"); + ps2.push_kb(KeyCode::KeyA, true); + ps2.type_char('b'); + ps2.push_mouse_input(1, 3, 4, 0); + assert_eq!(ps2.mouse_pending(), 2, "a move and a button change wait for the guest"); + assert_eq!(ack(interrupt(&*k, 1)), [0, 0, 4, 0, 0, 0, 0, 0]); + assert_eq!(ack(interrupt(&*k, 2)), [0, 0, 4, 5, 0, 0, 0, 0]); + assert_eq!(ack(interrupt(&*m, 1)), [0, 3, 4]); + assert_eq!(ack(interrupt(&*m, 2)), [1, 0, 0]); +} + +// ---- Through the OHCI model ------------------------------------------------ + +struct Memory(Mutex>); +impl Memory { + fn new() -> Arc { + Arc::new(Self(Mutex::new(vec![0xa5; 0x10000]))) + } + fn words(&self, at: usize, words: &[u32]) { + for (i, w) in words.iter().enumerate() { + self.0.lock()[at + i * 4..at + i * 4 + 4].copy_from_slice(&w.to_le_bytes()); + } + } + fn word(&self, at: usize) -> u32 { + u32::from_le_bytes(self.0.lock()[at..at + 4].try_into().unwrap()) + } +} +impl PciDma for Memory { + fn dma_read(&self, addr: u64, dst: &mut [u8]) -> PciDmaStatus { + match self.0.lock().get(addr as usize..addr as usize + dst.len()) { + Some(s) => { + dst.copy_from_slice(s); + PciDmaStatus::Ok + } + None => PciDmaStatus::MasterAbort, + } + } + fn dma_write(&self, addr: u64, src: &[u8]) -> PciDmaStatus { + match self.0.lock().get_mut(addr as usize..addr as usize + src.len()) { + Some(s) => { + s.copy_from_slice(src); + PciDmaStatus::Ok + } + None => PciDmaStatus::MasterAbort, + } + } +} + +/// An OHCI controller with the device on port 0, reset and enabled, with +/// the periodic list holding one interrupt-IN endpoint at `address`. +fn ohci_with(f: Arc, address: u8) -> (Arc, Arc) { + let h = Ohci::new(); + let m = Memory::new(); + h.connect_dma(m.clone()); + h.attach(0, f.clone()).unwrap(); + h.start(); + h.bar_write(0, 0x54, 0x100, 4); + h.bar_write(0, 0x54, 0x10, 4); + h.advance_frames(10); + // The port reset returned the device to its default state: enumerate it. + enumerate(&*f, address); + m.words(0x1000, &[0; 64]); + // Interrupt table: every slot points at the endpoint. ED: address, endpoint 1, + // direction from the TD, low speed, max packet 8; TD queue 0x3000 .. tail 0x3020. + m.words(0x1000, &[0x2000; 32]); + m.words(0x2000, &[u32::from(address) | (1 << 7) | (1 << 13) | (8 << 16), 0x3020, 0x3000, 0]); + // TD: direction IN (2), buffer 0x4000..0x4007, buffer rounding allowed, interrupt delay 0. + m.words(0x3000, &[0xf000_0000 | (2 << 19) | (1 << 18), 0x4000, 0x3020, 0x4007]); + h.config().write(1, 6); + h.bar_write(0, 0x18, 0x1000, 4); + h.bar_write(0, 0x04, 0x84, 4); + (h, m) +} + +/// A report waiting in the device is carried by an interrupt TD into guest +/// memory, with the TD retired as a success. +#[test] +fn an_interrupt_transfer_returns_a_keyboard_report_through_ohci() { + let k = UsbKeyboard::new(); + let (h, m) = ohci_with(k.clone(), 5); + k.key(KeyCode::KeyA, true); + h.advance_frames(2); + let report = m.0.lock()[0x4000..0x4009].to_vec(); + // The done queue is published once the delay for a low-priority completion passes. + h.advance_frames(10); + assert_eq!(report, vec![0, 0, 4, 0, 0, 0, 0, 0, 0xa5], "the report, and nothing past its end"); + assert_eq!(m.word(0x3000) >> 28, 0, "completed without error"); + assert_eq!(m.word(0x1084) & !15, 0x3000, "the TD is on the done queue"); +} + +/// With nothing to report the device NAKs and the transfer stays queued. +#[test] +fn an_interrupt_transfer_with_no_report_stays_pending() { + let m = UsbMouse::new(); + let (h, mem) = ohci_with(m.clone(), 5); + h.advance_frames(3); + assert_eq!(mem.word(0x3000) >> 28, 0xf, "not accessed"); + m.mouse(2, 1, 1, 0); + h.advance_frames(2); + assert_eq!(&mem.0.lock()[0x4000..0x4003], [0, 1, 1]); + assert_eq!(mem.word(0x3000) >> 28, 0); +} diff --git a/src/devlog.rs b/src/devlog.rs index 406858d4f..045cd6f4f 100644 --- a/src/devlog.rs +++ b/src/devlog.rs @@ -57,6 +57,7 @@ pub enum LogModule { Odyssey = 28, Bedrock = 29, L1 = 30, + Usb = 31, } // Mask bits shared by L1i / L1d / L2c @@ -65,7 +66,7 @@ pub const CACHE_LOG_MISS: u32 = 0x2; // log misses and fills pub const CACHE_LOG_OP: u32 = 0x4; // log CACHE instruction ops impl LogModule { - pub const COUNT: usize = 31; + pub const COUNT: usize = 32; pub fn from_str(s: &str) -> Option { match s { @@ -100,6 +101,7 @@ impl LogModule { "odyssey" => Some(Self::Odyssey), "bedrock" => Some(Self::Bedrock), "l1" => Some(Self::L1), + "usb" => Some(Self::Usb), _ => None, } } @@ -137,6 +139,7 @@ impl LogModule { Self::Odyssey => "odyssey", Self::Bedrock => "bedrock", Self::L1 => "l1", + Self::Usb => "usb", } } @@ -147,7 +150,7 @@ impl LogModule { Self::Vino, Self::Dcb, Self::Vc2, Self::Cmap, Self::Xmap, Self::Bt445, Self::Scc, Self::Ps2, Self::Rtc, Self::Eeprom, Self::L1i, Self::L1d, Self::L2c, Self::Ultra, Self::Scsi1, - Self::Nveeprom, Self::Gr2, Self::Rad1, Self::Odyssey, Self::Bedrock, Self::L1, + Self::Nveeprom, Self::Gr2, Self::Rad1, Self::Odyssey, Self::Bedrock, Self::L1, Self::Usb, ] } @@ -291,6 +294,7 @@ impl DevLog { ModuleLog::new(), // Odyssey ModuleLog::new(), // Bedrock ModuleLog::new(), // L1 + ModuleLog::new(), // Usb ], } } From 3bedd6c4a7c744ce494af686cbe111e89a829c62 Mon Sep 17 00:00:00 2001 From: James Braid Date: Fri, 30 Oct 2026 08:13:37 +0000 Subject: [PATCH 27/31] fuel: add the serial IP35 machine Build the Fuel IP34 board around its R14000, Bedrock, XBridge, IOC3 and dual-channel ISP12160. Use one Physical memory table with sparse processor and peripheral views, and retain the CPU memory fast paths after the PROM startup store retires. Uncached byte and halfword stores reach a device through BusDevice::write_narrow_from, which each window forwards at the device-local address; only the memory controller's ECC view reads more than the low bits of the register. With the jitv2 feature, start the compile pool after the memory windows are published, as the Octane and O2 do. Each worker copies the cache geometry once as it starts, and without the windows that geometry has no inline load or store path. Workers read code through the cached plane, which serves plain physical addresses; the uncached views would decode them as node registers. The machine describes where its memory can answer to the CPU: the four 1 GiB slots Bedrock's controller places its two 256 MiB banks in, and the reset PROM. Its disks on both ISP12160 channels are flushed at power-off, quit and exit like the other machines', so a copy-on-write overlay keeps its record of held sectors after a guest `halt -p`. The processor is R14000FuelShadowCache, the Octane's R14000 at revision 2.3 (PRId 0x0f23), the revision a public software inventory of a Fuel prints, where the Octane's part reports 2.4. The raw CP0 word, the FIR and the 128-byte lines are taken from the Octane part, not read from a Fuel. Like the Octane's R14000 it reports "R12000" as its name. The machine loads its PROM through prom_image. Feed the IOC3 PS/2 controller from the window, VNC and CI input, and let a USB keyboard on root port 0 and a USB mouse on port 1 share it. The Fuel's manual shows PS/2 as the standard keyboard and mouse, so ps2_input under [fuel] defaults on and usb_input defaults off; setting usb_input fits the USB pair, and ps2_input = false leaves the PS/2 pair out. The monitor's ps2 command reaches the keyboard and mouse too. PROM loader (prom_image): Add a device-independent loader for PROM images. It accepts a raw payload or a version-six stored or gzip container, decodes the recognized containers explicitly and reports malformed inputs with their path before any bytes reach guest memory. Config profile (config): The Fuel is a new machine profile with two memory banks, wide SCSI buses, no Ethernet adapter choice beyond its built-in one, and no graphics board by default. The [fuel] section selects the serial console (the L1 controller or either IOC3 port), the L1 TCP port and log, and the persistent boot flash store. l1_revision and brick_serial replace the L1's reported firmware revision (default 1.48.1) and brick serial (default EMUN00). ps2_input defaults on and usb_input off, as the Fuel's manual shows PS/2 as the standard keyboard and mouse; setting usb_input fits a USB keyboard and mouse, and ps2_input = false leaves the PS/2 pair out. The key names the part: machine.cpu = "r14000" (r10000, the family, is still accepted), and a Fuel config cannot claim another processor. The Octane's R14000 is revision 2.4; the Fuel's is 2.3. --- HELP.md | 6 +- src/config.rs | 278 ++++++++- src/cpu/mips_cache_shadow.rs | 8 + src/dev/l1.rs | 5 + src/dev/mod.rs | 1 + src/dev/prom_image.rs | 210 +++++++ src/ip35.rs | 368 ++++++++++++ src/ip35/tests.rs | 623 +++++++++++++++++++ src/ip35_machine.rs | 1096 ++++++++++++++++++++++++++++++++++ src/lib.rs | 3 + src/machine.rs | 8 +- src/workstation.rs | 1 + 12 files changed, 2580 insertions(+), 27 deletions(-) create mode 100644 src/dev/prom_image.rs create mode 100644 src/ip35.rs create mode 100644 src/ip35/tests.rs create mode 100644 src/ip35_machine.rs diff --git a/HELP.md b/HELP.md index 56c599dd0..fa25adf65 100644 --- a/HELP.md +++ b/HELP.md @@ -1,7 +1,9 @@ -# IRIS — SGI Indy and Indigo2 emulator +# IRIS — SGI workstation emulator The complete core/GUI feature and configuration inventory is in -[FEATURES.md](FEATURES.md). This guide explains everyday use; the monitor's full command reference is in [debug.md](debug.md). +[FEATURES.md](FEATURES.md). For the Fuel, see the [Fuel guide](docs/fuel-ip34.md). +This guide explains everyday use. The monitor's full command reference is in +[debug.md](debug.md). ## Quick start diff --git a/src/config.rs b/src/config.rs index b4e312a01..d8540d25d 100644 --- a/src/config.rs +++ b/src/config.rs @@ -612,6 +612,8 @@ pub enum MachineProfile { /// R10000. Configured by the shared keys and the `[octane]` section; see /// `src/ip30_machine.rs` and `docs/Octane/ip30-address-map.md`. OctaneIp30, + /// Serial-only Fuel IP34 board using the IP35 software platform. + FuelIp34, } impl MachineProfile { @@ -634,6 +636,7 @@ impl MachineProfile { Self::OctaneIp30 => &[0, 64, 128, 256, 512, 1024, 2048], Self::O2Ip32 => &[0, 32, 64, 128, 256], Self::Indigo2Ip28 => MC_BANK_SIZES, + Self::FuelIp34 => &[0, 256], _ => &[0, 8, 16, 32, 64, 128], } } @@ -657,12 +660,13 @@ impl MachineProfile { /// Every profile, in the order `workstation::build` lists them. `ALL` is /// only the ones the GUI offers. - pub const EVERY: [Self; 5] = [ + pub const EVERY: [Self; 6] = [ Self::IndyIp24, Self::Indigo2Ip22, Self::Indigo2Ip28, Self::O2Ip32, Self::OctaneIp30, + Self::FuelIp34, ]; /// The name `[machine] profile` and `--machine` give this profile. @@ -673,6 +677,7 @@ impl MachineProfile { Self::Indigo2Ip28 => "indigo2_ip28", Self::O2Ip32 => "o2_ip32", Self::OctaneIp30 => "octane_ip30", + Self::FuelIp34 => "fuel_ip34", } } @@ -707,8 +712,9 @@ impl MachineProfile { } /// Whether this profile can have `cpu`, and why not if it cannot. The - /// Octane takes an R10000, R12000 or R14000 and the IP28 an R10000; the - /// O2 constructor still refuses its own, after `validate()`. + /// Octane takes an R10000, R12000 or R14000, the IP28 an R10000 and the + /// Fuel an R14000; the O2 constructor still refuses its own, after + /// `validate()`. pub fn accepts_cpu(self, cpu: CpuModel) -> Result<(), String> { match self { Self::OctaneIp30 if !matches!(cpu, CpuModel::R10000 | CpuModel::R12000 | CpuModel::R14000) => Err( @@ -719,6 +725,11 @@ impl MachineProfile { { Err("an Indigo2 IMPACT (IP28) has an R10000 ([machine] cpu = \"r10000\")".into()) } + // The machine always models its one R14000; `r10000` names the + // family, as the key did before the R14000 could be named. + Self::FuelIp34 if !matches!(cpu, CpuModel::R10000 | CpuModel::R14000) => { + Err("a Fuel has an R14000: set [machine] cpu = \"r14000\"".into()) + } _ => Ok(()), } } @@ -730,6 +741,7 @@ impl MachineProfile { Self::Indigo2Ip28 => "SGI Indigo2 IMPACT (IP28)", Self::O2Ip32 => "SGI O2 (IP32)", Self::OctaneIp30 => "SGI Octane (IP30)", + Self::FuelIp34 => "SGI Fuel (IP34)", } } @@ -739,7 +751,7 @@ impl MachineProfile { /// connector), one on the Indy. pub fn scsi_controllers(self) -> u8 { match self { - Self::Indigo2Ip22 | Self::O2Ip32 | Self::OctaneIp30 => 2, + Self::Indigo2Ip22 | Self::O2Ip32 | Self::OctaneIp30 | Self::FuelIp34 => 2, _ => 1, } } @@ -748,18 +760,19 @@ impl MachineProfile { /// buses are wide, with ids 0 to 15 and the controller on 0. The Indy's /// and Indigo2's WD33C93A chips are narrow, with ids 0 to 7. The O2's /// Adaptec buses are wide too, but stay at 7 until they are widened on - /// purpose. + /// purpose. The Fuel's ISP12160 buses are wide. pub fn max_scsi_id(self) -> u8 { match self { - Self::OctaneIp30 => 15, + Self::OctaneIp30 | Self::FuelIp34 => 15, _ => 7, } } - /// How many memory banks the machine has: four on all of them (the MC's - /// four banks; the O2's four DIMM pairs). `banks` has room for more. + /// How many memory banks the machine has: four on the MC, HEART and O2 + /// (the O2's four DIMM pairs), two on Fuel. `banks` has room for more; + /// unused entries must be zero. pub fn memory_banks(self) -> usize { - 4 + if self == Self::FuelIp34 { 2 } else { 4 } } pub fn guinness(self) -> bool { @@ -769,8 +782,9 @@ impl MachineProfile { /// The CPU models this machine can be built with. The O2 has an R5000, /// an R10000 or an R12000 (the others are refused when the machine is /// built, not by `validate()`); the Octane has an R10000, R12000 or - /// R14000; the Indy and the Indigo2 take any part, except that the IP28 - /// is refused anything but the R10000 by `validate()`. + /// R14000; the Fuel has an R14000 (or the R10000 family key); the Indy and + /// the Indigo2 take any part, except that the IP28 is refused anything but + /// the R10000 by `validate()`. /// /// Single source of truth for the front-ends' Processor pickers, so they /// cannot offer a CPU the machine will refuse. @@ -778,6 +792,7 @@ impl MachineProfile { match self { Self::OctaneIp30 => &[CpuModel::R10000, CpuModel::R12000, CpuModel::R14000], Self::O2Ip32 => &[CpuModel::R5000, CpuModel::R10000, CpuModel::R12000], + Self::FuelIp34 => &[CpuModel::R10000, CpuModel::R14000], _ => &CpuModel::ALL, } } @@ -828,7 +843,7 @@ pub enum GraphicsBoard { #[clap(name = "v12", alias = "vpro:v12", alias = "vpro_v12")] V12, /// No graphics board: the machine runs on its serial console. Octane - /// only. + /// and Fuel. #[clap(name = "none")] None, } @@ -890,6 +905,7 @@ impl GraphicsBoard { pub fn supports(self, profile: MachineProfile) -> bool { match profile { MachineProfile::OctaneIp30 => self.is_impact() || self.is_vpro() || self == Self::None, + MachineProfile::FuelIp34 => self == Self::None, MachineProfile::Indigo2Ip22 => !self.is_vpro() && self != Self::None, _ => !self.is_vpro() && !matches!(self, Self::Extreme | Self::None), } @@ -899,6 +915,7 @@ impl GraphicsBoard { pub fn default_for(profile: MachineProfile) -> Self { match profile { MachineProfile::OctaneIp30 => Self::SolidImpact, + MachineProfile::FuelIp34 => Self::None, _ => Self::Newport, } } @@ -958,7 +975,7 @@ impl std::fmt::Display for GraphicsBoard { #[serde(deny_unknown_fields)] pub struct GraphicsSection { /// Graphics board: Newport, GR2 XZ/Extreme, IMPACT Solid/High/Maximum, - /// or none (Octane only). Unset: the machine's default + /// or none (Octane and Fuel). Unset: the machine's default /// ([`GraphicsBoard::default_for`]); read it with /// [`MachineConfig::graphics_board`]. Extreme is restricted to IP22, /// the Octane to IMPACT or none; non-Newport boards require one head @@ -1046,8 +1063,8 @@ pub enum CpuModel { /// MIPS R12000: the R10000's programming model. The Octane and the O2 /// have it as revision 3.5 with a 2 MB secondary cache. R12000, - /// MIPS R14000, revision 2.4, with a 2 MB secondary cache. Selectable on - /// the Octane. + /// MIPS R14000 with a 2 MB secondary cache: revision 2.4 on the Octane, + /// 2.3 on the Fuel. Selectable on those two. R14000, /// MIPS R16000, revision 2.2, 800 MHz part, with a 4 MB secondary cache. /// It lands ahead of the machines that use it; `iris-bench --cpu r16000` @@ -1655,6 +1672,80 @@ mod scsi_keys { } } +/// Fuel's independent serial connections. The selected console serves CI and serial_log. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum FuelConsole { + #[default] + L1, + Ioc3A, + Ioc3B, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct FuelSection { + /// Persistent 16 MiB boot flash store; absent means volatile flash. + #[serde(default)] + pub flash: Option, + #[serde(default)] + pub console: FuelConsole, + /// L1 host console TCP port; IOC3 keeps serial_port_a and serial_port_b. + #[serde(default)] + pub l1_port: Option, + /// Separate L1 log when an IOC3 port is selected as console. + #[serde(default)] + pub l1_log: Option, + /// Fit the PS/2 keyboard and mouse controller in the onboard IOC3 and + /// feed it the host's keyboard and mouse. + #[serde(default = "default_fuel_input")] + pub ps2_input: bool, + /// Plug a USB keyboard and mouse into the OHCI root ports and feed them + /// the same host input. Off by default: the Fuel's manual and the PROM + /// treat PS/2 as the standard keyboard and mouse. + #[serde(default)] + pub usb_input: bool, + /// L1 firmware revision as "major.minor.patch", reported to request + /// 0x0005. Unset: 1.48.1. + #[serde(default)] + pub l1_revision: Option, + /// Brick serial reported to L1 request 0x0004, at most 37 bytes. Unset: + /// the marked synthetic `EMUN00`. + #[serde(default)] + pub brick_serial: Option, +} + +fn default_fuel_input() -> bool { true } + +impl Default for FuelSection { + fn default() -> Self { + Self { + flash: None, + console: FuelConsole::default(), + l1_port: None, + l1_log: None, + ps2_input: true, + usb_input: false, + l1_revision: None, + brick_serial: None, + } + } +} + +impl FuelSection { + /// Left out of a written config when nothing in it was set. + fn is_default(&self) -> bool { + self.flash.is_none() + && self.console == FuelConsole::default() + && self.l1_port.is_none() + && self.l1_log.is_none() + && self.ps2_input + && !self.usb_input + && self.l1_revision.is_none() + && self.brick_serial.is_none() + } +} + /// Top-level machine configuration. /// /// Field order matters for TOML export: the `toml` serializer requires every @@ -1895,6 +1986,10 @@ pub struct MachineConfig { #[serde(default, skip_serializing_if = "OctaneSection::is_default")] pub octane: OctaneSection, + /// Serial console and flash storage for the Fuel. + #[serde(default, skip_serializing_if = "FuelSection::is_default")] + pub fuel: FuelSection, + /// Host performance tuning (`[perf]` section). #[serde(default)] pub perf: PerfConfig, @@ -2122,6 +2217,7 @@ impl Default for MachineConfig { jitv2: Jitv2Config::default(), o2: O2Section::default(), octane: OctaneSection::default(), + fuel: FuelSection::default(), perf: PerfConfig::default(), clock: ClockConfig::default(), rtc_offset: RtcOffset::default(), @@ -2171,6 +2267,9 @@ impl MachineConfig { } // The Octane has a PCI bridge with a free slot. MachineProfile::OctaneIp30 => Ok(()), + MachineProfile::FuelIp34 => Err( + "[virtio_serial] is not wired on the Fuel: its XBridge has no virtio-pci attachment".into(), + ), MachineProfile::O2Ip32 => Err( "[virtio_serial] is not wired on the O2 yet: the virtio-pci transport exists, but the \ MACE PCI bridge has no interrupt route for a third slot (see docs/virtio-serial.md)" @@ -2320,7 +2419,7 @@ impl MachineConfig { /// Validate bank sizes, returns a description of any errors. /// The graphics board: `[graphics] board`, or the machine's default - /// when it is not set (Solid IMPACT on the Octane, Newport elsewhere). + /// (Solid IMPACT on Octane, none on Fuel, Newport elsewhere). pub fn graphics_board(&self) -> GraphicsBoard { self.graphics.board.unwrap_or_else(|| GraphicsBoard::default_for(self.machine.profile)) } @@ -2387,9 +2486,9 @@ impl MachineConfig { "scsi.{} and scsi.{} both name {}", other, key, scsi_label(controller, id) )); } - if dev.is_daynaport() && self.machine.profile == MachineProfile::O2Ip32 { + if dev.is_daynaport() && matches!(self.machine.profile, MachineProfile::O2Ip32 | MachineProfile::FuelIp34) { return Err(format!( - "scsi.{}: kind = \"daynaport\" is not available on the O2 (its Ethernet is built in)", key + "scsi.{}: kind = \"daynaport\" is not available on {} (its Ethernet is built in)", key, self.machine.profile.label() )); } if dev.is_daynaport() && self.network.mode == NetMode::Stream { @@ -2424,6 +2523,19 @@ impl MachineConfig { } } crate::dev::crime_mem::check_kits(&self.o2_kits()).map_err(|e| format!("banks: {e}"))?; + } else if self.machine.profile == MachineProfile::FuelIp34 { + if self.banks[..2] != [256, 256] { + return Err("Fuel currently requires banks = [256, 256] (512 MiB)".into()); + } + if board != GraphicsBoard::None { + return Err("Fuel currently supports graphics.board = \"none\" only".into()); + } + if let Some(text) = &self.fuel.l1_revision { + crate::dev::l1::parse_revision(text).map_err(|e| format!("fuel.l1_revision: {e}"))?; + } + if self.fuel.brick_serial.as_ref().is_some_and(|s| s.len() > 37 || s.contains('\0')) { + return Err("fuel.brick_serial must be at most 37 bytes with no NUL".into()); + } } else if self.machine.profile == MachineProfile::OctaneIp30 { // A HEART bank is a matched DIMM pair: 64 MB to 2 GB, filled from // bank 0 without a gap, 8 GB at most (SGI's memory upgrade guide). @@ -2469,8 +2581,8 @@ impl MachineConfig { } } match (self.network.mode, self.machine.profile) { - (NetMode::Off, p) if p != MachineProfile::O2Ip32 && p != MachineProfile::OctaneIp30 => { - return Err("network.mode = \"off\" is only available on the O2 and the Octane for now; the Indy and Indigo2 always have their Ethernet".into()); + (NetMode::Off, p) if p != MachineProfile::O2Ip32 && p != MachineProfile::OctaneIp30 && p != MachineProfile::FuelIp34 => { + return Err("network.mode = \"off\" is only available on the O2, Octane and Fuel; the Indy and Indigo2 always have their Ethernet".into()); } (NetMode::Stream, _) => { let ep = self.network.stream.as_deref() @@ -3458,7 +3570,7 @@ mod export_tests { assert_eq!(apply(&["--ip22"]).machine.profile, MachineProfile::Indigo2Ip22); assert!(Cli::try_parse_from(["iris", "--ip22", "--machine", "o2_ip32"]).is_err()); let e = MachineProfile::from_name("vax").unwrap_err(); - assert_eq!(e, "unknown machine \"vax\" (valid: indy_ip24, indigo2_ip22, indigo2_ip28, o2_ip32, octane_ip30)"); + assert_eq!(e, "unknown machine \"vax\" (valid: indy_ip24, indigo2_ip22, indigo2_ip28, o2_ip32, octane_ip30, fuel_ip34)"); let list = MachineProfile::list(); assert_eq!(list.lines().count(), MachineProfile::EVERY.len()); assert!(list.starts_with("indy_ip24 SGI Indy (IP24)\n"), "{list}"); @@ -3467,7 +3579,7 @@ mod export_tests { for p in MachineProfile::EVERY { match p { MachineProfile::IndyIp24 | MachineProfile::Indigo2Ip22 | MachineProfile::Indigo2Ip28 - | MachineProfile::O2Ip32 | MachineProfile::OctaneIp30 => {} + | MachineProfile::O2Ip32 | MachineProfile::OctaneIp30 | MachineProfile::FuelIp34 => {} } let cfg: MachineConfig = toml::from_str(&format!("[machine]\nprofile = \"{}\"", p.name())).unwrap(); assert_eq!(cfg.machine.profile, p); @@ -3482,6 +3594,7 @@ mod export_tests { let accepted = |p: MachineProfile| CpuModel::ALL.into_iter().filter(|&c| p.accepts_cpu(c).is_ok()).collect::>(); assert_eq!(accepted(MachineProfile::OctaneIp30), [CpuModel::R10000, CpuModel::R12000, CpuModel::R14000]); assert_eq!(accepted(MachineProfile::IndyIp24), CpuModel::ALL); + assert_eq!(accepted(MachineProfile::FuelIp34), [CpuModel::R10000, CpuModel::R14000]); // The counts are what the checks in validate() and the Octane // constructor allowed before they moved here: one on every machine @@ -3599,6 +3712,11 @@ mod export_tests { cfg.machine.cpu = CpuModel::R10000; cfg.validate().expect("the IP28 takes the R10000"); + // The Fuel's part check is accepts_cpu's, and validate() reports it. + cfg.machine.profile = MachineProfile::FuelIp34; + cfg.machine.cpu = CpuModel::R12000; + assert_eq!(cfg.validate().unwrap_err(), "a Fuel has an R14000: set [machine] cpu = \"r14000\""); + // Both machines check the part before they look for a PROM. let refusal = |profile, cpu| { let mut cfg = MachineConfig::default(); @@ -3968,3 +4086,119 @@ mod ci_overlay_tests { assert!(a.contains(&format!("iris-ci-{}-", std::process::id())), "{a}"); } } + +#[cfg(test)] +mod fuel_input_tests { + use super::*; + + /// A config saved for another machine must not grow a `[fuel]` table, + /// and a Fuel's own settings must survive a save and reload. + #[test] + fn only_a_configured_fuel_section_is_saved_and_it_round_trips() { + let other = MachineConfig::default(); + let saved = toml::to_string(&other).unwrap(); + assert!(!saved.contains("[fuel]"), "{saved}"); + let mut fuel = MachineConfig::default(); + fuel.fuel.console = FuelConsole::Ioc3B; + fuel.fuel.flash = Some("flash.bin".into()); + fuel.fuel.l1_port = Some(18882); + fuel.fuel.l1_log = Some("l1.log".into()); + fuel.fuel.usb_input = true; + fuel.fuel.ps2_input = false; + let saved = toml::to_string(&fuel).unwrap(); + assert!(saved.contains("[fuel]"), "{saved}"); + let loaded: MachineConfig = toml::from_str(&saved).unwrap(); + assert_eq!(loaded.fuel.console, FuelConsole::Ioc3B); + assert_eq!(loaded.fuel.flash.as_deref(), Some("flash.bin")); + assert_eq!(loaded.fuel.l1_port, Some(18882)); + assert_eq!(loaded.fuel.l1_log.as_deref(), Some("l1.log")); + assert!(!loaded.fuel.ps2_input && loaded.fuel.usb_input); + assert!(!MachineProfile::ALL.contains(&MachineProfile::FuelIp34)); + } + + /// PS/2 is the Fuel's standard keyboard and mouse; USB is opt-in, and a + /// `[fuel]` table that names neither key keeps both defaults. + #[test] + fn fuel_input_defaults_to_ps2_with_usb_opt_in() { + let none: MachineConfig = toml::from_str("[fuel]\nflash = \"flash.bin\"\n").unwrap(); + assert!(none.fuel.ps2_input && !none.fuel.usb_input); + assert!(MachineConfig::default().fuel.is_default()); + let usb: MachineConfig = toml::from_str("[fuel]\nusb_input = true\n").unwrap(); + assert!(usb.fuel.ps2_input && usb.fuel.usb_input); + assert!(!usb.fuel.is_default(), "an opt-in is written back"); + } + + /// The Fuel's processor is fixed, so `machine.cpu` may only name it (or the + /// family, as the key did before the R14000 had a name); the default + /// `r4400` is refused rather than silently ignored. + /// The L1 identity keys parse, validate and round trip; unset they are + /// left out of a written config. + #[test] + fn fuel_l1_identity_keys_are_validated_and_round_trip() { + let base = "banks = [256, 256]\n[machine]\nprofile = \"fuel_ip34\"\ncpu = \"r14000\"\n"; + let ok: MachineConfig = toml::from_str(&format!( + "{base}[fuel]\nl1_revision = \"1.50.2\"\nbrick_serial = \"EMUN03\"\n" + )).unwrap(); + ok.validate().unwrap(); + let saved = toml::to_string(&ok).unwrap(); + let back: MachineConfig = toml::from_str(&saved).unwrap(); + assert_eq!(back.fuel.l1_revision.as_deref(), Some("1.50.2")); + assert_eq!(back.fuel.brick_serial.as_deref(), Some("EMUN03")); + let plain: MachineConfig = toml::from_str(base).unwrap(); + plain.validate().unwrap(); + assert!(plain.fuel.l1_revision.is_none() && plain.fuel.brick_serial.is_none()); + let bad_rev: MachineConfig = toml::from_str(&format!("{base}[fuel]\nl1_revision = \"1.48\"\n")).unwrap(); + assert!(bad_rev.validate().unwrap_err().contains("fuel.l1_revision")); + let long = "x".repeat(38); + let bad_serial: MachineConfig = toml::from_str(&format!("{base}[fuel]\nbrick_serial = \"{long}\"\n")).unwrap(); + assert!(bad_serial.validate().unwrap_err().contains("fuel.brick_serial")); + } + + #[test] + fn fuel_requires_the_r14000_cpu_key() { + let mut cfg = MachineConfig::default(); + cfg.machine.profile = MachineProfile::FuelIp34; + cfg.banks.fill(0); + cfg.banks[0] = 256; + cfg.banks[1] = 256; + cfg.scsi.clear(); + assert!(cfg.validate().unwrap_err().contains("r14000")); + cfg.machine.cpu = CpuModel::R10000; + cfg.validate().unwrap(); + cfg.machine.cpu = CpuModel::R14000; + cfg.validate().unwrap(); + cfg.machine.cpu = CpuModel::R12000; + assert!(cfg.validate().is_err(), "no source has a Fuel with an R12000"); + } + + #[test] + fn fuel_requires_its_fixed_memory_population_and_no_graphics() { + let mut cfg = MachineConfig::default(); + cfg.machine.profile = MachineProfile::FuelIp34; + cfg.machine.cpu = CpuModel::R10000; + cfg.banks.fill(0); + cfg.banks[0] = 256; + cfg.banks[1] = 256; + cfg.scsi.clear(); + assert_eq!(cfg.graphics_board(), GraphicsBoard::None); + cfg.network.mode = NetMode::Off; + cfg.validate().unwrap(); + cfg.network.mode = NetMode::Pcap; + cfg.validate().unwrap(); + cfg.graphics.board = Some(GraphicsBoard::Newport); + assert!(cfg.validate().unwrap_err().contains("none")); + cfg.graphics.board = None; + cfg.banks[1] = 0; + assert!(cfg.validate().unwrap_err().contains("256, 256")); + } + + #[test] + fn fuel_accepts_wide_scsi_ids_on_both_channels_without_changing_narrow_profiles() { + let text = "banks = [256, 256]\n[machine]\nprofile = \"fuel_ip34\"\ncpu = \"r10000\"\n[scsi.15]\npath = \"disk.raw\"\n[scsi.31]\npath = \"disk1.raw\"\n"; + let mut cfg: MachineConfig = toml::from_str(text).unwrap(); + cfg.validate().unwrap(); + // The Octane's buses are wide as well; the Indy's WD33C93A is narrow. + cfg.machine.profile = MachineProfile::IndyIp24; + assert!(cfg.validate().unwrap_err().contains("out of range")); + } +} diff --git a/src/cpu/mips_cache_shadow.rs b/src/cpu/mips_cache_shadow.rs index 2832438c3..9de683f3d 100644 --- a/src/cpu/mips_cache_shadow.rs +++ b/src/cpu/mips_cache_shadow.rs @@ -807,6 +807,14 @@ pub const R14000_PRID: u32 = 0x0000_0f24; pub type R12000O2ShadowCache = ShadowCache<32768, 64, 32768, 32, 2097152, 64, true, 0x0000_0e35, 0x0000_0900, 64, { model::R12000 }>; +/// R14000, as the Fuel has it: the Octane's part at revision 2.3 (PRId +/// 0x0f23), the revision a public software inventory of a Fuel prints. The +/// raw CP0 word, the FIR and the 128-byte lines are taken from the Octane +/// part (`R12000ShadowCache` with `R14000_PRID`), not read from a Fuel. Like +/// the Octane's R14000 it reports "R12000" as `NAME`. +pub type R14000FuelShadowCache = + ShadowCache<32768, 64, 32768, 32, 2097152, 128, true, 0x0000_0f23, 0x0000_0900, 64, { model::R12000 }>; + /// R16000 2.2, the 800 MHz part, with a 4 MB secondary cache. Only /// `iris-bench` builds it for now. /// diff --git a/src/dev/l1.rs b/src/dev/l1.rs index c23b6c256..c09e9f6a6 100644 --- a/src/dev/l1.rs +++ b/src/dev/l1.rs @@ -192,6 +192,11 @@ impl L1 { self.state.lock().config.mac } + #[cfg(test)] + pub fn config(&self) -> L1Config { + self.state.lock().config.clone() + } + #[cfg(test)] pub fn display_line(&self) -> String { self.state.lock().display.clone() diff --git a/src/dev/mod.rs b/src/dev/mod.rs index 33c4182d3..c769b803e 100644 --- a/src/dev/mod.rs +++ b/src/dev/mod.rs @@ -74,5 +74,6 @@ pub mod ics1890; pub mod isp12160; pub mod l1; pub mod ohci; +pub mod prom_image; pub mod timekeeper; pub mod xbridge; diff --git a/src/dev/prom_image.rs b/src/dev/prom_image.rs new file mode 100644 index 000000000..1a50f89b9 --- /dev/null +++ b/src/dev/prom_image.rs @@ -0,0 +1,210 @@ +//! SGI PROM image loading: version-six single-segment containers and raw +//! payloads. A recognized but malformed container is rejected rather than +//! retried as a raw image. +//! +//! Ported from hw/mips/sgi/prom-image.c in jamesbraid/qemu-sgi-origami. + +use std::io::{self, Read}; + +fn invalid(message: &str) -> io::Error { + io::Error::new(io::ErrorKind::InvalidData, message) +} +fn word(data: &[u8], off: usize) -> u64 { + u64::from_be_bytes(data[off..off + 8].try_into().unwrap()) +} +fn byte_sum(data: &[u8]) -> u64 { + data.iter().map(|b| *b as u64).sum() +} + +/// Read a PROM file and decode its payload, retaining the path in errors. +pub fn load(path: &std::path::Path, payload_limit: usize) -> io::Result> { + let with_path = |error: io::Error| { + io::Error::new(error.kind(), format!("PROM image {}: {error}", path.display())) + }; + let file = std::fs::File::open(path).map_err(with_path)?; + // Bound the encoded file by zlib's compression bound, the container + // header, gzip framing and the accepted zero tail. + let size = payload_limit as u64; + let file_limit = size + .saturating_add(size >> 12) + .saturating_add(size >> 14) + .saturating_add(size >> 25) + .saturating_add(13 + 4096 + 12 + 7); + if file.metadata().map_err(with_path)?.len() > file_limit { + return Err(with_path(invalid( + "encoded PROM image exceeds flash capacity", + ))); + } + let mut bytes = Vec::new(); + file.take(file_limit.saturating_add(1)) + .read_to_end(&mut bytes) + .map_err(with_path)?; + if bytes.len() as u64 > file_limit { + return Err(with_path(invalid( + "encoded PROM image exceeds flash capacity", + ))); + } + decode(&bytes, payload_limit).map_err(with_path) +} + +/// Accept raw payload bytes or the supported stored/gzip container subset. +/// Gzip output is bounded before decoding, and exactly one member is required. +pub fn decode(file: &[u8], payload_limit: usize) -> io::Result> { + if file.len() < 0x48 || word(file, 0x40) != 0x4a464b535743534d { + if file.len() > payload_limit { + return Err(invalid("PROM image exceeds flash capacity")); + } + return Ok(file.to_vec()); + } + if file.len() < 4096 { + return Err(invalid("truncated SGI PROM container header")); + } + if word(file, 0x48) != 6 { + return Err(invalid("unsupported SGI PROM container version")); + } + if word(file, 0x50) != file.len() as u64 { + return Err(invalid("SGI PROM declared end does not match file size")); + } + if u32::from_be_bytes(file[0x7c..0x80].try_into().unwrap()) != 1 { + return Err(invalid("unsupported SGI PROM segment count")); + } + if word(file, 0x98) != 4096 { + return Err(invalid("unsupported SGI PROM payload offset")); + } + let flags = word(file, 0x90); + if flags != 0x10 && flags != 0x13 { + return Err(invalid("unsupported SGI PROM flags")); + } + let length = word(file, 0xb0); + if length == 0 || length > payload_limit as u64 { + return Err(invalid( + "SGI PROM decoded length is zero or exceeds capacity", + )); + } + let payload = if flags == 0x10 { + if word(file, 0xb8) != 0 || word(file, 0xc8) != 0 || word(file, 0xd0) != length { + return Err(invalid("unsupported SGI PROM uncompressed metadata")); + } + if length != (file.len() - 4096) as u64 { + return Err(invalid("invalid SGI PROM stored length")); + } + file[4096..].to_vec() + } else { + let stored = word(file, 0xb8); + if stored == 0 || stored > (file.len() - 4096) as u64 { + return Err(invalid("invalid SGI PROM stored length")); + } + let end = 4096 + stored as usize; + if file.len() - end > 7 || file[end..].iter().any(|b| *b != 0) { + return Err(invalid("unsupported SGI PROM trailing bytes")); + } + if byte_sum(&file[4096..end]) != word(file, 0xc8) { + return Err(invalid("SGI PROM stored checksum mismatch")); + } + let mut decoder = flate2::bufread::GzDecoder::new(&file[4096..end]); + let mut decoded = Vec::new(); + (&mut decoder).take(length + 1).read_to_end(&mut decoded)?; + if decoded.len() as u64 != length || !decoder.get_ref().is_empty() { + return Err(invalid("invalid SGI PROM gzip segment or decoded length")); + } + decoded + }; + if byte_sum(&payload) != word(file, 0xc0) { + return Err(invalid("SGI PROM decoded checksum mismatch")); + } + Ok(payload) +} + +#[cfg(test)] +mod tests { + // Cases follow tests/unit/test-sgi-prom-image.c in + // jamesbraid/qemu-sgi-origami: wrapped, gzip, raw, malformed and files. + use super::*; + use std::io::Write; + fn put(v: &mut [u8], off: usize, n: u64) { + v[off..off + 8].copy_from_slice(&n.to_be_bytes()); + } + fn container(payload: &[u8], gzip: bool) -> Vec { + let stored = if gzip { + let mut enc = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::default()); + enc.write_all(payload).unwrap(); + enc.finish().unwrap() + } else { + payload.to_vec() + }; + let mut v = vec![0; 4096]; + put(&mut v, 0x40, 0x4a464b535743534d); + put(&mut v, 0x48, 6); + put(&mut v, 0x50, (4096 + stored.len()) as u64); + v[0x7c..0x80].copy_from_slice(&1u32.to_be_bytes()); + put(&mut v, 0x90, if gzip { 0x13 } else { 0x10 }); + put(&mut v, 0x98, 4096); + put(&mut v, 0xb0, payload.len() as u64); + put(&mut v, 0xc0, byte_sum(payload)); + if gzip { + put(&mut v, 0xb8, stored.len() as u64); + put(&mut v, 0xc8, byte_sum(&stored)); + } else { + put(&mut v, 0xd0, payload.len() as u64); + } + v.extend(stored); + v + } + /// Equivalent PROM payloads must boot from each supported image format. + #[test] + fn raw_stored_and_gzip_images_load_with_the_same_capacity_limit() { + let path = std::env::temp_dir().join(format!("iris-prom-load-{}", std::process::id())); + for file in [vec![9, 8, 7, 6], container(&[9, 8, 7, 6], false), container(&[9, 8, 7, 6], true)] { + std::fs::write(&path, file).unwrap(); + assert_eq!(load(&path, 4).unwrap(), [9, 8, 7, 6]); + assert!(load(&path, 3).is_err(), "decoded payload exceeds capacity"); + } + std::fs::write(&path, []).unwrap(); + assert!(load(&path, 4).unwrap().is_empty()); + std::fs::remove_file(path).unwrap(); + } + + /// An invalid PROM must identify its input file instead of executing a + /// malformed container. + #[test] + fn invalid_or_unavailable_images_report_their_path() { + let path = std::env::temp_dir().join(format!("iris-prom-invalid-{}", std::process::id())); + let error = load(&path, 4).unwrap_err(); + assert_eq!(error.kind(), io::ErrorKind::NotFound); + assert!(error.to_string().contains(&path.display().to_string())); + let reject = |file: &[u8]| { + std::fs::write(&path, file).unwrap(); + let error = load(&path, 4).unwrap_err(); + assert_eq!(error.kind(), io::ErrorKind::InvalidData); + assert!(error.to_string().contains(&path.display().to_string())); + }; + reject(&vec![0; 5000]); // encoded file bound + let original = container(&[4, 3, 2, 1], false); + reject(&original[..100]); // recognized but truncated header + for (off, n) in [(0x48, 7), (0x50, 4096), (0x90, 0x14), (0x98, 4095), (0xb0, 0), (0xd0, 3)] { + let mut file = original.clone(); + put(&mut file, off, n); + reject(&file); + } + for gzip in [false, true] { + let mut file = container(&[4, 3, 2, 1], gzip); + file[0xc7] ^= 1; + reject(&file); + } + let gzip = container(&[4, 3, 2, 1], true); + for tail in [7, 8] { + let mut file = gzip.clone(); + file.extend(vec![0; tail]); + let size = file.len(); + put(&mut file, 0x50, size as u64); + if tail == 7 { + std::fs::write(&path, &file).unwrap(); + assert_eq!(load(&path, 4).unwrap(), [4, 3, 2, 1]); + } else { + reject(&file); + } + } + std::fs::remove_file(path).unwrap(); + } + +} diff --git a/src/ip35.rs b/src/ip35.rs new file mode 100644 index 000000000..ff2af349f --- /dev/null +++ b/src/ip35.rs @@ -0,0 +1,368 @@ +//! The IP35 processor bus: Bedrock's four XKPHYS access-space views. +//! +//! Raw address bits 58:57 select Hub-special, I/O, fetch operations or uncached +//! RAM. Each view has a small fixed set of MMIO apertures, decoded by sorted +//! windows rather than a full physical-address table. Maps replace whole 4 KiB +//! pages as Physical does, while each endpoint still rejects padding and +//! enforces its cell permission. +//! +//! Cached RAM alone uses Physical, including its direct bank pointers, ppmem +//! and generation mappings. The MMIO views and register sub-bus need none of +//! those RAM facilities. Giving each its own full table would consume several +//! entries in the process-wide fault-handler registry per machine. +//! +//! Ported from hw/mips/sgi/sn1-node.c in jamesbraid/qemu-sgi-origami. + +use crate::dev::bedrock_mem::{BANK_SPAN, FITTED_BANK}; +use crate::physical::Physical; +use crate::traits::*; +#[cfg(feature = "jitv2")] +use std::sync::{atomic::AtomicU64, OnceLock}; +use std::sync::Arc; + +pub(crate) const CPU_PHYS_MASK: u64 = ((1u64 << 40) - 1) | (3u64 << 57); +const PA_MASK: u64 = (1u64 << 40) - 1; + +/// Where the Fuel has memory: four 1 GiB slots Bedrock's memory controller +/// can place its two 256 MiB banks in, depending on how the PROM programs +/// it, and the reset PROM between the first two. Uncached fetches of the +/// PROM's own code land in its range. +pub fn memory_map() -> crate::memory_map::MemoryMap { + use crate::memory_map::{MemKind, MemRange, MemoryMap}; + const NAMES: [&str; 4] = ["ram slot 0", "ram slot 1", "ram slot 2", "ram slot 3"]; + let mut ranges: Vec = NAMES.iter().enumerate().map(|(slot, &name)| { + let base = slot as u64 * BANK_SPAN; + MemRange::new(name, MemKind::Ram, base, base + FITTED_BANK) + }).collect(); + ranges.push(MemRange::new("prom", MemKind::Rom, 0x1fc0_0000, 0x2000_0000)); + MemoryMap::new(ranges) +} + +struct Unclaimed; +impl BusDevice for Unclaimed {} + +/// An exact-bounds chip-relative window, optionally restricted by a cell permission. +/// Physical rounds maps to pages, so the adapter also refuses their padding. +pub(crate) struct Window { + base: u64, + size: u64, + device: Arc, + permission: Option bool + Send + Sync>>, +} +impl Window { + pub(crate) fn new(base: u64, size: u64, device: Arc) -> Self { + Self { base, size, device, permission: None } + } + pub(crate) fn permitted(mut self, permission: Arc bool + Send + Sync>) -> Self { + self.permission = Some(permission); + self + } + pub(crate) fn contains(&self, a: u64, n: u64) -> bool { self.local(a, n).is_some() } + fn local(&self, a: u64, n: u64) -> Option { + let local = a.checked_sub(self.base)?; + (local.checked_add(n)? <= self.size + && self.permission.as_ref().is_none_or(|allowed| allowed())) + .then_some(local) + } + fn write_masked_bytes(&self, a: u64, val: u64, mask: u64) -> u32 { + for i in 0..8usize { + let shift = (7 - i) * 8; + if (mask >> shift) & 0xff == 0 { + continue; + } + let Some(local) = self.local(a + i as u64, 1) else { + return BUS_ERR; + }; + let status = self.device.write8(local, (val >> shift) as u8); + if status != BUS_OK { + return status; + } + } + BUS_OK + } +} +macro_rules! window_access { + ($r:ident,$w:ident,$t:ty,$out:ident,$n:expr) => { + fn $r(&self, a: u64) -> $out { + self.local(a, $n).map(|at| self.device.$r(at)).unwrap_or_else($out::err) + } + fn $w(&self, a: u64, v: $t) -> u32 { + self.local(a, $n).map(|at| self.device.$w(at, v)).unwrap_or(BUS_ERR) + } + }; +} +impl BusDevice for Window { + #[cfg(feature = "jitv2")] + fn gen_ptr(&self, address: u64) -> *const AtomicU64 { + self.local(address, 1).map(|local| self.device.gen_ptr(local)).unwrap_or(std::ptr::null()) + } + fn write64_masked(&self, a: u64, val: u64, mask: u64) -> u32 { + if a & 7 != 0 { + return BUS_ERR; + } + if mask == 0 { + return BUS_OK; + } + if let Some(local) = self.local(a, 8) { + self.device.write64_masked(local, val, mask) + } else { + self.write_masked_bytes(a, val, mask) + } + } + fn write_narrow_from(&self, a: u64, size: usize, gpr: u64) -> u32 { + self.local(a, size as u64) + .map(|local| self.device.write_narrow_from(local, size, gpr)) + .unwrap_or(BUS_ERR) + } + window_access!(read8, write8, u8, BusRead8, 1); + window_access!(read16, write16, u16, BusRead16, 2); + window_access!(read32, write32, u32, BusRead32, 4); + window_access!(read64, write64, u64, BusRead64, 8); +} +pub(crate) fn map_device( + view: &Physical, + base: u64, + size: u64, + device: Arc, + name: &str, +) { + assert!(size > 0 && base.checked_add(size).is_some_and(|end| end <= PA_MASK + 1)); + view.map(base, base + size, Arc::new(Window::new(base, size, device)), name); +} + +struct DecodeRange { + start: u64, + end: u64, + window: Arc, +} + +/// Small MMIO maps have no RAM, direct pointers or host fault-handler entries. +/// Last map wins over whole pages, including padding rejected by its Window. +#[derive(Default)] +pub(crate) struct SparseWindowMap { + ranges: Vec, +} +impl SparseWindowMap { + pub(crate) fn map(&mut self, base: u64, size: u64, device: Arc) { + const PAGE: u64 = 1 << crate::ppmem::devmap::SLOT_SHIFT; + let end = base.checked_add(size).filter(|&end| size != 0 && end <= PA_MASK + 1) + .expect("IP35 window exceeds physical address space"); + let start = base & !(PAGE - 1); + let end = end.next_multiple_of(PAGE); + let window = Arc::new(Window::new(base, size, device)); + let mut ranges = Vec::with_capacity(self.ranges.len() + 2); + for old in self.ranges.drain(..) { + if old.end <= start || old.start >= end { + ranges.push(old); + } else { + if old.start < start { + ranges.push(DecodeRange { start: old.start, end: start, window: old.window.clone() }); + } + if old.end > end { + ranges.push(DecodeRange { start: end, end: old.end, window: old.window }); + } + } + } + ranges.push(DecodeRange { start, end, window }); + ranges.sort_unstable_by_key(|range| range.start); + self.ranges = ranges; + } + fn target(&self, address: u64) -> Option<&Window> { + let i = self.ranges.partition_point(|range| range.start <= address).checked_sub(1)?; + let range = &self.ranges[i]; + (address < range.end).then_some(range.window.as_ref()) + } +} +macro_rules! sparse_access { + ($read:ident,$write:ident,$ty:ty,$result:ident) => { + fn $read(&self, address: u64) -> $result { + self.target(address).map(|window| window.$read(address)).unwrap_or_else($result::err) + } + fn $write(&self, address: u64, value: $ty) -> u32 { + self.target(address).map(|window| window.$write(address, value)).unwrap_or(BUS_ERR) + } + }; +} +impl BusDevice for SparseWindowMap { + #[cfg(feature = "jitv2")] + fn gen_ptr(&self, address: u64) -> *const AtomicU64 { + self.target(address).map(|window| window.gen_ptr(address)).unwrap_or(std::ptr::null()) + } + fn write64_masked(&self, address: u64, value: u64, mask: u64) -> u32 { + self.target(address).map(|window| window.write64_masked(address, value, mask)) + .unwrap_or_else(|| Unclaimed.write64_masked(address, value, mask)) + } + fn write_narrow_from(&self, address: u64, size: usize, gpr: u64) -> u32 { + self.target(address).map(|window| window.write_narrow_from(address, size, gpr)) + .unwrap_or(BUS_ERR) + } + sparse_access!(read8, write8, u8, BusRead8); + sparse_access!(read16, write16, u16, BusRead16); + sparse_access!(read32, write32, u32, BusRead32); + sparse_access!(read64, write64, u64, BusRead64); +} + +/// The four uncached processor views decode MMIO separately from cached RAM. +pub struct Ip35Bus { + pub phys: Arc, + views: [SparseWindowMap; 4], + #[cfg(feature = "jitv2")] + ram_generations: OnceLock>, +} +impl Ip35Bus { + pub fn new(phys: Arc) -> Self { + phys.set_default(Arc::new(Unclaimed), "IP35 unclaimed cached address"); + Self { + phys, + views: std::array::from_fn(|_| SparseWindowMap::default()), + #[cfg(feature = "jitv2")] + ram_generations: OnceLock::new(), + } + } + /// Code pages use stable logical RAM generations across bank placement and + /// diagnostic remaps; non-RAM pages keep their own device generations. + #[cfg(feature = "jitv2")] + pub(crate) fn set_ram_generations(&self, memory: Arc) { + assert!(self.ram_generations.set(memory).is_ok(), "IP35 RAM generations already wired"); + } + /// Raw XKPHYS bits 58:57 select Hub-special, I/O, fetchop, or uncached RAM. + pub fn map(&mut self, raw_uc: u8, base: u64, size: u64, device: Arc) { + assert!(raw_uc < 4); + self.views[raw_uc as usize].map(base, size, device); + } + pub fn set_fallback(&mut self, raw_uc: u8, end: u64, device: Arc) { + self.map(raw_uc, 0, end, device); + } + fn target(&self, a: u64, n: u64) -> Option<(&SparseWindowMap, u64)> { + let pa = plain_address(a, n)?; + Some((&self.views[(a >> 57) as usize], pa)) + } + pub fn cached_view(&self, plain_memory: Arc) -> Arc { + Arc::new(CachedPhysical { + phys: self.phys.clone(), + plain_memory, + }) + } +} +fn plain_address(a: u64, n: u64) -> Option { + if a & !CPU_PHYS_MASK != 0 { + return None; + } + let pa = a & PA_MASK; + (pa.checked_add(n)? <= PA_MASK + 1).then_some(pa) +} +struct CachedPhysical { + phys: Arc, + plain_memory: Arc, +} +impl CachedPhysical { + fn memory_target(&self, a: u64, n: u64) -> Option<(&dyn BusDevice, u64)> { + self.target(a, n).map(|(physical, pa)| { + // MD owns every guest write to a possible fitted-RAM prefix, so + // all paths update its stable logical generation counters. PROM + // and other devices retain the cached Physical connection. + let memory: &dyn BusDevice = if pa / BANK_SPAN < 4 && pa % BANK_SPAN < FITTED_BANK { + self.plain_memory.as_ref() + } else { physical }; + (memory, pa) + }) + } + fn target(&self, a: u64, n: u64) -> Option<(&Physical, u64)> { + // UC selectors belong to the uncached SysAd connection. Reject them + // here rather than turning a malformed cached request into an alias. + if a & !PA_MASK != 0 || a & (n - 1) != 0 { + return None; + } + Some((self.phys.as_ref(), plain_address(a, n)?)) + } +} +macro_rules! processor_access { + ($r:ident,$w:ident,$t:ty,$out:ident,$n:expr,$writes:ident) => { + fn $r(&self, a: u64) -> $out { + self + .target(a, $n) + .map(|(v, pa)| v.$r(pa)) + .unwrap_or_else($out::err) + } + fn $w(&self, a: u64, v: $t) -> u32 { + self + .$writes(a, $n) + .map(|(view, pa)| view.$w(pa, v)) + .unwrap_or(BUS_ERR) + } + }; +} +impl BusDevice for Ip35Bus { + #[cfg(feature = "jitv2")] + fn gen_ptr(&self, address: u64) -> *const AtomicU64 { + if address & !PA_MASK == 0 { + // JIT page lookup passes plain physical code addresses, without + // a cache attribute. Raw UC0 therefore cannot select the MMIO map + // here: high cached RAM and the reset PROM use the cached plane. + if let Some(memory) = self.ram_generations.get() { + let generation = memory.gen_ptr(address); + if !generation.is_null() { return generation; } + } + return self.phys.gen_ptr(address); + } + self.target(address, 1).map(|(view, pa)| view.gen_ptr(pa)).unwrap_or(std::ptr::null()) + } + fn write64_masked(&self, a: u64, val: u64, mask: u64) -> u32 { + if a & 7 != 0 { + return BUS_ERR; + } + self.target(a, 8) + .map(|(view, pa)| view.write64_masked(pa, val, mask)) + .unwrap_or(BUS_ERR) + } + fn write_narrow_from(&self, a: u64, size: usize, gpr: u64) -> u32 { + self.target(a, size as u64) + .map(|(view, pa)| view.write_narrow_from(pa, size, gpr)) + .unwrap_or(BUS_ERR) + } + processor_access!(read8, write8, u8, BusRead8, 1, target); + processor_access!(read16, write16, u16, BusRead16, 2, target); + processor_access!(read32, write32, u32, BusRead32, 4, target); + processor_access!(read64, write64, u64, BusRead64, 8, target); +} +impl BusDevice for CachedPhysical { + fn write64_masked(&self, address: u64, value: u64, mask: u64) -> u32 { + self.memory_target(address, 8).map(|(view, pa)| view.write64_masked(pa, value, mask)) + .unwrap_or(BUS_ERR) + } + fn mem_ptr(&self, a: u64) -> Option<*const u64> { + self.target(a, 8).and_then(|(view, pa)| view.mem_ptr(pa)) + } + #[cfg(feature = "jitv2")] + fn gen_ptr(&self, a: u64) -> *const AtomicU64 { + self.target(a, 1).map(|(view, pa)| { + let generation = self.plain_memory.gen_ptr(pa); + if generation.is_null() { view.gen_ptr(pa) } else { generation } + }).unwrap_or(std::ptr::null()) + } + fn read_block(&self, a: u64, out: &mut [u64]) -> u32 { + self.target(a, 8) + .map(|(view, pa)| view.read_block(pa, out)).unwrap_or(BUS_ERR) + } + fn write_block(&self, a: u64, data: &[u64]) -> u32 { + self.memory_target(a, 8) + .map(|(view, pa)| view.write_block(pa, data)).unwrap_or(BUS_ERR) + } + processor_access!(read8, write8, u8, BusRead8, 1, memory_target); + processor_access!(read16, write16, u16, BusRead16, 2, memory_target); + processor_access!(read32, write32, u32, BusRead32, 4, memory_target); + processor_access!(read64, write64, u64, BusRead64, 8, memory_target); + fn dma_read64(&self, a: u64) -> BusRead64 { + plain_address(a, 8) + .map(|pa| self.plain_memory.dma_read64(pa)) + .unwrap_or_else(BusRead64::err) + } + fn dma_write64(&self, a: u64, v: u64) -> u32 { + plain_address(a, 8) + .map(|pa| self.plain_memory.dma_write64(pa, v)) + .unwrap_or(BUS_ERR) + } +} + +#[cfg(test)] +mod tests; diff --git a/src/ip35/tests.rs b/src/ip35/tests.rs new file mode 100644 index 000000000..b7cd7bb51 --- /dev/null +++ b/src/ip35/tests.rs @@ -0,0 +1,623 @@ +use super::*; +use crate::traits::{BusDevice, BUS_OK}; + +struct Word(u64); +impl BusDevice for Word { + fn read64(&self, _: u64) -> BusRead64 { + BusRead64::ok(self.0) + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +enum NarrowWriteCall { + Byte(u64, u8), + Half(u64, u16), + From(usize, u64, u64), +} +/// Records every narrow write it sees. With `sees_register` it overrides +/// `write_narrow_from` and keeps the whole source register, as Bedrock's ECC +/// view does; otherwise it takes the trait's default. +#[derive(Default)] +struct NarrowWriteSpy { + sees_register: bool, + calls: std::sync::Mutex>, +} +impl NarrowWriteSpy { + fn new(sees_register: bool) -> Arc { + Arc::new(Self { sees_register, ..Default::default() }) + } + fn calls(&self) -> Vec { + self.calls.lock().unwrap().clone() + } +} +impl BusDevice for NarrowWriteSpy { + fn write8(&self, address: u64, value: u8) -> u32 { + self.calls.lock().unwrap().push(NarrowWriteCall::Byte(address, value)); + BUS_OK + } + fn write16(&self, address: u64, value: u16) -> u32 { + self.calls.lock().unwrap().push(NarrowWriteCall::Half(address, value)); + BUS_OK + } + fn write_narrow_from(&self, address: u64, size: usize, gpr: u64) -> u32 { + if !self.sees_register { + return if size == 1 { self.write8(address, gpr as u8) } else { self.write16(address, gpr as u16) }; + } + self.calls.lock().unwrap().push(NarrowWriteCall::From(size, address, gpr)); + BUS_OK + } +} + +/// An uncached SB or SH reaches a device with the whole source register, at the +/// device-local address; only a device that overrides the call sees more than +/// the low bits, and a window outside every map is a bus error. +/// The Fuel's processor is the R14000 at revision 2.3, with the Octane +/// R14000's caches; the Octane's own part is revision 2.4. +#[test] +fn the_fuel_runs_the_r14000_at_revision_2_3() { + use crate::cpu::mips_cache_shadow::{R14000FuelShadowCache, R14000_PRID}; + use crate::cpu::mips_cache_v2::{CpuModel, MipsCache}; + assert_eq!(::PRID, 0x0f23); + assert_eq!(R14000_PRID, 0x0f24); + assert_eq!(::L2_SIZE, 2 << 20); + // Typed R12000, it takes the R10000 family's behaviour: reserved + // secondary-tag bits read as zero, and a refused uncached store posts. + assert!(::R10K_FAMILY); + assert_ne!(::SECONDARY_TAG_MASK, u64::MAX); + assert!(::POSTS_UNCACHED_STORES); + assert!(::PARTIAL_LWR_KEEPS_UPPER); +} + +#[test] +fn narrow_stores_carry_the_register_to_devices_that_ask_for_it() { + let mut bus = Ip35Bus::new(Physical::new(Vec::new())); + let plain = NarrowWriteSpy::new(false); + let ecc = NarrowWriteSpy::new(true); + bus.map(3, 0x1000, 0x1000, plain.clone()); + bus.map(3, 0x2000, 0x1000, ecc.clone()); + + let source = 0x1122_3344_5566_77a5u64; + let uc = 3u64 << 57; + assert_eq!(bus.write_narrow_from(uc | 0x1003, 1, source), BUS_OK); + assert_eq!(bus.write_narrow_from(uc | 0x1004, 2, source), BUS_OK); + assert_eq!( + plain.calls(), + vec![NarrowWriteCall::Byte(3, source as u8), NarrowWriteCall::Half(4, source as u16)], + "an ordinary window gets the architectural low bits", + ); + + assert_eq!(bus.write_narrow_from(uc | 0x2004, 2, source), BUS_OK); + assert_eq!(ecc.calls(), vec![NarrowWriteCall::From(2, 4, source)]); + + assert_eq!(bus.write_narrow_from(uc | 0x8000, 2, source), BUS_ERR, "no window there"); +} + +/// Equal physical offsets in different XKPHYS access spaces reach different devices. +#[test] +fn uc_spaces_keep_equal_offsets_distinct() { + let mut bus = Ip35Bus::new(Physical::new(Vec::new())); + for space in 0..4 { + bus.map( + space, + 0x0100_0000, + 0x1000, + Arc::new(Word(0x100 + space as u64)), + ); + } + for space in 0..4 { + let value = bus.read64(((space as u64) << 57) | 0x0100_0000); + assert_eq!(value.status, BUS_OK); + assert_eq!(value.data, 0x100 + space as u64); + } +} + +/// The processor must retain both XKPHYS access-space bits through translation. +#[test] +fn xkphys_uc_bits_survive_the_board_cpu_mask() { + // MipsExecutor and the TLB's inline arrays exceed the test worker's stack. + std::thread::Builder::new() + .stack_size(4 * 1024 * 1024) + .spawn(|| { + let bus = Arc::new(Ip35Bus::new(Physical::new(Vec::new()))); + let mut cpu = crate::ip35_machine::build_cpu(bus); + cpu.core.write_cp0( + 12, + (cpu.core.cp0_status | crate::cpu::mips_core::STATUS_KX) as u64, + ); + for raw_uc in 0..4u64 { + let va = 0x9000_0000_0100_0020 | (raw_uc << 57); + for _ in 0..2 { + let translated = cpu.debug_translate(va); + assert_eq!(translated.status, crate::cpu::mips_exec::TR_UNCACHED); + assert_eq!(translated.phys, (raw_uc << 57) | 0x0100_0020); + } + } + }) + .expect("spawn") + .join() + .expect("thread panicked"); +} + +/// RAM placed above the first GiB must remain shared by cached, uncached and DMA paths. +#[test] +fn cached_and_uncached_memory_keep_high_addresses() { + let board = crate::ip35_machine::FuelBoard::new(&[], Arc::new(|| 0)).unwrap(); + assert_eq!(board.md.md_bus().write64(8, 0x33), BUS_OK); + let at = (1 << 30) + 0x1000; + assert_eq!(board.md.memory_view().write64(at, 0x12345678), BUS_OK); + assert_eq!(board.cached_bus().read64(at).data, 0x12345678); + assert_eq!(board.bus.read64((3 << 57) | at).data, 0x12345678); + assert_eq!(board.bus.read64(at).status, BUS_ERR); + assert_eq!(board.md.dma_view().read64(at).data, 0x12345678); +} + +/// Memory diagnostics must notify both processor interfaces and refuse bad uncached data. +#[test] +fn memory_errors_reach_both_interfaces_and_uncached_reads_fail() { + let board = crate::ip35_machine::FuelBoard::new(&[], Arc::new(|| 0)).unwrap(); + let correctable = 1 << 61; + board.pi[0].write64(0xb0, correctable); + board.md.md_bus().write64(8, 0x33); + for directory in [false, true] { + if directory { + board.md.md_bus().write64(0x40000, 2); + board.md.directory_bus().write64(0, 1 << 16); + board.md.md_bus().write64(0x40000, 0); + } else { + board.md.ecc_bus().write8(7, 1); + } + assert_eq!(board.bus.read64(0).status, BUS_OK); + for pi in &board.pi { + assert_eq!(pi.read64(0xa0).data & correctable, correctable); + } + assert_eq!(board.irq.bits() & (1 << 11), 1 << 11); + for pi in &board.pi { + pi.write64(0x90, 64 + 61); + assert_eq!(pi.read64(0xa0).data & correctable, 0); + } + assert_eq!(board.irq.bits() & (1 << 11), 0); + } + + let board = crate::ip35_machine::FuelBoard::new(&[], Arc::new(|| 0)).unwrap(); + board.md.md_bus().write64(8, 0x33); + board.md.ecc_bus().write8(7, 3); + for selector in [0, 3] { + assert_eq!(board.bus.read64(selector << 57).status, BUS_ERR); + } + assert_eq!(board.cached_bus().read64(0).status, BUS_OK); + assert_eq!(board.md.dma_view().read64(0).status, BUS_OK); +} + +/// Firmware resets the PCI link before reinitializing the attached storage controller. +#[test] +fn fuel_link_reset_resets_the_attached_scsi_engine() { + use crate::dev::isp1040::{mbox, reg, CTRL_INT_ENABLE, CTRL_RISC_INT_ENABLE, HOST_CMD_SHIFT}; + use crate::pci::PciBusDevice; + let board = crate::ip35_machine::FuelBoard::new(&[], Arc::new(|| 0)).unwrap(); + let isp = &board.scsi; + isp.config().write(1, 6); + isp.config().write(5, 0x02000000); + isp.bar_write(1, reg::BUS_CTRL, u32::from(CTRL_INT_ENABLE | CTRL_RISC_INT_ENABLE), 2); + isp.bar_write(1, reg::MAILBOX_BASE, u32::from(mbox::NOP), 2); + isp.bar_write(1, reg::HOST_CMD, 0x5 << HOST_CMD_SHIFT, 2); + assert_eq!(isp.bar_read(1, reg::MAILBOX_BASE, 2), u32::from(mbox::COMPLETE)); + assert_ne!(isp.bar_read(1, reg::BUS_CTRL, 2), 0); + assert_eq!(board.xbridge.widget(0).unwrap().write32(0x2f4, 1), BUS_OK); + assert_eq!(isp.config().command(), 0); + assert_eq!(isp.config().bar_base(1), 0); + assert_eq!(isp.bar_read(1, reg::BUS_CTRL, 2), 0); + assert_eq!(isp.bar_read(1, reg::MAILBOX_BASE, 2), 0); + assert_eq!(isp.bar_read(1, reg::MAILBOX_BASE + 2, 2), 0x4953); +} + +/// The PROM reads the station address from the L1 and IRIX from the +/// Ethernet; both must report the configured one. +#[test] +fn the_configured_station_address_reaches_the_l1_and_the_ethernet() { + let mac = [0x08, 0x00, 0x69, 0x21, 0x43, 0x65]; + let board = crate::ip35_machine::FuelBoard::with_config(crate::ip35_machine::FuelBoardConfig { + payload: &[], clock: Arc::new(|| 0), flash_path: None, + station_mac: mac, + timekeeper: Arc::new(crate::dev::timekeeper::Timekeeper::fuel().unwrap()), + ps2_input: true, usb_input: true, + l1: Default::default(), + }).unwrap(); + assert_eq!(board.l1.station_mac(), mac); + assert_eq!(board.eth.mac(), mac); +} + +/// The L1's reported revision and brick serial keep their defaults unless +/// the board is built with replacements. +#[test] +fn the_l1_identity_overrides_reach_the_controller() { + use crate::ip35_machine::{FuelBoard, FuelBoardConfig, L1Identity}; + let build = |l1| FuelBoard::with_config(FuelBoardConfig { + payload: &[], clock: Arc::new(|| 0), flash_path: None, + station_mac: [8, 0, 0x69, 0x12, 0x34, 0x56], + timekeeper: Arc::new(crate::dev::timekeeper::Timekeeper::fuel().unwrap()), + ps2_input: true, usb_input: false, l1, + }).unwrap().l1.config(); + let stock = build(L1Identity::default()); + assert_eq!(stock.revision, Some([1, 48, 1])); + assert_eq!(stock.brick_serial.as_deref(), Some("EMUN00")); + let set = build(L1Identity { revision: Some([2, 3, 4]), brick_serial: Some("EMUN09".into()) }); + assert_eq!(set.revision, Some([2, 3, 4])); + assert_eq!(set.brick_serial.as_deref(), Some("EMUN09")); +} + +/// The Fuel's cache reports primary data tag stores to the memory controller: +/// the PROM's installs over its window start the private page, and its sweep +/// ends it. Other caches' tag stores and non-installs do nothing. +#[test] +fn the_prom_tag_stores_reach_the_memory_controller_through_the_cpu_cache() { + // An executor is large; a test thread's stack is not. + std::thread::Builder::new().stack_size(64 << 20).spawn(prom_tag_stores).unwrap().join().unwrap(); +} + +fn prom_tag_stores() { + use crate::cpu::mips_cache_v2::MipsCache; + use crate::cpu::mips_isa::{CACH_PD, CACH_PI, C_IST}; + let board = crate::ip35_machine::FuelBoard::new(&[], Arc::new(|| 0)).unwrap(); + let cpu = board.build_cpu(); + let window = |at: u64| board.md.physical().read64(at).status == BUS_OK; + assert!(!window(0x10_0000)); + cpu.cache.cache_op(C_IST | CACH_PD, 0xa800_0000_0010_0000, 0xffff_ffcf); + cpu.cache.cache_op(C_IST | CACH_PI, 0xa800_0000_0010_0000, 0x100c0); + assert!(!window(0x10_0000), "a pattern and an instruction cache tag start nothing"); + cpu.cache.cache_op(C_IST | CACH_PD, 0xa800_0000_0010_0000, 0x100c0); + assert!(window(0x10_0000)); + cpu.cache.cache_op(C_IST | CACH_PD, 0xa800_0000_0010_0000, 0); + assert!(!window(0x10_0000)); +} + +fn fuel_dma_board() -> crate::ip35_machine::FuelBoard { + let board = crate::ip35_machine::FuelBoard::new(&[], Arc::new(|| 0)).unwrap(); + assert_eq!(board.md.md_bus().write64(8, 0x33), BUS_OK); + board.xbridge.head(15).write32(0x24, 15); + board.xbridge.set_wide_slot_mask(15, 0); + board +} + +fn fuel_dma_address(address: u64) -> u64 { + (10 << 60) | (1 << 55) | address +} + +/// Storage DMA buffers can begin unaligned and cross a physical page boundary. +#[test] +fn dma_preserves_byte_order_and_crosses_ragged_pages() { + use crate::pci::PciDmaStatus; + let board = fuel_dma_board(); + let dma = board.xbridge.dma(15, 0); + let memory = board.md.memory_view(); + for (address, length) in [(0x1030, 8), (0x1103, 27), (0x1ffb, 19)] { + let data: Vec = (0..length).map(|i| 0x31 + i as u8).collect(); + assert_eq!(dma.dma_write(fuel_dma_address(address), &data), PciDmaStatus::Ok); + for (i, &byte) in data.iter().enumerate() { + assert_eq!(memory.read8(address + i as u64).data, byte); + } + let mut back = vec![0xff; length]; + assert_eq!(dma.dma_read(fuel_dma_address(address), &mut back), PciDmaStatus::Ok); + assert_eq!(back, data); + } + // Without byte swapping enabled, the bridge reverses each 32-bit word. + assert_eq!(dma.dma_write((10 << 60) | 0x3000, &[0, 1, 2, 3, 4, 5, 6, 7]), PciDmaStatus::Ok); + assert_eq!(memory.read64(0x3000).data, 0x0302010007060504); +} + +/// A failed DMA read must clear only refused chunks and preserve completed data. +#[test] +fn dma_refusals_clear_failed_reads_without_losing_completed_chunks() { + use crate::pci::PciDmaStatus; + let board = fuel_dma_board(); + let dma = board.xbridge.dma(15, 0); + let memory = board.md.memory_view(); + // Bank zero has 256 MiB fitted. The second qword is outside its backing. + let last = 0x0fff_fff8; + assert_eq!(memory.write64(last, 0x0102030405060708), BUS_OK); + let mut bytes = [0xff; 16]; + assert_eq!(dma.dma_read(fuel_dma_address(last), &mut bytes), PciDmaStatus::MasterAbort); + assert_eq!(bytes, [1, 2, 3, 4, 5, 6, 7, 8, 0, 0, 0, 0, 0, 0, 0, 0]); + assert_eq!(dma.dma_write(fuel_dma_address(last), &[0x42; 16]), PciDmaStatus::MasterAbort); + assert_eq!(memory.read64(last).data, 0x4242424242424242); + board.io.disable_device(15, 0); + bytes.fill(0xff); + assert_eq!(dma.dma_read(fuel_dma_address(last), &mut bytes), PciDmaStatus::MasterAbort); + assert_eq!(bytes, [0; 16]); + assert_eq!(dma.dma_write(fuel_dma_address(last), &[0x99; 8]), PciDmaStatus::MasterAbort); + assert_eq!(memory.read64(last).data, 0x4242424242424242); +} + +/// A wide DMA leg must report an uncorrectable word once without publishing bad bytes. +#[test] +fn fuel_dma_checks_an_aligned_uncorrectable_word_once() { + use crate::dev::bedrock_md::MemoryError; + use crate::pci::PciDmaStatus; + use std::sync::atomic::{AtomicU64, Ordering}; + let board = fuel_dma_board(); + assert_eq!(board.md.memory_view().write64(0, 0), BUS_OK); + assert_eq!(board.md.ecc_bus().write8(7, 3), BUS_OK); + let count = Arc::new(AtomicU64::new(0)); + let seen = count.clone(); + let md = Arc::downgrade(&board.md); + board.md.on_error(Arc::new(move |kind| { + assert_eq!(kind, MemoryError::Uncorrectable); + assert_eq!(md.upgrade().unwrap().current_cell(), u64::MAX); + seen.fetch_add(1, Ordering::Relaxed); + })); + // The former callback read the same persistent UCE once for each byte. + for address in 0..8 { + assert_eq!(board.io.inbound_read(15, Some(0), address, 1).status, BUS_OK); + } + assert_eq!(count.swap(0, Ordering::Relaxed), 8); + assert_ne!(board.md.md_bus().read64(0x60018).data & (1 << 61), 0); + let mut bytes = [0xff; 8]; + assert_eq!(board.xbridge.dma(15, 0).dma_read(fuel_dma_address(0), &mut bytes), PciDmaStatus::Ok); + assert_eq!(bytes, [0; 8]); + assert_eq!(count.load(Ordering::Relaxed), 1); + let record = board.md.md_bus().read64(0x60010).data; + assert_ne!(record & (1 << 60), 0); + assert_eq!(record & (1 << 61), 0); + assert_eq!(board.md.uncached_view().read64(0).status, BUS_ERR); +} + +/// Inbound Hub writes must preserve access width and the requesting device identity. +#[test] +fn dma_hub_access_preserves_width_and_source_permissions() { + use crate::pci::PciDmaStatus; + let board = fuel_dma_board(); + let dma = board.xbridge.dma(15, 0); + let hub = (1 << 47) | 0x1800000; + for (interface, offset) in [(0, 0), (1, 0x200000)] { + let address = fuel_dma_address(hub + offset + 0x90); + assert_eq!(dma.dma_write(address, &0x10cu64.to_be_bytes()), PciDmaStatus::Ok); + assert_ne!(board.pi[interface].read64(0x98).data & (1 << 12), 0); + for width in [1, 2, 4] { + assert_eq!(dma.dma_write(address, &[0; 8][..width]), PciDmaStatus::MasterAbort); + assert_ne!(board.pi[interface].read64(0x98).data & (1 << 12), 0); + } + assert_eq!(dma.dma_write(address, &12u64.to_be_bytes()), PciDmaStatus::Ok); + assert_eq!(board.pi[interface].read64(0x98).data & (1 << 12), 0); + } + let address = fuel_dma_address(hub + 0x90); + // I/O modify must retain PI region protection and requesting-slot identity. + assert_eq!(board.pi[0].write64(0x10, 0), BUS_OK); + assert_eq!(dma.dma_write(address, &0x10cu64.to_be_bytes()), PciDmaStatus::MasterAbort); + assert_eq!(board.pi[0].read64(0x98).data & (1 << 12), 0); + assert_eq!(board.pi[0].write64(0x10, 1), BUS_OK); + board.io.disable_device(15, 0); + assert_eq!(dma.dma_write(address, &0x10cu64.to_be_bytes()), PciDmaStatus::MasterAbort); + assert_eq!(board.pi[0].read64(0x98).data & (1 << 12), 0); + assert_eq!(board.xbridge.dma(15, 1).dma_write(address, &0x10cu64.to_be_bytes()), PciDmaStatus::Ok); + assert_ne!(board.pi[0].read64(0x98).data & (1 << 12), 0); + let mut bytes = [0xff; 8]; + assert_eq!(board.xbridge.dma(15, 1).dma_read( + fuel_dma_address(hub + 0x40), &mut bytes, + ), PciDmaStatus::MasterAbort); + assert_eq!(bytes, [0; 8]); + assert_eq!(board.bus.read64((1 << 57) | 0x1800040).status, BUS_OK); +} + +/// Rounded MMIO pages must not expose padding or bypass cell permissions. +#[test] +fn exact_windows_reject_padding_and_revoked_permissions() { + use std::sync::atomic::{AtomicBool, Ordering}; + struct Offset; + impl BusDevice for Offset { + fn read64(&self, offset: u64) -> BusRead64 { BusRead64::ok(offset) } + fn write64(&self, _offset: u64, _value: u64) -> u32 { BUS_OK } + } + let permitted = Arc::new(AtomicBool::new(true)); + let permission = permitted.clone(); + let window = Window::new(0x1000, 0x88, Arc::new(Offset)) + .permitted(Arc::new(move || permission.load(Ordering::Acquire))); + assert_eq!(window.read64(0x1080).data, 0x80); + assert_eq!(window.read64(0x1088).status, BUS_ERR); + assert_eq!(window.read64(0x1084).status, BUS_ERR); + assert_eq!(window.read64(0x0ff8).status, BUS_ERR); + assert_eq!(window.read64(u64::MAX).status, BUS_ERR); + permitted.store(false, Ordering::Release); + assert_eq!(window.read64(0x1000).status, BUS_ERR); + assert_eq!(window.write64(0x1000, 1), BUS_ERR); + permitted.store(true, Ordering::Release); + assert_eq!(window.write64(0x1000, 1), BUS_OK); +} + +/// Board default setup must preserve the direct RAM path and its shared DMA contents. +#[test] +fn direct_cached_banks_remain_visible_after_board_defaults_are_installed() { + let board = crate::ip35_machine::FuelBoard::new(&[], Arc::new(|| 0)).unwrap(); + assert_eq!(board.md.md_bus().write64(8, 0x33), BUS_OK); + let cached = board.cached_bus(); + assert!(cached.mem_ptr(0x2000).is_some()); + assert_eq!(cached.write64(0x2000, 0x123456789abcdef0), BUS_OK); + assert_eq!(board.md.dma_view().read64(0x2000).data, 0x123456789abcdef0); + assert!(cached.mem_ptr(3 << 57).is_none(), "UC selectors cannot alias cached RAM"); +} + +/// A device overlay replaces its pages, including padding, while its neighbors +/// keep their original address translation. +#[test] +fn sparse_overlays_keep_neighbors_and_refuse_the_new_windows_padding() { + let mut map = SparseWindowMap::default(); + map.map(0x1000, 0x4000, Arc::new(Word(1))); + map.map(0x2040, 0x80, Arc::new(Word(2))); + assert_eq!(map.read64(0x1000).data, 1); + assert_eq!(map.read64(0x2040).data, 2); + assert_eq!(map.read64(0x20b8).data, 2); + assert_eq!(map.read64(0x20bc).status, BUS_ERR); + assert_eq!(map.read64(0x2000).status, BUS_ERR); + assert_eq!(map.read64(0x20c0).status, BUS_ERR); + assert_eq!(map.read64(0x3000).data, 1); + assert_eq!(map.read64(0x5000).status, BUS_ERR); + map.map(0x1000, 0x4000, Arc::new(Word(3))); + assert_eq!(map.ranges.len(), 1); + assert_eq!(map.read64(0x2040).data, 3); +} + +/// Sparse MMIO holes remain bus errors and a revoked cell permission does not +/// fall through to an older mapping beneath the protected window. +#[test] +fn sparse_holes_and_revoked_permissions_never_expose_older_devices() { + use std::sync::atomic::{AtomicBool, Ordering}; + let permission = Arc::new(AtomicBool::new(true)); + let allowed = permission.clone(); + let mut map = SparseWindowMap::default(); + map.map(0, 0x3000, Arc::new(Word(1))); + map.map(0x1000, 8, Arc::new(Window::new(0, 8, Arc::new(Word(2))) + .permitted(Arc::new(move || allowed.load(Ordering::Acquire))))); + assert_eq!(map.read64(0x1000).data, 2); + permission.store(false, Ordering::Release); + assert_eq!(map.read64(0x1000).status, BUS_ERR); + assert_eq!(map.read64(0x1008).status, BUS_ERR); + assert_eq!(map.read64(0x2000).data, 1); + assert_eq!(map.read64(0x3000).status, BUS_ERR); +} + +/// Code-page lookup receives plain physical addresses, while tagged uncached +/// requests retain their view. Cached stores must also update the MD view. +#[cfg(feature = "jitv2")] +#[test] +fn code_generations_use_the_cached_plane_and_stable_ram_endpoint() { + struct Generations { + ram_only: bool, + generation: AtomicU64, + writes: std::sync::Mutex>, + } + impl Generations { + fn new(ram_only: bool) -> Arc { + Arc::new(Self { ram_only, generation: AtomicU64::new(0), writes: Default::default() }) + } + } + impl BusDevice for Generations { + fn gen_ptr(&self, address: u64) -> *const AtomicU64 { + if !self.ram_only || address < 0x1000 { &self.generation } else { std::ptr::null() } + } + fn write64(&self, address: u64, value: u64) -> u32 { + self.writes.lock().unwrap().push((address, value)); + BUS_OK + } + fn write64_masked(&self, address: u64, value: u64, mask: u64) -> u32 { + self.write64(address, value & mask) + } + fn write_block(&self, address: u64, values: &[u64]) -> u32 { + for (index, value) in values.iter().enumerate() { + self.write64(address + index as u64 * 8, *value); + } + BUS_OK + } + } + let physical = Physical::new(Vec::new()); + let mut bus = Ip35Bus::new(physical.clone()); + let raw_ram = Generations::new(false); + let stable_ram = Generations::new(true); + let rom = Generations::new(false); + map_device(&physical, 0, 0x1000, raw_ram.clone(), "raw RAM generation"); + map_device(&physical, 0x1fc00000, 0x1000, rom.clone(), "ROM generation"); + bus.set_ram_generations(stable_ram.clone()); + bus.map(3, 0, 0x1000, stable_ram.clone()); + bus.map(3, 0x3000, 8, stable_ram.clone()); + let cached = bus.cached_view(stable_ram.clone()); + assert_eq!(bus.gen_ptr(0x40), &stable_ram.generation as *const AtomicU64); + assert_eq!(cached.gen_ptr(0x40), &stable_ram.generation as *const AtomicU64); + assert_eq!(bus.gen_ptr(0x1fc00000), &rom.generation as *const AtomicU64); + assert_eq!(cached.gen_ptr(0x1fc00000), &rom.generation as *const AtomicU64); + assert_eq!(bus.gen_ptr((3 << 57) | 0x40), &stable_ram.generation as *const AtomicU64); + assert_eq!(bus.gen_ptr((3 << 57) | 0x3000), &stable_ram.generation as *const AtomicU64); + assert!(bus.gen_ptr((3 << 57) | 0x3008).is_null()); + assert!(bus.gen_ptr((1 << 57) | 0x1fc00000).is_null()); + assert!(bus.gen_ptr(1 << 50).is_null()); + assert_eq!(cached.write64(0x40, 0x1234), BUS_OK); + assert_eq!(cached.write_block(0x48, &[0x5678]), BUS_OK); + assert_eq!(cached.write64_masked(0x50, 0xaabb, 0xff), BUS_OK); + assert_eq!(*stable_ram.writes.lock().unwrap(), [(0x40, 0x1234), (0x48, 0x5678), (0x50, 0xbb)]); + assert_eq!(cached.write64(0x1fc00000, 0x4321), BUS_OK); + assert_eq!(*rom.writes.lock().unwrap(), [(0, 0x4321)], "the ROM window translates to a device-local offset"); + assert!(raw_ram.writes.lock().unwrap().is_empty()); +} + +/// Unmodeled Hub heads retain the PROM compatibility readback without +/// absorbing the adjacent, independently implemented L1 UART window. +#[test] +fn compatibility_hub_heads_do_not_hide_the_l1_uart_window() { + let board = crate::ip35_machine::FuelBoard::new(&[], Arc::new(|| 0)).unwrap(); + assert_ne!(board.md.current_cell() & 1, 0); + for base in [0x1000_0000, 0x3000_0000] { + for offset in [0, 8, 0x40, 0x78] { + let address = base + offset; + assert_eq!(board.bus.write8(address, 0xa5), BUS_OK); + assert_eq!(board.bus.write16(address, 0xa5a5), BUS_OK); + assert_eq!(board.bus.write32(address, 0xa5a5a5a5), BUS_OK); + assert_eq!(board.bus.write64(address, 0x2000), BUS_OK); + let value = board.bus.read8(address); + assert_eq!(value.status, BUS_OK); + assert_eq!(value.data, 0); + let value = board.bus.read16(address); + assert_eq!(value.status, BUS_OK); + assert_eq!(value.data, 0); + let value = board.bus.read32(address); + assert_eq!(value.status, BUS_OK); + assert_eq!(value.data, 0); + let value = board.bus.read64(address); + assert_eq!(value.status, BUS_OK); + assert_eq!(value.data, 0); + } + assert_eq!(board.bus.read64(base + 0x80).status, BUS_OK); + } +} + +/// A busy host port or unusable transcript path must not prevent a serial +/// machine from starting, including when any of its three ports serves CI. +#[test] +fn unavailable_serial_ports_and_logs_do_not_abort_the_machine() { + use crate::config::{FuelConsole, MachineConfig}; + let listeners: Vec<_> = (0..3).map(|_| std::net::TcpListener::bind(("127.0.0.1", 0)).unwrap()).collect(); + let ports: Vec<_> = listeners.iter().map(|socket| socket.local_addr().unwrap().port()).collect(); + let board = crate::ip35_machine::FuelBoard::new(&[], Arc::new(|| 0)).unwrap(); + let mut config = MachineConfig::default(); + config.fuel.l1_port = Some(ports[0]); + config.serial_port_a = Some(ports[1]); + config.serial_port_b = Some(ports[2]); + config.ci = true; + let missing = std::env::temp_dir().join(format!("iris-fuel-missing-log-{}", std::process::id())).join("absent").join("serial.txt"); + assert!(!missing.parent().unwrap().exists(), "the log parent must be absent"); + config.serial_log = Some(missing.to_string_lossy().into_owned()); + config.fuel.l1_log = config.serial_log.clone(); + for (console, selection) in [FuelConsole::L1, FuelConsole::Ioc3A, FuelConsole::Ioc3B].into_iter().enumerate() { + config.fuel.console = selection; + crate::ip35_machine::FuelMachine::connect_serial_for_test(&config, &board, console).unwrap(); + } + config.ci = false; + crate::ip35_machine::FuelMachine::connect_serial_for_test(&config, &board, 0).unwrap(); +} + +/// The Fuel's OHCI root ports each carry a low-speed device, and host +/// input reaches them through the IOC3's PS/2 controller object; the USB +/// pair is fitted only when asked for. +#[test] +fn the_fuel_plugs_a_usb_keyboard_and_mouse_only_when_configured_on() { + use crate::pci::PciBusDevice; + let ports = |usb_input| { + let board = crate::ip35_machine::FuelBoard::with_config(crate::ip35_machine::FuelBoardConfig { + payload: &[], clock: Arc::new(|| 0), flash_path: None, + station_mac: [8, 0, 0x69, 0x12, 0x34, 0x56], + timekeeper: Arc::new(crate::dev::timekeeper::Timekeeper::fuel().unwrap()), + ps2_input: true, usb_input, + l1: Default::default(), + }).unwrap(); + // Power each port: one with a device shows it connected. + board.usb.bar_write(0, 0x54, 0x100, 4); + board.usb.bar_write(0, 0x58, 0x100, 4); + let status = [0x54, 0x58].map(|at| board.usb.bar_read(0, at, 4)); + (board, status) + }; + let (board, status) = ports(true); + for s in status { + assert_eq!(s & 1, 1, "device connected"); + assert_eq!(s & 0x200, 0x200, "low speed"); + } + // The mouse the USB side fits makes the shared controller report a mouse + // only once the guest has configured it. + assert!(!board.ioc3.ps2().mouse_enabled()); + let (_, status) = ports(false); + assert_eq!(status.map(|s| s & 1), [0, 0], "nothing plugged in"); +} diff --git a/src/ip35_machine.rs b/src/ip35_machine.rs new file mode 100644 index 000000000..255bcc563 --- /dev/null +++ b/src/ip35_machine.rs @@ -0,0 +1,1096 @@ +//! The Fuel (IP34 board, IP35 software platform): an R14000 on Bedrock, +//! XBridge and the base I/O devices, with Iris's CPU thread, monitor and serial console. +//! +//! Selected with `[machine] profile = "fuel_ip34"`. Shared keys select the +//! PROM, NVRAM, memory and SCSI devices; `[fuel]` selects the serial console +//! and writable flash. `ip35::Ip35Bus` maps the processor views over the +//! same Physical memory banks. Graphics is not attached, and the IOC3 has +//! no keyboard or mouse ports. +//! +//! During PROM cache-as-memory bootstrap, cached startup bytes live in a +//! Fuel cache adapter until a successful cached fetch leaves the PROM homes. +//! This retirement rule is an approximation, not an established hardware event. +//! +//! # What is approximated here +//! +//! The two Hub-special head windows return zero on reads and acknowledge +//! writes without side effects. PROM 6.210 writes the 64-bit value 0x2000 +//! at relative offsets 0 and 0x40 before DRAM discovery. Their hardware +//! meaning is unknown; Bedrock logging records accesses to these windows. +//! +//! Board composition ported from hw/mips/sgi/sn1-node.c +//! in jamesbraid/qemu-sgi-origami. + +use crate::config::{scsi_label, FuelConsole, MachineConfig}; +use crate::cpu::mips_exec::{CpuDevice, MipsCpu, MipsCpuConfig, MipsExecutor}; +use crate::cpu::mips_tlb::MipsTlb; +use crate::dev::bedrock_flash::{Flash, PART_SIZE}; +use crate::dev::bedrock_io::{BedrockIo, SIZE as IO_SIZE}; +use crate::dev::bedrock_lb::{BedrockLb, SIZE as LB_SIZE}; +use crate::dev::bedrock_md::{BedrockMd, MemoryError}; +use crate::dev::bedrock_ni::{BedrockNi, SIZE as NI_SIZE}; +use crate::dev::bedrock_pi::{reg as pi_reg, BedrockPi, BedrockPiConfig, SIZE as PI_SIZE}; +use crate::dev::bedrock_xb::{BedrockXb, SIZE as XB_SIZE}; +use crate::dev::heart::{host_clock, Clock}; +use crate::dev::ioc3::{Ioc3, Ioc3Config}; +use crate::dev::ioc3_eth::Ioc3Eth; +use crate::dev::isp12160::Isp12160; +use crate::dev::l1::{L1, L1Config, L1Event}; +use crate::dev::ohci::Ohci; +use crate::dev::usb_hid::{UsbKeyboard, UsbMouse}; +use crate::dev::prom_image; +use crate::dev::ps2::Ps2Controller; +use crate::dev::serial_host::{null_serial_backend, open_log, tcp_serial_backend, CiSerialBackend, LogMode, SerialBackend, TeeBackend}; +use crate::dev::timekeeper::Timekeeper; +use crate::dev::xbridge::XBridge; +use crate::devlog::{init_devlog, LogModule}; +use crate::gdb_stub::CpuDebug; +use crate::gfx_display::GfxDisplay; +use crate::hptimer::TimerManager; +use crate::cpu::mips_cache_shadow::R14000FuelShadowCache; +use crate::cpu::mips_cache_v2::MipsCache; +use crate::ip35::{map_device, Ip35Bus, SparseWindowMap, Window, CPU_PHYS_MASK}; +use crate::irq::{InterruptController, IrqLine}; +use crate::machine_state::GuestState; +use crate::monitor::{Monitor, DEFAULT_PORT as DEFAULT_MONITOR_PORT}; +use crate::net::{GatewayConfig, Link, NatControl}; +use crate::perf_monitor::PerfMonitor; +use crate::ppmem::devmap::SLOT_SHIFT; +use crate::scsi::{attach_from_config, ScsiBus}; +use crate::traits::{BusDevice, BusRead8, BusRead16, BusRead32, BusRead64, Device, MachineEvent, Resettable, Signal, BUS_ERR, BUS_OK}; +use crate::workstation::{register_system_controller, DiscChanger, MachineRef, MachineSlot, Workstation}; +use parking_lot::Mutex; +use std::net::{Ipv4Addr, SocketAddr}; +use std::path::{Path, PathBuf}; +use std::sync::mpsc::{channel, Receiver}; +use std::sync::{atomic::{AtomicBool, AtomicU64, Ordering}, Arc}; + +pub struct FuelMachine { + board: Arc, + cpu: Arc, + timers: Arc, + monitor: Arc, + monitor_port: u16, + ci_serial: Option>, + console: usize, + ci: bool, + scratch: Option, + timekeeper: Arc, + nvram: String, + frames: Option>, + nat: Option<( + Arc, + (Ipv4Addr, Ipv4Addr, Ipv4Addr), + )>, + events: Option>, + slot: Option, + lifecycle: GuestState, +} + +impl FuelMachine { + pub fn new(cfg: &MachineConfig) -> Result { + cfg.validate()?; + let prom = prom_image::load(Path::new(&cfg.prom), 8 * 1024 * 1024) + .map_err(|e| format!("Fuel {e}"))?; + let timekeeper = Arc::new(Timekeeper::fuel_with_nvram_file( + Path::new(&cfg.nvram), + )?); + let board = Arc::new(FuelBoard::with_config(FuelBoardConfig { + payload: &prom, + clock: host_clock(), + flash_path: cfg.fuel.flash.as_deref().map(Path::new), + station_mac: cfg.network().mac, + timekeeper: timekeeper.clone(), + ps2_input: cfg.fuel.ps2_input, + usb_input: cfg.fuel.usb_input, + l1: L1Identity { + revision: cfg.fuel.l1_revision.as_deref().map(crate::dev::l1::parse_revision).transpose()?, + brick_serial: cfg.fuel.brick_serial.clone(), + }, + })?); + let timers = Arc::new(TimerManager::new()); + let mut exec = board.build_cpu(); + exec.core.set_timer_manager(timers.clone()); + exec.core.set_count_hz( + cfg.clock + .fixed_mhz + .map_or(250_000_000, |mhz| (mhz * 1_000_000.0) as u64) + .max(1), + ); + exec.core.count_write_samples = true; + let poll_board = board.clone(); + exec.batch_hook = Some(Box::new(move |_| poll_board.poll())); + let cpu: Arc = Arc::new(MipsCpu::<_, R14000FuelShadowCache>::new(*exec)); + if let Some(space) = board.md.physical().ppmem_space() { + use crate::ppmem::MappedMemory; + unsafe { + space.set_bitmap_sink(cpu.ppmem_bitmap_ptr()); + cpu.set_tcache_window(space.window_base()); + let bitmap = cpu.tcache_bitmap_ptr(); + if !bitmap.is_null() { space.set_bitmap_sink2(bitmap); } + #[cfg(feature = "jitv2")] + cpu.set_tcache_gen_window(board.md.generation_base()); + } + board.md.physical().resync_ppmem_bitmap(); + board.md.rebuild_memory(); + } + // After the windows above: each compile worker takes the cache's + // geometry and the CPU model's policies once, as it starts, and + // without the windows the geometry has no inline load or store path. + // Workers read plain physical code addresses, which the cached plane + // serves; the uncached views would decode them as Hub registers. + #[cfg(feature = "jitv2")] + { + let cpu_device: Arc = cpu.clone().as_device(); + let jit = cpu.jitv2(); + let mut g = jit.lock(); + g.compile_queue.set_thread_count(cfg.jitv2.threads.max(1)); + g.compile_queue.set_cpu(Arc::downgrade(&cpu_device)); + g.compile_queue.set_owner(Arc::downgrade(&jit)); + // Not under lockstep, as on the other machines: lockstep compiles + // inline on the CPU thread, and a running pool both owns the + // Codegen inline needs and deadlocks the CPU thread's flush. + #[cfg(not(feature = "jitv2_lockstep"))] + { + let stats = g.stats.clone(); + g.compile_queue.start(board.cached_bus(), stats); + } + // The compile pool keeps only a weak handle on the CPU. + std::mem::forget(cpu_device); + } + board.pi[0].set_interrupt_controller(Arc::new(CpuIrq(Arc::downgrade(&cpu)))); + let (tx, rx) = channel(); + let reset_tx = tx.clone(); + board.lb.connect_soft_reset(Arc::new(move || { + let _ = reset_tx.send(MachineEvent::HardReset); + })); + let reset_tx = tx.clone(); + board.ni.connect_local_reset(Arc::new(move || { + let _ = reset_tx.send(MachineEvent::HardReset); + })); + let weak_cpu = Arc::downgrade(&cpu); + board.l1.set_event_handler(Arc::new(move |event| { + match event { + L1Event::Power(false) | L1Event::Shutdown => { + let _ = tx.send(MachineEvent::PowerOff); + } + L1Event::Power(true) | L1Event::Reset => { + let _ = tx.send(MachineEvent::HardReset); + } + L1Event::SoftReset => { + if let Some(cpu) = weak_cpu.upgrade() { + cpu.as_device().signal(Signal::Reset(true)); + } + } + L1Event::Nmi => dlog!(LogModule::L1, "Fuel: NMI entry is not implemented by the CPU"), + } + })); + let scratch = Self::attach_scsi(cfg, &board)?; + let console = match cfg.fuel.console { + FuelConsole::L1 => 0, + FuelConsole::Ioc3A => 1, + FuelConsole::Ioc3B => 2, + }; + let ci_serial = cfg.ci.then(|| board.serial[console].clone()); + Self::connect_serial(cfg, &board, console)?; + let monitor = Monitor::new(); + monitor.register_device(init_devlog()); + monitor.register_device(Arc::new(crate::cpu::cpu_monitor::CpuMonitor::new(vec![cpu.clone()]))); + monitor.register_device(PerfMonitor::new( + cpu.running_flag(), cpu.cycles_ptr(), cpu.fasttick_count(), None, None, + )); + monitor.register_device(board.md.physical().clone()); + monitor.register_device(board.l1.clone()); + monitor.register_device(board.scsi.clone()); + // `ps2 type` and friends: the keyboard and mouse from the monitor. + monitor.register_device(board.ioc3.ps2()); + cpu.register_locks(); + board.scsi.register_locks(); + board.ioc3.register_locks(); + board.xbridge.register_locks(); + board.flash.register_locks(); + board.md.register_locks(); + let (mut nat, mut frames) = (None, None); + let net = GatewayConfig::from_network(&cfg.network()); + if let Some(link) = Link::with_gateway(net.clone())? { + frames = Some(link.frames()); + nat = Some((link.nat(), (net.client_ip, net.gateway_ip, net.netmask))); + board.eth.attach(link); + } + eprintln!( + "iris: Fuel IP34, R14000 500 MHz, 512 MiB, serial console {:?}", + cfg.fuel.console + ); + Ok(Self { + board, + cpu, + timers, + monitor: Arc::new(monitor), + monitor_port: cfg.monitor_port.unwrap_or(DEFAULT_MONITOR_PORT), + ci_serial, + console, + ci: cfg.ci, + scratch, + timekeeper, + nvram: cfg.nvram.clone(), + nat, + frames, + events: Some(rx), + slot: None, + lifecycle: GuestState::Idle, + }) + } + + fn attach_scsi(cfg: &MachineConfig, board: &FuelBoard) -> Result, String> { + let attached = attach_from_config(cfg, Some("fuel"), &|channel| { + board.scsi.bus_for_channel(channel) + .ok_or_else(|| format!("controller {channel} (a Fuel has channels 0 and 1)")) + })?; + if let Some(device) = attached.daynaports.first() { + return Err(format!("{}: kind = \"daynaport\" is not available on the Fuel", + scsi_label(device.controller, device.id))); + } + Ok(attached.scratch) + } + + fn connect_serial( + cfg: &MachineConfig, + board: &FuelBoard, + console: usize, + ) -> Result<(), String> { + let [a, b] = cfg.serial_ports(); + let ports = [cfg.fuel.l1_port.unwrap_or(8882), a, b]; + if ports[0] == ports[1] || ports[0] == ports[2] || ports[1] == ports[2] { + return Err("Fuel L1 and IOC3 serial ports must be distinct".into()); + } + let labels = ["L1", "IOC3 A", "IOC3 B"]; + let settings = ["fuel.l1_port", "serial_port_a", "serial_port_b"]; + let mut log_config = cfg.clone(); + for (index, port) in ports.into_iter().enumerate() { + log_config.serial_log = if index == console { + cfg.serial_log.clone() + } else if index == 0 { + cfg.fuel.l1_log.clone() + } else { + None + }; + let tag = format!("Fuel {}", labels[index]); + let log = open_log(&log_config, &tag, LogMode::Fresh); + let backend: Arc = if cfg.ci && index == console { + if let Some(log) = log { + board.serial[index].set_log(log); + } + board.serial[index].clone() + } else { + let mut backend = null_serial_backend(); + match tcp_serial_backend(port) { + Some((socket, actual_port)) => { + eprintln!("iris: {tag} on 127.0.0.1:{actual_port}"); + backend = socket; + } + None => eprintln!( + "iris: {tag}: port {port} unavailable; is another iris using it? ({})", + settings[index] + ), + } + match log { + Some(log) => Arc::new(TeeBackend::new(backend, log)), + None => backend, + } + }; + if index == 0 { + board.l1.set_serial_backend(backend); + } else { + board.ioc3.set_serial_backend(index - 1, backend); + } + } + Ok(()) + } + + #[cfg(test)] + pub(crate) fn connect_serial_for_test(cfg: &MachineConfig, board: &FuelBoard, console: usize) -> Result<(), String> { + Self::connect_serial(cfg, board, console) + } + +} + +struct CpuIrq(std::sync::Weak); +impl InterruptController for CpuIrq { + fn set_line(&self, line: u32, level: bool) { + if line < 5 { + if let Some(cpu) = self.0.upgrade() { + cpu.as_device().signal(Signal::Interrupt(line + 2, level)); + } + } + } +} + +impl Workstation for FuelMachine { + fn start(&mut self) { + self.lifecycle = GuestState::Starting; + self.monitor + .clone() + .start_server(format!("127.0.0.1:{}", self.monitor_port)); + self.restart_peripherals(); + if !self.ci { + self.cpu_start(); + } + } + fn stop(&mut self) { + self.cpu.clone().as_device().stop(); + self.board.stop(); + if let Err(e) = self.timekeeper.flush_nvram() { + dlog!(LogModule::Bedrock, "Fuel NVRAM: {e}"); + } + if self.lifecycle != GuestState::Idle { + self.lifecycle = GuestState::Halted; + } + } + fn reset(&mut self) { + self.stop(); + self.cpu.power_on(); + for pi in &self.board.pi { + pi.power_on(); + } + self.board.md.power_on(); + self.board.lb.power_on(); + self.board.io.power_on(); + self.board.ni.power_on(); + self.board.xb.power_on(); + self.board.l1.power_on(); + self.board.flash.power_on(); + self.board.xbridge.power_on(); + self.board.ioc3.power_on(); + self.board.eth.power_on(); + self.board.scsi.power_on(); + self.board.usb.power_on(); + self.timekeeper.power_on(); + self.restart_peripherals(); + } + fn restart_peripherals(&mut self) { + self.board.start(); + } + fn cpu_start(&self) { + self.cpu.clone().as_device().start(); + } + fn cpu_is_running(&self) -> bool { + self.cpu.clone().as_device().is_running() + } + fn machine_slot(&mut self) -> MachineSlot { + let ptr = self as *mut Self as *mut dyn Workstation; + self.slot + .get_or_insert_with(|| Arc::new(Mutex::new(MachineRef(ptr)))) + .clone() + } + fn register_system_controller(&mut self) { + let slot = self.machine_slot(); + let events = self + .events + .take() + .expect("Fuel system controller registered twice"); + register_system_controller(&self.monitor, slot, Some(events)); + } + fn monitor_addr(&self) -> Option { + self.monitor.bound_addr() + } + fn lifecycle(&self) -> GuestState { + self.lifecycle + } + fn set_lifecycle(&mut self, state: GuestState) { + self.lifecycle = state; + } + fn get_display(&self) -> Option> { + None + } + fn register_monitor_device(&self, device: Arc) { + self.monitor.register_device(device); + } + fn get_ps2(&self) -> Arc { + self.board.ioc3.ps2() + } + fn get_timer_manager(&self) -> Arc { + self.timers.clone() + } + fn get_cpu_debug(&self) -> Arc { + self.cpu.clone().debug_adapter() + } + fn fasttick_count(&self) -> Arc { + self.cpu.fasttick_count() + } + fn get_ci_serial(&self) -> Option> { + self.ci_serial.clone() + } + fn inject_serial_console(&self, bytes: &[u8]) { + self.board.serial[self.console].push_host(bytes); + } + fn as_any_mut(&mut self) -> &mut dyn std::any::Any { + self + } + fn save_nvram(&self, path: Option<&str>) -> Result { + if path.is_some_and(|p| p != self.nvram) { + return Err("Fuel NVRAM save uses the configured path".into()); + } + self.timekeeper.flush_nvram()?; + Ok(self.nvram.clone()) + } + fn scratch_path(&self) -> Option<&Path> { + self.scratch.as_deref() + } + fn disc_changer(&self) -> Option> { + Some(self.board.scsi.clone() as Arc) + } + fn pending_chd_sync_count(&self) -> usize { + (0..2).map(|channel| self.board.scsi.bus_for_channel(channel).unwrap().pending_chd_sync_count()).sum() + } + fn flush_disks(&self) -> std::io::Result<()> { + self.board.flush_disks() + } + fn sync_chd_disks( + &self, + only: Option, + progress: &mut dyn FnMut(usize, usize, f32), + cancel: &dyn Fn() -> bool, + ) -> std::io::Result { + let buses: Vec<_> = (0..2).map(|channel| self.board.scsi.bus_for_channel(channel).unwrap()).collect(); + ScsiBus::sync_chd_disks(&buses, only, progress, cancel) + } + fn net_frames(&self) -> Option> { self.frames.clone() } + fn nat_control(&self) -> Option> { + self.nat.as_ref().map(|(n, _)| n.clone()) + } + fn nat_expected(&self) -> Option<(Ipv4Addr, Ipv4Addr, Ipv4Addr)> { + self.nat.as_ref().map(|(_, a)| *a) + } +} + +impl Drop for FuelMachine { + fn drop(&mut self) { + if let Some(slot) = self.slot.take() { + slot.lock().0 = MachineRef::null(); + } + self.stop(); + self.monitor.shutdown(); + } +} + +pub(crate) type FuelCpu = MipsExecutor; + +// The 500 MHz / 2 MiB PROM record selects these boot-mode inputs. +const FUEL_CP0_CONFIG: u32 = 0x6c9a_a985; +const FUEL_BOARD_ID: u64 = 0x4004; + +/// The local node register blocks projected into the processor's I/O view. +pub mod reg { + pub const PI1_BASE: u64 = 0x20_0000; + pub const IO_BASE: u64 = 0x40_0000; + pub const LB_BASE: u64 = 0x60_0000; + pub const NI_BASE: u64 = 0x68_0000; + pub const XB_BASE: u64 = 0x70_0000; + pub const MD_BASE: u64 = 0x78_0000; + pub const MD_SIZE: u64 = 0x8_0000; +} + + +/// Select the named R14000 model and the Fuel PROM boot-mode inputs. +pub(crate) fn build_cpu(bus: Arc) -> Box { + let cfg = MipsCpuConfig::for_model::(Arc::new(crate::ip35::memory_map())); + let mut cpu = Box::new(MipsExecutor::new( + bus as Arc, + MipsTlb::new(cfg.tlb_entries), + &cfg, + )); + cpu.phys_mask = CPU_PHYS_MASK; + // Fuel's 500 MHz/2 MiB flash record selects these boot-mode inputs. + // A mismatch enters the PROM's error routine before GP is initialized. + cpu.core.cp0_config = FUEL_CP0_CONFIG; + cpu +} + +/// Shared device composition for a single-processor Fuel. +/// Graphics is absent; the serial workstation shares these base I/O devices. +pub(crate) struct FuelBoard { + pub bus: Arc, + pub pi: [Arc; 2], + pub md: Arc, + pub l1: Arc, + pub flash: Arc, + pub lb: Arc, + pub io: Arc, + pub ni: Arc, + pub xb: Arc, + pub xbridge: Arc, + pub ioc3: Arc, + pub eth: Arc, + pub scsi: Arc, + pub usb: Arc, + pub serial: [Arc; 3], + #[cfg(test)] + pub irq: Arc, + clock: Clock, + last_clock: AtomicU64, + running: AtomicBool, +} + +pub(crate) struct FuelBoardConfig<'a> { + pub payload: &'a [u8], + pub clock: Clock, + pub flash_path: Option<&'a Path>, + pub station_mac: [u8; 6], + pub timekeeper: Arc, + /// The IOC3's PS/2 keyboard and mouse controller. + pub ps2_input: bool, + /// A USB keyboard and mouse on the OHCI root ports. + pub usb_input: bool, + /// Replacements for the L1's reported identity; unset keeps its defaults. + pub l1: L1Identity, +} + +/// What a Fuel config may replace in the L1's identity replies. +#[derive(Clone, Debug, Default)] +pub(crate) struct L1Identity { + pub revision: Option<[u32; 3]>, + pub brick_serial: Option, +} + +impl FuelBoard { + /// Flush the disks on both channels of the storage controller. + pub fn flush_disks(&self) -> std::io::Result<()> { + (0..2).try_for_each(|channel| self.scsi.bus_for_channel(channel).unwrap().flush()) + } + + #[cfg(test)] + pub fn new(payload: &[u8], clock: Clock) -> Result { + Self::with_config(FuelBoardConfig { + payload, clock, flash_path: None, + station_mac: L1Config::default().mac, + timekeeper: Arc::new(Timekeeper::fuel()?), + ps2_input: true, + usb_input: false, + l1: L1Identity::default(), + }) + } + + pub fn with_config(config: FuelBoardConfig<'_>) -> Result { + let FuelBoardConfig { payload, clock, flash_path, station_mac, timekeeper, ps2_input, usb_input, l1: l1_identity } = config; + let md = BedrockMd::with_clock(clock.clone()); + let phys = md.physical().clone(); + let pi: [Arc; 2] = std::array::from_fn(|interface| { + BedrockPi::new( + BedrockPiConfig { + interface: interface as u8, + request_region: 0, + }, + clock.clone(), + ) + }); + // Only CPU A is fitted on the single-processor board. + pi[0].write64(pi_reg::CPU_A_ENABLE, 1); + let mut l1_config = L1Config { mac: station_mac, ..Default::default() }; + if let Some(revision) = l1_identity.revision { + l1_config.revision = Some(revision); + } + if let Some(serial) = l1_identity.brick_serial { + l1_config.brick_serial = Some(serial); + } + let l1 = Arc::new(L1::new(l1_config)?); + let flash = Arc::new(match flash_path { + Some(path) => Flash::with_file(payload, path), + None => Flash::new(payload), + }.map_err(|e| e.to_string())?); + let lb = Arc::new(BedrockLb::new(clock.clone())); + let io = Arc::new(BedrockIo::new()); + let ni = Arc::new(BedrockNi::new()); + let xb = Arc::new(BedrockXb::new()); + let mut registers = SparseWindowMap::default(); + registers.map( + 0, + PI_SIZE, + pi[0].clone(), + ); + registers.map( + reg::PI1_BASE, + PI_SIZE, + pi[1].clone(), + ); + registers.map( + reg::IO_BASE, + IO_SIZE, + io.clone(), + ); + registers.map( + reg::LB_BASE, + LB_SIZE, + lb.clone(), + ); + registers.map( + reg::NI_BASE, + NI_SIZE, + ni.clone(), + ); + registers.map( + reg::XB_BASE, + XB_SIZE, + xb.clone(), + ); + registers.map( + reg::MD_BASE, + reg::MD_SIZE, + md.md_bus(), + ); + let registers = Arc::new(registers); + io.connect_hub(Arc::new(IoHub(pi.clone()))); + io.connect_memory(md.dma_view()); + let p = pi.clone(); + lb.connect_error(Arc::new(move |level| { + for pi in &p { + pi.set_lb_error(level); + } + })); + let p = pi.clone(); + ni.connect_error(Arc::new(move |level| { + for pi in &p { + pi.set_ni_error(level); + } + })); + let p = pi.clone(); + xb.connect_error(Arc::new(move |level| { + for pi in &p { + pi.set_xb_error(level); + } + })); + let io_node = Arc::downgrade(&io); + lb.connect_node_changed(Arc::new(move |node| { + if let Some(io) = io_node.upgrade() { + io.set_node_number(node); + } + })); + let p = pi.clone(); + md.on_error(Arc::new(move |kind| { + for pi in &p { + match kind { + MemoryError::Correctable => pi.set_pending(1, 61), + MemoryError::Uncorrectable => { + pi.set_memory_uncorrectable(); + } + } + } + })); + #[cfg(test)] + let irq = Arc::new(PendingIrq::default()); + #[cfg(test)] + pi[0].set_interrupt_controller(irq.clone()); + l1.set_irq(IrqLine::new(Arc::new(UartIrq(pi.clone())), 0)); + let serial = std::array::from_fn(|_| Arc::new(CiSerialBackend::new())); + l1.set_serial_backend(serial[0].clone()); + let xbridge = XBridge::new(md.dma_view()); + for widget in [0, 10, 14, 15] { + io.connect_widget(widget, xbridge.widget(widget).unwrap()); + } + let weak_io = Arc::downgrade(&io); + xbridge.set_interrupt_sender(Arc::new(move |packet| { + if packet.widget != 10 { + return false; + } + weak_io.upgrade().is_some_and(|io| { + io.inbound_write( + packet.source_widget, + None, + packet.address, + 8, + packet.vector as u64 | if packet.level { 1 << 8 } else { 0 }, + ) == BUS_OK + }) + })); + let weak_io = Arc::downgrade(&io); + xbridge.set_dma_sender(Arc::new(move |packet, bytes| { + if packet.widget != 10 || packet.half_read_requested || packet.half_write_requested { + dlog!(LogModule::Bedrock, + "Fuel: unsupported XBridge DMA destination or halfword placement {packet:?}" + ); + return false; + } + let Some(io) = weak_io.upgrade() else { + return false; + }; + // Preserve the width of XBridge's naturally aligned packet legs, + // including Hub registers whose narrower accesses are refused. + let size = bytes.len(); + if matches!(size, 1 | 2 | 4 | 8) + && packet.address & (size as u64 - 1) == 0 + { + if packet.write { + let value = bytes.iter().fold(0u64, |value, byte| (value << 8) | *byte as u64); + return io.inbound_write( + packet.source_widget, Some(packet.slot), packet.address, size as u8, value, + ) == BUS_OK; + } + let value = io.inbound_read( + packet.source_widget, Some(packet.slot), packet.address, size as u8, + ); + if value.status != BUS_OK { + return false; + } + bytes.copy_from_slice(&value.data.to_be_bytes()[8 - size..]); + return true; + } + for (i, byte) in bytes.iter_mut().enumerate() { + let a = packet.address + i as u64; + if packet.write { + if io.inbound_write(packet.source_widget, Some(packet.slot), a, 1, *byte as u64) + != BUS_OK + { + return false; + } + } else { + let value = io.inbound_read(packet.source_widget, Some(packet.slot), a, 1); + if value.status != BUS_OK { + return false; + } + *byte = value.data as u8; + } + } + true + })); + let mac = l1.station_mac(); + let mut ioc3_config = Ioc3Config::fuel(mac); + ioc3_config.keyboard_mouse = ps2_input; + let ioc3 = Arc::new(Ioc3::new(ioc3_config)); + let eth = Ioc3Eth::new_fuel(mac); + ioc3.attach_eth(eth.clone()); + ioc3.attach_timekeeper(timekeeper); + ioc3.set_serial_backend(0, serial[1].clone()); + ioc3.set_serial_backend(1, serial[2].clone()); + let scsi = Arc::new(Isp12160::new( + Isp12160::new_bus(), + Isp12160::new_bus(), + )); + xbridge.attach(15, 1, scsi.clone()); + xbridge.attach(15, 4, ioc3.clone()); + let usb = Ohci::new(); + usb.set_clock(clock.clone()); + if usb_input { + // Host input reaches the USB devices through the same controller + // object the GUI, VNC and CI already drive; each device drops + // what its guest has not enabled. + let keyboard = UsbKeyboard::new(); + let mouse = UsbMouse::new(); + usb.attach(0, keyboard.clone())?; + usb.attach(1, mouse.clone())?; + ioc3.ps2().add_sink(keyboard); + ioc3.ps2().add_sink(mouse); + } + xbridge.attach(15, 5, usb.clone()); + for slot in [1, 4, 5] { + xbridge.set_pci_irq(15, slot, 0, xbridge.pin_line(15, slot)); + } + ioc3.connect_superio_interrupt(xbridge.pin_line(15, 0)); + let reset_ioc3 = ioc3.clone(); + let reset_eth = eth.clone(); + let reset_scsi = scsi.clone(); + let reset_usb = usb.clone(); + xbridge.set_reset_output(Arc::new(move |port| { + if port == 15 { + reset_ioc3.power_on(); + reset_eth.power_on(); + reset_scsi.power_on(); + reset_usb.power_on(); + } + })); + let mut bus = Ip35Bus::new(phys.clone()); + #[cfg(feature = "jitv2")] + bus.set_ram_generations(md.dma_view()); + md.rebuild_memory(); + bus.set_fallback(1, 1 << 32, Arc::new(IoPio(io.clone()))); + // The two XIO register forms select the requester's PI0 interface. + let permission_md = Arc::downgrade(&md); + let permission: Arc bool + Send + Sync> = Arc::new(move || { + permission_md.upgrade().is_some_and(|md| md.current_cell() & 1 != 0) + }); + let protected_regs = Arc::new(Window::new(0, 0x80_0000, registers.clone()).permitted(permission.clone())); + for base in [0x0100_0000, 0x0180_0000] { + bus.map(1, base, 0x80_0000, protected_regs.clone()); + } + // Cached RAM uses MD's direct bank mappings; diagnostic views stay separate. + bus.map(3, 0, 1 << 33, md.uncached_view()); + // Hub-special low memory: its first 256 MiB is the node memory view. + bus.map(0, 0, 0x1000_0000, md.uncached_view()); + let prom = Arc::new( + flash + .array_view(0, PART_SIZE) + .map_err(|e| e.to_string())?, + ); + let boot = Arc::new(flash.boot_view()); + for base in [0x1800_0000, 0x3800_0000] { + bus.map(0, base, 0x800_0000, boot.clone()); + } + // Reset execution always sees stored bytes, independently of commands. + bus.map(0, 0x1fc0_0000, 0x40_0000, prom.clone()); + if !payload.is_empty() { + map_device( + &phys, + 0x1fc0_0000, + payload.len() as u64, + prom.clone(), + "Fuel cached reset PROM", + ); + } + let leds = Arc::new(Leds { + md: md.clone(), + values: Mutex::new([0; 4]), + }); + // The three windows occupy the same processor register page. + let special = Arc::new(HubSpecialPage { + windows: [ + Window::new(0, 0x80, Arc::new(HubHead)).permitted(permission.clone()), + Window::new(0x80, 0x40, l1.clone()).permitted(permission.clone()), + Window::new(0xc0, 0x20, leds).permitted(permission.clone()), + ], + }); + for base in [0x1000_0000, 0x3000_0000] { + bus.map(0, base, 1 << SLOT_SHIFT, special.clone()); + bus.map(0, base + 0x400_0000, 8, Arc::new(BoardId(md.clone()))); + } + bus.map(0, 0x1_c000_0000, 1 << 30, md.directory_bus()); + bus.map(0, 0x1_4000_0000, 1 << 30, md.ecc_bus()); + bus.map(0, 0x1_0000_0000, 1 << 30, md.ecc_scrub_bus()); + bus.map(2, 0, 1 << 33, md.fetchop_bus()); + Ok(Self { + bus: Arc::new(bus), + pi, + md, + l1, + flash, + lb, + io, + ni, + xb, + xbridge, + ioc3, + eth, + scsi, + usb, + serial, + #[cfg(test)] + irq, + last_clock: AtomicU64::new(clock()), + clock, + running: AtomicBool::new(false), + }) + } + + pub fn cached_bus(&self) -> Arc { + self.bus.cached_view(self.md.dma_view()) + } + + pub fn build_cpu(&self) -> Box { + let mut cpu = build_cpu(self.bus.clone()); + cpu.cache = R14000FuelShadowCache::new(self.cached_bus()); + // The PROM keeps its stack in cache lines it tags for a window with no + // memory behind it yet; the controller answers that window while they stand. + let memory = self.md.clone(); + cpu.cache.set_pd_tag_store_hook(Box::new(move |tag| memory.startup_tag_store(tag))); + cpu + } + + pub fn start(&self) { + self.last_clock + .store((self.clock)(), Ordering::Release); + self.md.start_clock(); + self.running + .store(true, Ordering::Release); + for pi in &self.pi { + pi.start(); + } + self.l1.start(); + self.ioc3.start(); + self.eth.start(); + self.scsi.start(); + self.usb.start(); + } + pub fn poll(&self) { + if self.running.load(Ordering::Acquire) { + let now = (self.clock)(); + let previous = self + .last_clock + .swap(now, Ordering::AcqRel); + self.flash.advance_ns(now.saturating_sub(previous)); + self.usb.sync_clock(); + for pi in &self.pi { + pi.poll(); + } + } + } + pub fn stop(&self) { + self.poll(); + self.running + .store(false, Ordering::Release); + self.md.stop_clock(); + self.usb.stop(); + self.scsi.stop(); + self.eth.stop(); + self.ioc3.stop(); + self.l1.stop(); + for pi in &self.pi { + pi.stop(); + } + } +} + +// XIO register packets reach only the two PI interrupt-modify registers. +// The processor's register projection has a separate, wider interface. +struct IoHub([Arc; 2]); +impl BusDevice for IoHub { + fn write64(&self, a: u64, v: u64) -> u32 { + if matches!(a, 0x90 | 0x20_0090) && self.0[(a >> 21) as usize].io_modify(v, 0) { + BUS_OK + } else { + BUS_ERR + } + } +} + +struct HubSpecialPage { + windows: [Window; 3], +} +impl HubSpecialPage { + fn target(&self, a: u64, n: u64) -> Option<&Window> { + self.windows.iter().find(|window| window.contains(a, n)) + } +} +macro_rules! hub_page_access { + ($r:ident,$w:ident,$t:ty,$out:ident,$n:expr) => { + fn $r(&self, a: u64) -> $out { + self.target(a, $n).map(|window| window.$r(a)).unwrap_or_else($out::err) + } + fn $w(&self, a: u64, value: $t) -> u32 { + self.target(a, $n).map(|window| window.$w(a, value)).unwrap_or(BUS_ERR) + } + }; +} +impl BusDevice for HubSpecialPage { + hub_page_access!(read8, write8, u8, BusRead8, 1); + hub_page_access!(read16, write16, u16, BusRead16, 2); + hub_page_access!(read32, write32, u32, BusRead32, 4); + hub_page_access!(read64, write64, u64, BusRead64, 8); +} + +struct BoardId(Arc); +impl BusDevice for BoardId { + fn read64(&self, a: u64) -> BusRead64 { + if a == 0 && self.0.current_cell() & 1 != 0 { + BusRead64::ok(FUEL_BOARD_ID) + } else { + BusRead64::err() + } + } +} +// Compatibility head windows: their register contract is not established. +// PROM 6.210 writes 0x2000 at offsets 0 and 0x40 before DRAM discovery. +// Retain the existing zero-read/acknowledged-write model, with accesses +// observable through the Bedrock log, until the register behavior is known. +struct HubHead; +macro_rules! head_access { + ($r:ident,$w:ident,$ty:ty,$result:ident) => { + fn $r(&self, a: u64) -> $result { + dlog!(LogModule::Bedrock, + "Fuel: unidentified Hub-special read {} bytes at {a:#x}", + std::mem::size_of::<$ty>() + ); + $result::ok(0) + } + fn $w(&self, a: u64, v: $ty) -> u32 { + dlog!(LogModule::Bedrock, + "Fuel: unidentified Hub-special write {} bytes at {a:#x} value {v:#x}", + std::mem::size_of::<$ty>() + ); + BUS_OK + } + }; +} +impl BusDevice for HubHead { + head_access!(read8, write8, u8, BusRead8); + head_access!(read16, write16, u16, BusRead16); + head_access!(read32, write32, u32, BusRead32); + head_access!(read64, write64, u64, BusRead64); +} +struct Leds { + md: Arc, + values: Mutex<[u8; 4]>, +} +impl BusDevice for Leds { + fn read64(&self, a: u64) -> BusRead64 { + if a < 32 && a & 7 == 0 && self.md.current_cell() & 1 != 0 { + BusRead64::ok(self.values.lock()[a as usize / 8] as u64) + } else { + BusRead64::err() + } + } + fn write64(&self, a: u64, v: u64) -> u32 { + if a < 32 && a & 7 == 0 && self.md.current_cell() & 1 != 0 { + self.values.lock()[a as usize / 8] = v as u8; + BUS_OK + } else { + BUS_ERR + } + } +} + +struct IoPio(Arc); +macro_rules! pio_access { + ($r:ident,$w:ident,$ty:ty,$result:ident,$size:expr) => { + fn $r(&self, a: u64) -> $result { + let r = self.0.pio_read(a, $size); + $result { + data: r.data as $ty, + status: r.status, + } + } + fn $w(&self, a: u64, v: $ty) -> u32 { + self.0.pio_write(a, $size, v as u64) + } + }; +} +impl BusDevice for IoPio { + pio_access!(read8, write8, u8, BusRead8, 1); + pio_access!(read16, write16, u16, BusRead16, 2); + pio_access!(read32, write32, u32, BusRead32, 4); + pio_access!(read64, write64, u64, BusRead64, 8); +} +impl Drop for FuelBoard { + fn drop(&mut self) { + self.stop(); + } +} + +#[cfg(test)] +#[derive(Default)] +pub(crate) struct PendingIrq(AtomicU64); +#[cfg(test)] +impl PendingIrq { + pub fn bits(&self) -> u64 { + self.0.load(Ordering::Acquire) + } +} +#[cfg(test)] +impl InterruptController for PendingIrq { + fn set_line(&self, line: u32, level: bool) { + if line >= 5 { + return; + } + let mask = 1 << (10 + line); + if level { + self.0.fetch_or(mask, Ordering::Release); + } else { + self.0 + .fetch_and(!mask, Ordering::Release); + } + } +} +struct UartIrq([Arc; 2]); +impl InterruptController for UartIrq { + fn set_line(&self, _: u32, level: bool) { + for pi in &self.0 { + pi.set_uart_irq(level); + } + } +} diff --git a/src/lib.rs b/src/lib.rs index c4b68b397..44165db2c 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -181,6 +181,9 @@ pub mod ip32_machine; /// The SGI Octane (IP30): its bus and machine -- see `docs/Octane/ip30-address-map.md`. pub mod ip30; pub mod ip30_machine; +/// Fuel has an IP34 board and uses the IP35 software platform and processor views. +pub mod ip35; +pub mod ip35_machine; pub mod machine_state; pub mod platform; pub mod physical; diff --git a/src/machine.rs b/src/machine.rs index 3808a7430..058dd1854 100644 --- a/src/machine.rs +++ b/src/machine.rs @@ -53,7 +53,7 @@ pub fn set_emulator_profile(profile: MachineProfile) { fn emulator_profile() -> Option { let raw = EMULATOR_PROFILE.load(std::sync::atomic::Ordering::Relaxed); - [MachineProfile::IndyIp24, MachineProfile::Indigo2Ip22, MachineProfile::Indigo2Ip28, MachineProfile::O2Ip32, MachineProfile::OctaneIp30] + [MachineProfile::IndyIp24, MachineProfile::Indigo2Ip22, MachineProfile::Indigo2Ip28, MachineProfile::O2Ip32, MachineProfile::OctaneIp30, MachineProfile::FuelIp34] .into_iter().find(|p| *p as u8 == raw) } @@ -68,6 +68,7 @@ fn emulator_name_thirds(profile: Option) -> &'static [&'static s Some(MachineProfile::Indigo2Ip28) => &["IRIX", "Iris", "IPsomething", "Indigo2", "IP28"], Some(MachineProfile::O2Ip32) => &["IRIX", "Iris", "IPsomething", "IP32"], Some(MachineProfile::OctaneIp30) => &["IRIX", "Iris", "IPsomething", "IP30"], + Some(MachineProfile::FuelIp34) => &["IRIX", "Iris", "IPsomething", "IP34"], None => &["IRIX", "Iris", "IPsomething"], } } @@ -2201,11 +2202,12 @@ mod emulator_name_tests { Some(MachineProfile::Indigo2Ip28) => &["Indy", "IP24", "IP22", "IP32", "O2"], Some(MachineProfile::O2Ip32) => &["Indy", "Indigo2", "IP24", "IP22", "IP28", "IP30", "O2"], Some(MachineProfile::OctaneIp30) => &["Indy", "Indigo2", "IP24", "IP22", "IP28", "IP32", "O2"], + Some(MachineProfile::FuelIp34) => &["Indy", "Indigo2", "IP24", "IP22", "IP28", "IP30", "IP32", "O2"], None => &["Indy", "Indigo2", "IP24", "IP22", "IP28", "IP30", "IP32", "O2"], } }; for p in [None, Some(MachineProfile::IndyIp24), Some(MachineProfile::Indigo2Ip22), - Some(MachineProfile::Indigo2Ip28), Some(MachineProfile::O2Ip32), Some(MachineProfile::OctaneIp30)] { + Some(MachineProfile::Indigo2Ip28), Some(MachineProfile::O2Ip32), Some(MachineProfile::OctaneIp30), Some(MachineProfile::FuelIp34)] { let thirds = emulator_name_thirds(p); assert!(!thirds.is_empty()); for w in thirds { @@ -2217,7 +2219,7 @@ mod emulator_name_tests { #[test] fn profile_round_trips_through_the_global() { - for p in [MachineProfile::IndyIp24, MachineProfile::Indigo2Ip22, MachineProfile::Indigo2Ip28, MachineProfile::O2Ip32, MachineProfile::OctaneIp30] { + for p in [MachineProfile::IndyIp24, MachineProfile::Indigo2Ip22, MachineProfile::Indigo2Ip28, MachineProfile::O2Ip32, MachineProfile::OctaneIp30, MachineProfile::FuelIp34] { set_emulator_profile(p); assert_eq!(emulator_profile(), Some(p)); } diff --git a/src/workstation.rs b/src/workstation.rs index 0d5537a80..8ea301da6 100644 --- a/src/workstation.rs +++ b/src/workstation.rs @@ -40,6 +40,7 @@ use crate::traits::{Device, MachineEvent}; pub fn build(cfg: MachineConfig) -> Result, String> { crate::machine::set_emulator_profile(cfg.machine.profile); match cfg.machine.profile { + crate::config::MachineProfile::FuelIp34 => Ok(Box::new(crate::ip35_machine::FuelMachine::new(&cfg)?)), // The O2 is its own machine (CRIME, MACE, GBE), not an MC/HPC3 one. crate::config::MachineProfile::O2Ip32 => Ok(Box::new(crate::ip32_machine::O2Machine::new(&cfg)?)), // So is the Octane (HEART, Xbow, Bridge). From 45b9a5b61069adf321a44f6831a1ec2a9899189f Mon Sep 17 00:00:00 2001 From: James Braid Date: Sat, 31 Oct 2026 08:26:55 -0700 Subject: [PATCH 28/31] jitv2: compile a page through the view the CPU fetched it by The IP35 PROM fetches its own code uncached at plain physical 0x1fcxxxxx. Bedrock serves that from the reset PROM, but the compile pool read through the cached view, and once a memory bank covers the window the cached view answers RAM there. The memory test zeroes it, so the JIT compiled pages of NOPs in place of PROM routines and the PROM faulted (an instruction fetch bus error, or endless refills). Code pages were keyed by frame alone, so the two views could not be told apart. Where the machine says its uncached view differs (the Fuel; other machines do not opt in and keep one page per frame), an uncached fetch keys frame | UNCACHED_VIEW_PFN. The page is read and its generation counter taken through UNCACHED_VIEW_ADDR on a compile bus that sends such addresses to the uncached attribute-0 view, the device the CPU's own fetch reached. Writes invalidate by the device that holds the changed bytes: a cached store to RAM over the reset window leaves the PROM's page alone, a flash command invalidates it, and where both views reach the same RAM they share its counter. The view bit is a constant, bit 22 of the key (the 16 GiB window's frame count), so the fetch and SMC paths strip it with an immediate mask and the per-instruction path is untouched. No machine has a frame at or past it, so the strip is a no-op where the bit is never set: an 8 GB Octane's frames above 4 GiB, which use bit 20, keep their true addresses. Any machine could opt in. Opting in grows its frame map from its memory's frames to the view bit plus those frames (16 MiB plus four bytes a frame), so both key spaces index it directly; the other machines keep the map sized to their memory. The Fuel's CHANGELOG entry sits under an "SGI Fuel" heading whose first bullet arrives with the documentation commit. The compile pool restarted after a page-pool flush used the CPU's uncached bus instead of the cached view it was started with; the executor now keeps the compile bus and passes it to every restart and to the inline compile. The monitor uses it too: `j2 inline off`, `dumppcp`, `corpus` and `j2 html` read pages through it, and `j2 clear` and `j2 deny` take a trailing `uncached` to name the uncached view's page. Tests: a pool restart after a flush keeps the compile bus; an executor that has not opted in keeps the plain key and the frame map sized to its memory; uncached attributes 1 to 3 and addresses over 4 GiB, even where a device answers them, get no code page; frames above 4 GiB keep their true address and bus address without the view, and the uncached view still names its frame and view for frames above 4 GiB. Both new tests fail with the view bit at 20. --- CHANGELOG.md | 9 ++ HACKING.md | 16 +++ src/bin/jitv2_analyze.rs | 4 +- src/bin/jitv2_pcp_dump.rs | 2 +- src/cpu/jitv2/comp.rs | 5 +- src/cpu/jitv2/jitv2.rs | 128 ++++++++++++++++++- src/cpu/jitv2/pcp_dump.rs | 4 +- src/cpu/jitv2_html_j2wp.rs | 4 +- src/cpu/mips_exec.rs | 97 +++++++++++--- src/cpu/mips_exec/monitor.rs | 46 ++++--- src/ip35.rs | 40 ++++++ src/ip35/tests.rs | 240 +++++++++++++++++++++++++++++++++++ src/ip35_machine.rs | 19 ++- 13 files changed, 563 insertions(+), 51 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e8c55951b..8e3c43f73 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -262,6 +262,15 @@ hashes are given where a change is easiest to understand by reading the commit. - Memory past 4 GB is as fast as the rest: such a machine gets a 16 GB direct window, and the JIT compiles code anywhere in it. - A store-conditional on memory outside the window is now atomic with two CPUs. +### SGI Fuel (IP34 board, IP35 software platform) + +- **jitv2 compiles uncached code from the uncached view.** The PROM runs from the + reset window uncached; once memory covers that window the cached view returns + RAM there, and the compile pool used to compile RAM's bytes in place of the + PROM's. An uncached fetch now keys a code page of its own and is compiled and + invalidated through the view the CPU fetched it by. Other machines are + unchanged. + ### PCI configuration - A PCI function can describe its own writable header fields and diff --git a/HACKING.md b/HACKING.md index 6b89acbf0..bfce0fbab 100644 --- a/HACKING.md +++ b/HACKING.md @@ -572,6 +572,22 @@ code into host code with Cranelift. The concepts: pfn map publishes a slot after its claim, and inline stores bump generations atomically. Compiled code is freed only by a full flush, and every flush parks all CPUs first (`jitv2_flush_from_here`, `run_leader_flush`). +- **Views of a frame.** On most machines a physical frame has one set of bytes, + so a page is keyed by frame alone. The IP35 bus decodes an uncached fetch + (attribute 0) differently from a cached one: the reset PROM answers the + uncached view at 0x1fc00000 whatever RAM sits behind the cached view there. + `MipsExecutor::enable_jitv2_uncached_views` makes an uncached fetch key + `frame | UNCACHED_VIEW_PFN` (`jitv2::code_key`). Such a page is read, and its + generation counter taken, through `UNCACHED_VIEW_ADDR` on the compile bus + (`Ip35Bus::code_view`), which reaches the same device the CPU's uncached fetch + reached. A write bumps the counter of the device that holds the bytes it changed: + a cached store to RAM over the reset window leaves the PROM's page alone, a flash + command on the PROM invalidates it, and where both views reach the same RAM + they share its counter. Frames past the end of the machine's memory and uncached + attributes 1 to 3 still run in the interpreter. The view bit (bit 22) is above + every frame of every machine (the largest physical window is 16 GiB), so a + machine that does not opt in keeps its true frame numbers whatever its memory; + one that does has a frame map of the view bit plus its memory's frames. - Compiled pages can optionally be cached on disk across runs (`[jitv2] cache`, [docs/jitv2-persistent-cache.md](docs/jitv2-persistent-cache.md)). diff --git a/src/bin/jitv2_analyze.rs b/src/bin/jitv2_analyze.rs index aba0b247e..5b2ca948d 100644 --- a/src/bin/jitv2_analyze.rs +++ b/src/bin/jitv2_analyze.rs @@ -108,8 +108,8 @@ fn main() { }; let entry_word = entry.offset / 4; - let page_base = entry.pfn.wrapping_mul(iris::cpu::jitv2::PAGE_SIZE); - let (result, non_empty) = analyzer.walk(&page, entry_word, page_base); + let page_base = iris::cpu::jitv2::jitv2::key_phys_base(entry.pfn); + let (result, non_empty) = analyzer.walk(&page, entry_word, page_base as u32); stats.files_walked += 1; if entry_word == 1 { diff --git a/src/bin/jitv2_pcp_dump.rs b/src/bin/jitv2_pcp_dump.rs index 06f7765f0..1868d5c25 100644 --- a/src/bin/jitv2_pcp_dump.rs +++ b/src/bin/jitv2_pcp_dump.rs @@ -76,7 +76,7 @@ fn main() { } }; - let page_base_phys = dump.pfn * PAGE_SIZE; + let page_base_phys = iris::cpu::jitv2::jitv2::key_phys_base(dump.pfn); println!("=== {} ===", path); println!("pfn={:#010x} phys_base={:#010x}", dump.pfn, page_base_phys); println!("current_gen={} entry_gen={} {}", diff --git a/src/cpu/jitv2/comp.rs b/src/cpu/jitv2/comp.rs index 1979b36ee..bb6ce1e23 100644 --- a/src/cpu/jitv2/comp.rs +++ b/src/cpu/jitv2/comp.rs @@ -127,7 +127,8 @@ fn prepare_multi_entry_compile( #[cfg(feature = "developer")] stats: &crate::cpu::jitv2::JitStats, ) -> PrepareOutcome { let page = unsafe { &*req.page }; - let phys_base = page.pfn as u64 * PAGE_SIZE as u64; + let key = page.pfn; + let phys_base = page.phys_base(); // Dirty-cache gate. The seqlock snapshot below reads RAM off the bus; the // guest CPU sees RAM overlaid with its own dirty L1-D/L2 lines, and a @@ -166,7 +167,7 @@ fn prepare_multi_entry_compile( let gen0 = page.current_gen(); let mut read_ok = true; for (i, w) in words.iter_mut().enumerate() { - let r = bus.read32(phys_base + i as u64 * 4); + let r = bus.read32(crate::cpu::jitv2::jitv2::key_bus_addr(key, (i as u32) * 4)); if !r.is_ok() { read_ok = false; break; diff --git a/src/cpu/jitv2/jitv2.rs b/src/cpu/jitv2/jitv2.rs index 9fd970165..766285d9a 100644 --- a/src/cpu/jitv2/jitv2.rs +++ b/src/cpu/jitv2/jitv2.rs @@ -80,6 +80,43 @@ pub fn pfn_of(phys_addr: u64, frames: u64) -> Option { (pfn < frames).then_some(pfn as Pfn) } +/// Set in a page key when the CPU fetched the page uncached on a machine +/// whose uncached bus view decodes differently from its cached one (IP35). +/// The key is `pfn | UNCACHED_VIEW_PFN`; the other machines never set it. +/// It is bit 22, above every frame of every machine: the largest physical +/// window is 16 GiB, [`PFN_MAP_ENTRIES`] = 1 << 22 frames, and [`pfn_of`] +/// admits no frame at or past the machine's `frames`. So stripping it is a +/// no-op on a machine that never sets it, however much memory that machine +/// has (an 8 GB Octane's frames reach bit 21). A constant, so the hot paths +/// that strip it need no load. See [`code_key`]. +pub const UNCACHED_VIEW_PFN: Pfn = PFN_MAP_ENTRIES as Pfn; + +/// Set in the bus address a code page is read or generation-tracked through +/// when the page was fetched uncached. A bus that has no separate uncached +/// code view never sees it (the key bit is never set there). +pub const UNCACHED_VIEW_ADDR: u64 = 1 << 63; + +/// The page key for a fetch at `phys_addr`: its frame below `frames`, plus +/// [`UNCACHED_VIEW_PFN`] when `uncached_view` says this fetch reached memory +/// through the uncached view. +#[inline(always)] +pub fn code_key(phys_addr: u64, frames: u64, uncached_view: bool) -> Option { + pfn_of(phys_addr, frames).map(|pfn| if uncached_view { pfn | UNCACHED_VIEW_PFN } else { pfn }) +} + +/// Physical base address of the frame a page key names, without the view bit. +#[inline(always)] +pub fn key_phys_base(key: Pfn) -> u64 { + (key & !UNCACHED_VIEW_PFN) as u64 * PAGE_SIZE as u64 +} + +/// Bus address for reading or tracking the page `key`, in the view it names. +#[inline(always)] +pub fn key_bus_addr(key: Pfn, offset: u32) -> u64 { + let addr = key_phys_base(key) + offset as u64; + if key & UNCACHED_VIEW_PFN != 0 { addr | UNCACHED_VIEW_ADDR } else { addr } +} + /// Direct-mapped `pfn -> PageSlot` lookup, replacing the `HashMap` both page pools used to carry. /// @@ -1285,6 +1322,18 @@ impl PhysicalCodePage { self.pfn != UNCLAIMED_PFN } + /// Physical base address of the frame this page covers. + #[inline] + pub fn phys_base(&self) -> u64 { + key_phys_base(self.pfn) + } + + /// Whether the CPU fetched this page through the uncached view. + #[inline] + pub fn is_uncached_view(&self) -> bool { + self.pfn != UNCLAIMED_PFN && self.pfn & UNCACHED_VIEW_PFN != 0 + } + pub fn claim(&mut self, pfn: Pfn, gen: *const AtomicU64, fr1: bool) { debug_assert!(std::ptr::eq(self.gen, &NEVER_COMPILABLE_GEN) && self.pfn == UNCLAIMED_PFN, "claim() called on a slot that wasn't clean (pfn={:#x}) — every path that reuses a slot must reset it first (see reset_to_unclaimed)", @@ -2307,6 +2356,10 @@ pub struct Jitv2 { /// lands on a different PFN than the currently-tracked one) — not on /// every fetch. Direct-mapped, not hashed — see [`PfnMap`]. pfn_to_slot: PfnMap, + /// Frames of the machine's memory, the plain keys `pfn_to_slot` holds. + frames: usize, + /// `pfn_to_slot` also holds uncached-view keys (`enable_uncached_views`). + uncached_views: bool, /// Pool capacity, fixed at construction (== `pages.len()`). Claiming past /// this (i.e. with the free list exhausted) triggers `mega_flush` (the /// "ran out of PCPs" resource-exhaustion trigger). @@ -2414,6 +2467,8 @@ impl Jitv2 { pages, free_head: if capacity > 0 { 0 } else { NO_SLOT }, pfn_to_slot: PfnMap::new(frames), + frames, + uncached_views: false, capacity, compile_queue: CompileQueue::new(), codegen: Mutex::new(Some(crate::cpu::jitv2::codegen::Codegen::new())), @@ -2449,13 +2504,38 @@ impl Jitv2 { } let slot = self.free_head; self.free_head = self.pages[slot as usize].next; - let gen = bus.gen_ptr(phys_addr); + // The generation counter belongs to the view the page is read + // through: `phys_addr` names the frame, the key says which view. + let gen_addr = phys_addr | if pfn & UNCACHED_VIEW_PFN != 0 { UNCACHED_VIEW_ADDR } else { 0 }; + let gen = bus.gen_ptr(gen_addr); self.pages[slot as usize].claim(pfn, gen, fr1); self.pages[slot as usize].next = NO_SLOT; self.pfn_to_slot.insert(pfn, slot); Some(slot) } + /// Whether the pool holds uncached-view pages (`enable_uncached_views`). + pub fn uncached_views_enabled(&self) -> bool { + self.uncached_views + } + + /// Let the pool hold uncached-view pages ([`UNCACHED_VIEW_PFN`]): the map + /// grows from `frames` entries to the view bit plus `frames`, so each + /// view's keys index it directly, as plain frames do. Call before any + /// page is claimed and before an executor captures the lookup pointers + /// (`jitv2_bind_fast_lookup`); the map is reallocated, to 16 MiB plus + /// four bytes a frame. + pub fn enable_uncached_views(&mut self) { + if self.uncached_views { + return; + } + assert_eq!(self.pages_used(), 0, "uncached code views must be enabled before any page is claimed"); + assert!(self.frames <= UNCACHED_VIEW_PFN as usize, + "memory reaches past the page table's largest window ({} frames)", self.frames); + self.pfn_to_slot = PfnMap::new(UNCACHED_VIEW_PFN as usize + self.frames); + self.uncached_views = true; + } + /// Pointers for lock-free `pfn -> *mut PhysicalCodePage` fast lookup. pub fn fast_lookup_ptrs(&mut self) -> (*const AtomicU32, *mut PhysicalCodePage) { (self.pfn_to_slot.as_ptr(), self.pages.as_mut_ptr()) @@ -5167,6 +5247,52 @@ mod tests { assert_eq!(pfn_of(0x3000_0000, indy), None); } + /// A machine that never sets the view bit keeps its frames' true + /// addresses, including frames at or above 4 GiB (bit 20 of a frame + /// number), as an Octane with 8 GB has. + #[test] + fn frames_above_4_gib_keep_their_address_without_the_view() { + let octane = frames_for(&crate::ip30::memory_map(&[2048, 2048, 2048, 2048])); + for addr in [0x1_0000_1000u64, 0x1_ffff_f000, 0x2_1fff_f000] { + let key = code_key(addr, octane, false).unwrap(); + assert_eq!(key, (addr / PAGE_SIZE as u64) as Pfn); + assert_eq!(key & UNCACHED_VIEW_PFN, 0, "{addr:#x}"); + assert_eq!(key_phys_base(key), addr & !(PAGE_SIZE as u64 - 1), "{addr:#x}"); + assert_eq!(key_bus_addr(key, 8), (addr & !(PAGE_SIZE as u64 - 1)) + 8, "{addr:#x} has no view address bit"); + } + let dev = FakeDevice(AtomicU64::new(0)); + let mut jit = Jitv2::with_frames(4, octane as usize); + let slot = jit.page_for(0x100001, 0x1_0000_1000, &dev, false).unwrap(); + let page = unsafe { &*jit.page_ptr(slot) }; + assert_eq!(page.phys_base(), 0x1_0000_1000); + assert!(!page.is_uncached_view()); + assert_eq!(key_bus_addr(page.pfn, 0), 0x1_0000_1000); + } + + /// An uncached-view key still names its frame and view when the frame is + /// above 4 GiB, and the view bit stays clear of the largest frame. + #[test] + fn the_uncached_view_survives_frames_above_4_gib() { + let frames = PFN_MAP_ENTRIES as u64; + assert_eq!(UNCACHED_VIEW_PFN as u64, frames); + for addr in [0x1fc0_0000u64, 0x1_0000_1000, 0x3_ffff_f000] { + let key = code_key(addr, frames, true).unwrap(); + assert_ne!(key & UNCACHED_VIEW_PFN, 0); + assert_eq!(key_phys_base(key), addr & !(PAGE_SIZE as u64 - 1), "{addr:#x}"); + assert_eq!(key_bus_addr(key, 0), (addr & !(PAGE_SIZE as u64 - 1)) | UNCACHED_VIEW_ADDR, "{addr:#x}"); + } + let dev = FakeDevice(AtomicU64::new(0)); + let mut jit = Jitv2::with_frames(4, 0xd0000); + jit.enable_uncached_views(); + let key = code_key(0x1fc0_0000, 0xd0000, true).unwrap(); + let slot = jit.page_for(key, 0x1fc0_0000, &dev, false).unwrap(); + let page = unsafe { &*jit.page_ptr(slot) }; + assert!(page.is_uncached_view()); + assert_eq!(page.phys_base(), 0x1fc0_0000); + let plain = jit.page_for(0x1fc00, 0x1fc0_0000, &dev, false).unwrap(); + assert_ne!(plain, slot, "the two views of a frame are two pages"); + } + #[test] fn page_for_allocates_once_and_caches_by_pfn() { let dev = FakeDevice(AtomicU64::new(0)); diff --git a/src/cpu/jitv2/pcp_dump.rs b/src/cpu/jitv2/pcp_dump.rs index 7c0032b0c..d19bf713e 100644 --- a/src/cpu/jitv2/pcp_dump.rs +++ b/src/cpu/jitv2/pcp_dump.rs @@ -134,11 +134,11 @@ impl PcpDump { /// possibility for a debugging dump, not a correctness-critical path). /// Returns `Err` with the failing physical address if any word read fails. pub fn capture(page: &PhysicalCodePage, bus: &dyn BusDevice) -> Result { - let phys_base = page.pfn as u64 * PAGE_SIZE as u64; + let phys_base = page.phys_base(); let mut words = [0u32; ENTRIES_PER_PAGE]; for (i, w) in words.iter_mut().enumerate() { let addr = phys_base + i as u64 * 4; - let r = bus.read32(addr); + let r = bus.read32(crate::cpu::jitv2::jitv2::key_bus_addr(page.pfn, (i as u32) * 4)); if !r.is_ok() { return Err(addr); } diff --git a/src/cpu/jitv2_html_j2wp.rs b/src/cpu/jitv2_html_j2wp.rs index 3bf2585b5..0bcdf4adf 100644 --- a/src/cpu/jitv2_html_j2wp.rs +++ b/src/cpu/jitv2_html_j2wp.rs @@ -477,13 +477,13 @@ pub fn write_jitv2_html( // (Jitv2::code_bytes_used doc comment) — fine for an on-demand dump, // not a hot path. for page in jit.claimed_pages() { - let phys_addr = page.pfn as u64 * crate::cpu::jitv2::PAGE_SIZE as u64; + let phys_addr = page.phys_base(); let window = memory.find(phys_addr).map_or(OTHER_REGION, |r| r.name); let page_gen = page.current_gen(); let mut words = [0u32; crate::cpu::jitv2::ENTRIES_PER_PAGE]; for (i, w) in words.iter_mut().enumerate() { - *w = bus.read32(phys_addr + i as u64 * 4).data; + *w = bus.read32(crate::cpu::jitv2::jitv2::key_bus_addr(page.pfn, (i as u32) * 4)).data; } // §13: one function/gen/instr_count/code_size/call_count/ diff --git a/src/cpu/mips_exec.rs b/src/cpu/mips_exec.rs index 0bb4133b9..f71219fba 100644 --- a/src/cpu/mips_exec.rs +++ b/src/cpu/mips_exec.rs @@ -1003,6 +1003,16 @@ pub struct MipsExecutor { /// short loop). #[cfg(feature = "jitv2")] pub jitv2_inline_compile: bool, + /// The machine's uncached bus view decodes code differently from its + /// cached one (IP35), so an uncached fetch keys its own code page and + /// is compiled from what the uncached view returns. Set by + /// [`Self::enable_jitv2_uncached_views`]; off on every other machine. + #[cfg(feature = "jitv2")] + pub(crate) jitv2_uncached_views: bool, + /// The bus compiles read code through, when it is not `sysad` itself + /// (see [`Self::enable_jitv2_uncached_views`]). + #[cfg(feature = "jitv2")] + jitv2_code_bus: Option>, /// Scratch analyzer for `jitv2_inline_compile`'s synchronous path — same /// `comp::handle_request` compile sequence the real async compile-thread /// runs, just called directly on the CPU thread. Kept alive across calls @@ -2490,6 +2500,10 @@ impl MipsExecutor { #[cfg(feature = "jitv2")] jitv2_inline_compile: cfg!(feature = "jitv2_lockstep"), #[cfg(feature = "jitv2")] + jitv2_uncached_views: false, + #[cfg(feature = "jitv2")] + jitv2_code_bus: None, + #[cfg(feature = "jitv2")] jitv2_inline_analyzer: crate::cpu::jitv2::analyzer::Analyzer::with_isa(C::MIPS4), // IRIS_JIT_DISPATCH=off starts with the JIT dispatch gate shut: the // same binary as an interpreter-only reference, for comparing @@ -2788,7 +2802,7 @@ impl MipsExecutor { self.core.nanotlb[AT as usize].fill_raw(va_page, result.phys, result.status & 0x7); #[cfg(feature = "jitv2")] if AT == AccessType::Fetch as u8 { - self.jitv2_track_pcp(result.phys); + self.jitv2_track_pcp(result.phys, !result.is_cached()); } } result @@ -3002,7 +3016,7 @@ impl MipsExecutor { if !self.smc_seen.insert(key) { return; } - let page_base = pfn as u64 * crate::cpu::jitv2::PAGE_SIZE as u64; + let page_base = crate::cpu::jitv2::jitv2::key_phys_base(pfn); let word = ((phys_addr - page_base) / 4) as usize; eprintln!( "jitv2 SMC: write to the page being executed — pc={:#018x} wrote {} byte(s) \ @@ -3019,12 +3033,16 @@ va={:#018x} phys={:#010x} (code pfn {:#x}, page {:#010x}, word {}/{})", /// [`Self::smc_check_report`]. /// /// `cur_code_pfn == u32::MAX` (no page tracked) can never match a real - /// frame, so the "nothing tracked" case needs no separate test. + /// frame, so the "nothing tracked" case needs no separate test. The + /// comparison masks the view bit off `cur_code_pfn`, which leaves the + /// sentinel as 0xFFBFFFFF: still beyond any frame the page table holds. #[cfg(all(feature = "jitv2", feature = "jitv2_smc_check"))] #[inline(always)] fn smc_check_write(&mut self, virt_addr: u64, phys_addr: u64, size: usize) { let pfn = self.core.cur_code_pfn; - if (phys_addr / crate::cpu::jitv2::PAGE_SIZE as u64) as u32 == pfn { + if (phys_addr / crate::cpu::jitv2::PAGE_SIZE as u64) as u32 + == pfn & !crate::cpu::jitv2::jitv2::UNCACHED_VIEW_PFN + { self.smc_check_report(virt_addr, phys_addr, size, pfn); } } @@ -3063,7 +3081,7 @@ va={:#018x} phys={:#010x} (code pfn {:#x}, page {:#010x}, word {}/{})", /// may have made the flush unnecessary. #[cfg(feature = "jitv2")] pub(crate) fn jitv2_flush_from_here(&mut self, only_if_exhausted: bool) { - let bus = self.sysad.clone(); + let bus = self.jitv2_compile_bus(); let jit = self.jitv2.clone(); let flush = || { let mut j = jit.lock(); @@ -3092,6 +3110,40 @@ va={:#018x} phys={:#010x} (code pfn {:#x}, page {:#010x}, word {}/{})", paddr, self.memory)) } + /// The page key for a fetch at `phys_addr`: its frame, in the uncached + /// view's key space when `uncached` and this machine keeps one + /// (`jitv2::code_key`). `None` above the page table. + #[cfg(feature = "jitv2")] + pub(crate) fn jitv2_code_key(&self, phys_addr: u64, uncached: bool) -> Option { + crate::cpu::jitv2::jitv2::code_key(phys_addr, self.jitv2_frames, uncached && self.jitv2_uncached_views) + } + + /// Tell the JIT this machine's uncached bus view decodes code differently + /// from its cached view, so uncached fetches get their own code pages and + /// are compiled from `code_bus`, which must serve a page's bytes through + /// the view its key names (see `jitv2::key_bus_addr`). Call while the + /// executor is being built, before it is shared or run. Machines without + /// such views never call this and keep one page per frame. + #[cfg(feature = "jitv2")] + pub fn enable_jitv2_uncached_views(&mut self, code_bus: Arc) { + self.jitv2.lock().enable_uncached_views(); + self.jitv2_uncached_views = true; + self.jitv2_code_bus = Some(code_bus); + self.jitv2_bind_fast_lookup(); + } + + /// The code page the last fetch landed on, or null. + #[cfg(feature = "jitv2")] + pub fn jitv2_current_page(&self) -> *mut crate::cpu::jitv2::PhysicalCodePage { + self.pcp + } + + /// The bus compile workers and the inline compile read code through. + #[cfg(feature = "jitv2")] + pub fn jitv2_compile_bus(&self) -> Arc { + self.jitv2_code_bus.clone().unwrap_or_else(|| self.sysad.clone()) + } + #[cfg(feature = "jitv2")] fn jitv2_share(&mut self, jit: Arc>, world: Arc) { { @@ -3136,10 +3188,14 @@ va={:#018x} phys={:#010x} (code pfn {:#x}, page {:#010x}, word {}/{})", /// nanotlb hit path above and a single PFN comparison here. #[cfg(feature = "jitv2")] #[inline(always)] - fn jitv2_track_pcp(&mut self, phys_addr: u64) { + fn jitv2_track_pcp(&mut self, phys_addr: u64, uncached: bool) { // Code above the JIT's pfn map gets no page and runs in the - // interpreter (`step_jit` checks for the null page). - let Some(pfn) = crate::cpu::jitv2::jitv2::pfn_of(phys_addr, self.jitv2_frames) else { + // interpreter (`step_jit` checks for the null page). That includes + // uncached attributes 1 to 3: their bits 58:57 sit in `phys_addr`, + // so the frame range check rejects them. Where the uncached bus view + // differs from the cached one, an uncached fetch keys a page of its + // own (`code_key`). + let Some(pfn) = crate::cpu::jitv2::jitv2::code_key(phys_addr, self.jitv2_frames, uncached && self.jitv2_uncached_views) else { self.clear_pcp(); return; }; @@ -3158,7 +3214,7 @@ va={:#018x} phys={:#010x} (code pfn {:#x}, page {:#010x}, word {}/{})", // on the page-crossing path (not every fetch) to keep the common // same-page case free of this cost even when the toggle is on. if jitv2_pagewb_enabled() { - let page_base = pfn as u64 * crate::cpu::jitv2::PAGE_SIZE as u64; + let page_base = crate::cpu::jitv2::jitv2::key_phys_base(pfn); self.cache.writeback(None, page_base, crate::cpu::jitv2::PAGE_SIZE as u64); } // A page crossing landing exactly on word 0 (the exception/TLB-refill @@ -3194,7 +3250,7 @@ va={:#018x} phys={:#010x} (code pfn {:#x}, page {:#010x}, word {}/{})", return; } - let page_base = pfn as u64 * crate::cpu::jitv2::PAGE_SIZE as u64; + let page_base = crate::cpu::jitv2::jitv2::key_phys_base(pfn); // §13: live FR mode at first-arrival time, pinned into the page for // its whole lifetime (PhysicalCodePage::fr1's own doc comment) — // ignored by page_for on a lookup hit (an already-claimed page keeps @@ -3364,7 +3420,7 @@ va={:#018x} phys={:#010x} (code pfn {:#x}, page {:#010x}, word {}/{})", /// cost to the interpreter's per-instruction hot loop. pub fn exec(&mut self, instr: u32) -> ExecStatus { #[cfg(feature = "jitv2")] - self.jitv2_track_pcp(self.core.pc & 0xFFFF_FFFF); + self.jitv2_track_pcp(self.core.pc & 0xFFFF_FFFF, false); self.ins.raw = instr; self.ins.flags = FLAG_NOT_DECODED; decode_into::(&mut self.ins); @@ -9007,10 +9063,11 @@ va={:#018x} phys={:#010x} (code pfn {:#x}, page {:#010x}, word {}/{})", let over_threshold = self.jitv2.lock().codegen.lock().as_ref() .is_some_and(|c| c.packing_stats().1 > crate::cpu::jitv2::CODEGEN_ARENA_FLUSH_THRESHOLD_BYTES); if over_threshold { - let phys_page_base = page.pfn as u64 * crate::cpu::jitv2::PAGE_SIZE as u64; - unsafe { self.jitv2.lock().flush_from_cpu_thread(self.sysad.clone()); } + let phys_page_base = page.phys_base(); + let uncached = page.is_uncached_view(); + unsafe { self.jitv2.lock().flush_from_cpu_thread(self.jitv2_compile_bus()); } self.clear_pcp(); - self.jitv2_track_pcp(phys_page_base); + self.jitv2_track_pcp(phys_page_base, uncached); return EXEC_RETRY; } let mut codegen = self.jitv2.lock().codegen.lock().take(); @@ -9019,6 +9076,7 @@ va={:#018x} phys={:#010x} (code pfn {:#x}, page {:#010x}, word {}/{})", "jitv2_lockstep: inline compile found no Codegen (async compile queue stole it?) — \ verification would silently be OFF. The queue must not run under lockstep."); let mut ran_out_of_memory = false; + let compile_bus = self.jitv2_compile_bus(); if let Some(codegen) = codegen.as_mut() { // Stamp the L1-D geometry the inline load/store fast path needs. // Done on every take rather than once at startup so a cache @@ -9030,10 +9088,10 @@ va={:#018x} phys={:#010x} (code pfn {:#x}, page {:#010x}, word {}/{})", #[cfg(feature = "developer")] { let stats = self.jitv2.lock().stats.clone(); - ran_out_of_memory = crate::cpu::jitv2::comp::handle_request(&req, &self.sysad, &mut self.jitv2_inline_analyzer, codegen, &stats); + ran_out_of_memory = crate::cpu::jitv2::comp::handle_request(&req, &compile_bus, &mut self.jitv2_inline_analyzer, codegen, &stats); } #[cfg(not(feature = "developer"))] - { ran_out_of_memory = crate::cpu::jitv2::comp::handle_request(&req, &self.sysad, &mut self.jitv2_inline_analyzer, codegen); } + { ran_out_of_memory = crate::cpu::jitv2::comp::handle_request(&req, &compile_bus, &mut self.jitv2_inline_analyzer, codegen); } } *self.jitv2.lock().codegen.lock() = codegen; { @@ -9043,10 +9101,11 @@ va={:#018x} phys={:#010x} (code pfn {:#x}, page {:#010x}, word {}/{})", } if ran_out_of_memory { let page = unsafe { &mut *self.pcp }; - let phys_page_base = page.pfn as u64 * crate::cpu::jitv2::PAGE_SIZE as u64; - unsafe { self.jitv2.lock().flush_from_cpu_thread(self.sysad.clone()); } + let phys_page_base = page.phys_base(); + let uncached = page.is_uncached_view(); + unsafe { self.jitv2.lock().flush_from_cpu_thread(self.jitv2_compile_bus()); } self.clear_pcp(); - self.jitv2_track_pcp(phys_page_base); + self.jitv2_track_pcp(phys_page_base, uncached); return EXEC_RETRY; } // Re-check is_runnable: handle_request can decline this exact diff --git a/src/cpu/mips_exec/monitor.rs b/src/cpu/mips_exec/monitor.rs index 1e64255d8..c066e5919 100644 --- a/src/cpu/mips_exec/monitor.rs +++ b/src/cpu/mips_exec/monitor.rs @@ -2027,7 +2027,7 @@ impl MipsCpu { #[cfg(not(feature = "jitv2_lockstep"))] { exec.jitv2_inline_compile = false; - let bus = exec.sysad.clone(); + let bus = exec.jitv2_compile_bus(); let old = exec.jitv2.lock().codegen.lock().take(); match old { Some(old) => { @@ -2555,11 +2555,11 @@ impl MipsCpu { if result.is_exception() { return Err(format!("{:#018x} doesn't translate", pc)); } - let Some(pfn) = exec.jitv2_frame(result.phys) else { + let Some(pfn) = exec.jitv2_code_key(result.phys, !result.is_cached()) else { return Err(format!("phys {:#x} is above the JIT's physical range", result.phys)); }; - let page_base = pfn as u64 * crate::cpu::jitv2::PAGE_SIZE as u64; - let sysad = exec.sysad.clone(); + let page_base = crate::cpu::jitv2::jitv2::key_phys_base(pfn); + let sysad = exec.jitv2_compile_bus(); let mut jit = exec.jitv2.lock(); let fr1 = (exec.core.cp0_status & crate::cpu::mips_core::STATUS_FR) != 0; let slot = jit.page_for(pfn, page_base, sysad.as_ref(), fr1); @@ -2569,7 +2569,7 @@ impl MipsCpu { } }; let page = unsafe { &*page_ptr }; - let sysad = exec.sysad.clone(); + let sysad = exec.jitv2_compile_bus(); let dump = crate::cpu::jitv2::pcp_dump::PcpDump::capture(page, sysad.as_ref()) .map_err(|st| format!("capture failed: bus status {:#x}", st))?; // A path argument is whichever arg is not the address. @@ -2603,7 +2603,7 @@ impl MipsCpu { // the bus, and holding the jitv2 lock across a few // thousand of those plus the filesystem would stall // every dispatching CPU thread for the whole dump. - let sysad = exec.sysad.clone(); + let sysad = exec.jitv2_compile_bus(); let mut unreadable = 0usize; let dumps: Vec = { let jit = exec.jitv2.lock(); @@ -2697,12 +2697,12 @@ impl MipsCpu { writeln!(writer, "No PhysicalCodePage tracked (self.pcp is null) and PC {:#018x} doesn't translate.", pc).unwrap(); return Ok(()); } - let Some(pfn) = exec.jitv2_frame(result.phys) else { + let Some(pfn) = exec.jitv2_code_key(result.phys, !result.is_cached()) else { writeln!(writer, "No PhysicalCodePage tracked (self.pcp is null) and PC {:#018x} is at phys {:#x}, above the JIT's physical range.", pc, result.phys).unwrap(); return Ok(()); }; - let page_base = pfn as u64 * crate::cpu::jitv2::PAGE_SIZE as u64; - let sysad = exec.sysad.clone(); + let page_base = crate::cpu::jitv2::jitv2::key_phys_base(pfn); + let sysad = exec.jitv2_compile_bus(); let fr1 = (exec.core.cp0_status & crate::cpu::mips_core::STATUS_FR) != 0; let mut jit = exec.jitv2.lock(); match jit.page_for(pfn, page_base, sysad.as_ref(), fr1) { @@ -2857,14 +2857,18 @@ impl MipsCpu { if exec.core.smp { return Err("j2 clear: another CPU may be running this page; use j2 flush".to_string()); } - if actual_args.len() < 2 { return Err("Usage: j2 clear ".to_string()); } + if actual_args.len() < 2 { return Err("Usage: j2 clear [uncached]".to_string()); } let symbols_arc = exec.symbols.clone(); let symbols = symbols_arc.lock(); let paddr = monitor_phys(parse_cpu_arg(actual_args[1], &exec.core, Some(&symbols))?); drop(symbols); - let pfn = exec.jitv2_monitor_pfn(paddr)?; - let page_base = pfn as u64 * crate::cpu::jitv2::PAGE_SIZE as u64; - let sysad = exec.sysad.clone(); + // A trailing `uncached` names the page the CPU fetched + // through the uncached view (machines that keep one). + let uncached = actual_args.get(2) == Some(&"uncached") && exec.jitv2_uncached_views; + let frame = exec.jitv2_monitor_pfn(paddr)?; + let pfn = if uncached { frame | crate::cpu::jitv2::jitv2::UNCACHED_VIEW_PFN } else { frame }; + let page_base = crate::cpu::jitv2::jitv2::key_phys_base(pfn); + let sysad = exec.jitv2_compile_bus(); let fr1 = (exec.core.cp0_status & crate::cpu::mips_core::STATUS_FR) != 0; let mut jit = exec.jitv2.lock(); match jit.page_for(pfn, page_base, sysad.as_ref(), fr1) { @@ -2904,15 +2908,19 @@ impl MipsCpu { // still stops the specific `func` this call saw, but // a fresh compile can re-publish the same offset // right after. - if actual_args.len() < 2 { return Err("Usage: j2 deny ".to_string()); } + if actual_args.len() < 2 { return Err("Usage: j2 deny [uncached]".to_string()); } let symbols_arc = exec.symbols.clone(); let symbols = symbols_arc.lock(); let paddr = monitor_phys(parse_cpu_arg(actual_args[1], &exec.core, Some(&symbols))?); drop(symbols); - let pfn = exec.jitv2_monitor_pfn(paddr)?; - let page_base = pfn as u64 * crate::cpu::jitv2::PAGE_SIZE as u64; + // A trailing `uncached` names the page the CPU fetched + // through the uncached view (machines that keep one). + let uncached = actual_args.get(2) == Some(&"uncached") && exec.jitv2_uncached_views; + let frame = exec.jitv2_monitor_pfn(paddr)?; + let pfn = if uncached { frame | crate::cpu::jitv2::jitv2::UNCACHED_VIEW_PFN } else { frame }; + let page_base = crate::cpu::jitv2::jitv2::key_phys_base(pfn); let offset = (paddr & 0xFFF) as usize >> 2; - let sysad = exec.sysad.clone(); + let sysad = exec.jitv2_compile_bus(); let fr1 = (exec.core.cp0_status & crate::cpu::mips_core::STATUS_FR) != 0; let mut jit = exec.jitv2.lock(); match jit.page_for(pfn, page_base, sysad.as_ref(), fr1) { @@ -2976,7 +2984,7 @@ impl MipsCpu { // signal, not a crash risk. let page = unsafe { &*page_ptr }; writeln!(writer, " page: pfn={:#010x} phys_base={:#010x}", - page.pfn, page.pfn as u64 * crate::cpu::jitv2::PAGE_SIZE as u64).unwrap(); + page.pfn, page.phys_base()).unwrap(); } }; match jit.compile_queue.seal_queue_entries() { @@ -3012,7 +3020,7 @@ impl MipsCpu { // implementation. let path = actual_args.get(1).copied().unwrap_or("jitv2.html"); let jitv2_arc = exec.jitv2.clone(); - let bus = exec.sysad.clone(); + let bus = exec.jitv2_compile_bus(); crate::cpu::jitv2_html_j2wp::write_jitv2_html( &jitv2_arc, &bus, &exec.memory.clone(), &mut exec.jitv2_inline_analyzer, path, &mut *writer, )?; diff --git a/src/ip35.rs b/src/ip35.rs index ff2af349f..5c70104b9 100644 --- a/src/ip35.rs +++ b/src/ip35.rs @@ -17,6 +17,8 @@ use crate::dev::bedrock_mem::{BANK_SPAN, FITTED_BANK}; use crate::physical::Physical; use crate::traits::*; #[cfg(feature = "jitv2")] +use crate::cpu::jitv2::jitv2::UNCACHED_VIEW_ADDR; +#[cfg(feature = "jitv2")] use std::sync::{atomic::AtomicU64, OnceLock}; use std::sync::Arc; @@ -237,6 +239,14 @@ impl Ip35Bus { let pa = plain_address(a, n)?; Some((&self.views[(a >> 57) as usize], pa)) } + /// The bus the JIT compiles code through: a page the CPU fetched cached + /// reads the cached view, one it fetched uncached reads the uncached view + /// the CPU's own fetch used (`UNCACHED_VIEW_ADDR`). Bedrock decodes the + /// two differently at the reset PROM once memory covers it. + #[cfg(feature = "jitv2")] + pub fn code_view(self: &Arc, cached: Arc) -> Arc { + Arc::new(CodeViews { cached, uncached: self.clone() }) + } pub fn cached_view(&self, plain_memory: Arc) -> Arc { Arc::new(CachedPhysical { phys: self.phys.clone(), @@ -251,6 +261,25 @@ fn plain_address(a: u64, n: u64) -> Option { let pa = a & PA_MASK; (pa.checked_add(n)? <= PA_MASK + 1).then_some(pa) } +/// See [`Ip35Bus::code_view`]. Compiles read instruction words only. +#[cfg(feature = "jitv2")] +struct CodeViews { + cached: Arc, + uncached: Arc, +} +#[cfg(feature = "jitv2")] +impl BusDevice for CodeViews { + fn read32(&self, a: u64) -> BusRead32 { + if a & UNCACHED_VIEW_ADDR != 0 { + self.uncached.read32(a & !UNCACHED_VIEW_ADDR) + } else { + self.cached.read32(a) + } + } + fn gen_ptr(&self, a: u64) -> *const AtomicU64 { + if a & UNCACHED_VIEW_ADDR != 0 { self.uncached.gen_ptr(a) } else { self.cached.gen_ptr(a) } + } +} struct CachedPhysical { phys: Arc, plain_memory: Arc, @@ -295,6 +324,17 @@ macro_rules! processor_access { impl BusDevice for Ip35Bus { #[cfg(feature = "jitv2")] fn gen_ptr(&self, address: u64) -> *const AtomicU64 { + if address & UNCACHED_VIEW_ADDR != 0 { + // A page fetched uncached: its bytes come from the uncached view + // at that address (attribute 0, the reset PROM and node memory), + // so its generation is that device's. Writes through the cached + // view change other bytes where the views differ, and bump the + // same RAM counter where they coincide. + // The PROM view's counter covers the whole flash, so any flash + // write invalidates every PROM page (over-invalidation, safe). + let pa = address & !UNCACHED_VIEW_ADDR; + return if pa & !PA_MASK == 0 { self.views[0].gen_ptr(pa) } else { std::ptr::null() }; + } if address & !PA_MASK == 0 { // JIT page lookup passes plain physical code addresses, without // a cache attribute. Raw UC0 therefore cannot select the MMIO map diff --git a/src/ip35/tests.rs b/src/ip35/tests.rs index b7cd7bb51..26583801b 100644 --- a/src/ip35/tests.rs +++ b/src/ip35/tests.rs @@ -621,3 +621,243 @@ fn the_fuel_plugs_a_usb_keyboard_and_mouse_only_when_configured_on() { let (_, status) = ports(false); assert_eq!(status.map(|s| s & 1), [0, 0], "nothing plugged in"); } + +/// Where the cached and uncached views of a frame differ, the JIT keys and +/// compiles the page by the view the CPU fetched it through. The reset PROM +/// fetches uncached at the same plain address that cached RAM answers once +/// memory covers its window; the compile pool used to read the cached view and +/// compile the RAM's bytes in place of the PROM's. +#[cfg(feature = "jitv2")] +#[test] +fn uncached_fetch_compiles_the_uncached_view_of_the_page() { + // The executor and the code generator are large; the default test stack is not. + std::thread::Builder::new() + .stack_size(64 << 20) + .spawn(uncached_fetch_compiles_the_uncached_view_body) + .unwrap() + .join() + .expect("thread panicked"); +} + +#[cfg(feature = "jitv2")] +fn uncached_fetch_compiles_the_uncached_view_body() { + use crate::cpu::jitv2::jitv2::{CompileRequest, UNCACHED_VIEW_PFN}; + use crate::cpu::mips_isa::{JIT_REGION_BOUNDARY_SENTINEL, OP_ADDIU}; + + /// Every word the same; a region-boundary word compiles to nothing. + struct Words { + word: u32, + generation: AtomicU64, + } + impl Words { + fn new(word: u32) -> Arc { + Arc::new(Self { word, generation: AtomicU64::new(0) }) + } + } + impl BusDevice for Words { + fn read32(&self, _: u64) -> BusRead32 { + BusRead32::ok(self.word) + } + fn read64(&self, _: u64) -> BusRead64 { + BusRead64::ok((self.word as u64) << 32 | self.word as u64) + } + fn gen_ptr(&self, _: u64) -> *const AtomicU64 { + &self.generation + } + } + + let reset_vector = 0x1fc0_0000u64; + let addiu_r1 = (OP_ADDIU << 26) | (1 << 16) | 1; + let rom = Words::new(addiu_r1); + let ram = Words::new(JIT_REGION_BOUNDARY_SENTINEL); + let physical = Physical::new(Vec::new()); + let mut bus = Ip35Bus::new(physical.clone()); + map_device(&physical, reset_vector, 0x1000, ram.clone(), "RAM over the reset window"); + bus.set_ram_generations(ram.clone()); + bus.map(0, reset_vector, 0x1000, rom.clone()); + let bus = Arc::new(bus); + let code_bus = bus.code_view(bus.cached_view(ram.clone())); + // The two views really disagree here. + assert_eq!(bus.read32(reset_vector).data, addiu_r1); + assert_eq!(bus.cached_view(ram.clone()).read32(reset_vector).data, JIT_REGION_BOUNDARY_SENTINEL); + + let mut cpu = crate::ip35_machine::build_cpu(bus.clone()); + cpu.enable_jitv2_uncached_views(code_bus.clone()); + // The reset vector, KSEG1: uncached, plain physical address 0x1fc00000. + cpu.core.pc = 0xffff_ffff_bfc0_0000; + let _ = cpu.step_int(); + let page = cpu.jitv2_current_page(); + assert!(!page.is_null(), "the uncached fetch must land on a code page"); + let page = unsafe { &mut *page }; + assert_eq!(page.pfn, 0x1fc00 | UNCACHED_VIEW_PFN, "an uncached fetch keys the uncached view"); + assert!(page.is_uncached_view()); + assert_eq!(page.phys_base(), 0x1fc0_0000); + + page.mark_requested(0); + let req = CompileRequest { page: page as *mut _, compiled_for_fr1: true }; + let mut analyzer = crate::cpu::jitv2::analyzer::Analyzer::new(); + let mut codegen = crate::cpu::jitv2::codegen::Codegen::new(); + #[cfg(feature = "developer")] + crate::cpu::jitv2::comp::handle_request(&req, &cpu.jitv2_compile_bus(), &mut analyzer, &mut codegen, &crate::cpu::jitv2::JitStats::default()); + #[cfg(not(feature = "developer"))] + crate::cpu::jitv2::comp::handle_request(&req, &cpu.jitv2_compile_bus(), &mut analyzer, &mut codegen); + assert!(page.is_runnable(0), "the page must compile the PROM's ADDIU, not the RAM's boundary word"); + + // The cached view of the same frame is another page, with RAM's counter and bytes. + let jitv2 = cpu.jitv2.clone(); + let mut jit = jitv2.lock(); + let slot = jit.page_for(0x1fc00, 0x1fc0_0000, bus.as_ref(), false).unwrap(); + let cached = unsafe { &mut *jit.page_ptr(slot) }; + assert!(!cached.is_uncached_view()); + assert_ne!(cached as *mut _, page as *mut _); + cached.mark_requested(0); + let req = CompileRequest { page: cached as *mut _, compiled_for_fr1: true }; + #[cfg(feature = "developer")] + crate::cpu::jitv2::comp::handle_request(&req, &code_bus, &mut analyzer, &mut codegen, &crate::cpu::jitv2::JitStats::default()); + #[cfg(not(feature = "developer"))] + crate::cpu::jitv2::comp::handle_request(&req, &code_bus, &mut analyzer, &mut codegen); + assert!(!cached.is_runnable(0), "the cached page compiles RAM's boundary word to nothing"); + + // A write through one view invalidates only the pages that view's bytes feed. + ram.generation.fetch_add(1, std::sync::atomic::Ordering::Release); + assert!(page.is_runnable(0), "a cached-view write does not change the PROM's bytes"); + rom.generation.fetch_add(1, std::sync::atomic::Ordering::Release); + assert!(!page.is_runnable(0), "a write to the PROM's bytes invalidates its code"); +} + +#[cfg(feature = "jitv2")] +mod code_views { + use super::*; + use crate::cpu::jitv2::jitv2::{CompileRequest, UNCACHED_VIEW_PFN}; + use crate::cpu::mips_isa::{JIT_REGION_BOUNDARY_SENTINEL, OP_ADDIU}; + + struct Words(u32, AtomicU64); + impl BusDevice for Words { + fn read32(&self, _: u64) -> BusRead32 { + BusRead32::ok(self.0) + } + fn read64(&self, _: u64) -> BusRead64 { + BusRead64::ok((self.0 as u64) << 32 | self.0 as u64) + } + fn gen_ptr(&self, _: u64) -> *const AtomicU64 { + &self.1 + } + } + + const RESET: u64 = 0x1fc0_0000; + const ADDIU_R1: u32 = (OP_ADDIU << 26) | (1 << 16) | 1; + + /// A bus whose uncached attribute-0 view of the reset frame holds an ADDIU + /// and whose cached view holds a region boundary, as RAM over the PROM does. + fn bus_and_cpu(opted_in: bool) -> (Arc, Box) { + let rom = Arc::new(Words(ADDIU_R1, AtomicU64::new(0))); + let ram = Arc::new(Words(JIT_REGION_BOUNDARY_SENTINEL, AtomicU64::new(0))); + let physical = Physical::new(Vec::new()); + let mut bus = Ip35Bus::new(physical.clone()); + map_device(&physical, RESET, 0x1000, ram.clone(), "RAM over the reset window"); + bus.set_ram_generations(ram.clone()); + bus.map(0, RESET, 0x1000, rom.clone()); + // Code that is fetchable in the other views and above 4 GiB, so only + // the JIT's own range check keeps it out of a code page. + for attribute in 1..4 { + bus.map(attribute, RESET, 0x1000, rom.clone()); + } + bus.map(0, 0x1_0000_0000, 0x1000, rom); + let bus = Arc::new(bus); + let mut cpu = crate::ip35_machine::build_cpu(bus.clone()); + if opted_in { + let cached = bus.cached_view(ram); + cpu.enable_jitv2_uncached_views(bus.code_view(cached)); + } + (bus, cpu) + } + + /// The executor and the code generator are large; the default test stack is not. + fn big_stack(body: fn()) { + std::thread::Builder::new().stack_size(64 << 20).spawn(body).unwrap().join().expect("thread panicked"); + } + + fn wait_runnable(page: &crate::cpu::jitv2::PhysicalCodePage) -> bool { + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(20); + while std::time::Instant::now() < deadline { + if page.is_runnable(0) { return true; } + std::thread::sleep(std::time::Duration::from_millis(5)); + } + false + } + + /// After a page-pool flush the restarted compile pool must still read + /// uncached pages through the compile bus. Restarted on the CPU's own bus it + /// rejects the view address and the page never compiles. + #[test] + fn a_pool_restart_after_a_flush_keeps_the_compile_bus() { + big_stack(|| { + let (_bus, mut cpu) = bus_and_cpu(true); + { + let jit = cpu.jitv2.clone(); + let mut g = jit.lock(); + let stats = g.stats.clone(); + g.compile_queue.set_thread_count(1); + g.compile_queue.start(cpu.jitv2_compile_bus(), stats); + } + cpu.jitv2_flush_from_here(false); + cpu.core.pc = 0xffff_ffff_bfc0_0000; + let _ = cpu.step_int(); + let page = unsafe { &mut *cpu.jitv2_current_page() }; + assert!(page.is_uncached_view()); + page.mark_requested(0); + let jit = cpu.jitv2.clone(); + let g = jit.lock(); + let stats = g.stats.clone(); + assert!(g.compile_queue.send(CompileRequest { page: page as *mut _, compiled_for_fr1: true }, &stats)); + drop(g); + assert!(wait_runnable(page), "the restarted pool must compile the uncached view"); + jit.lock().compile_queue.stop(); + }); + } + + /// A machine that has not opted in keeps one page per frame and the + /// original-size frame map, whatever the cache attribute of the fetch. + #[test] + fn an_executor_that_has_not_opted_in_keeps_the_plain_key() { + big_stack(|| { + let (_bus, mut cpu) = bus_and_cpu(false); + assert!(!cpu.jitv2.lock().uncached_views_enabled()); + cpu.core.pc = 0xffff_ffff_bfc0_0000; + let _ = cpu.step_int(); + let page = unsafe { &*cpu.jitv2_current_page() }; + assert_eq!(page.pfn, 0x1fc00); + assert_eq!(page.pfn & UNCACHED_VIEW_PFN, 0); + assert!(!page.is_uncached_view()); + }); + } + + /// Uncached attributes 1 to 3 and anything over 4 GiB stay in the interpreter. + #[test] + fn attributes_one_to_three_and_high_addresses_get_no_code_page() { + big_stack(|| { + for pc in [ + 0x9000_0000_0000_0000u64 | RESET, // attribute 0: a page + 0x9200_0000_0000_0000u64 | RESET, // attribute 1 + 0x9400_0000_0000_0000u64 | RESET, // attribute 2 + 0x9600_0000_0000_0000u64 | RESET, // attribute 3 + 0x9000_0001_0000_0000u64, // 4 GiB, attribute 0 + ] { + // A fresh executor each time: a tracked page would survive a + // fetch that wrongly kept the same frame. + let (_bus, mut cpu) = bus_and_cpu(true); + // KX, so XKPHYS addresses translate. + let status = cpu.core.cp0_status | 0x80; + cpu.set_cp0_status(status); + cpu.core.pc = pc; + let _ = cpu.step_int(); + let page = cpu.jitv2_current_page(); + if pc == 0x9000_0000_0000_0000u64 | RESET { + assert!(!page.is_null(), "attribute 0 below 4 GiB is compiled"); + } else { + assert!(page.is_null(), "{pc:#x} must not get a code page"); + } + } + }); + } +} diff --git a/src/ip35_machine.rs b/src/ip35_machine.rs index 255bcc563..0df55974f 100644 --- a/src/ip35_machine.rs +++ b/src/ip35_machine.rs @@ -137,8 +137,11 @@ impl FuelMachine { // After the windows above: each compile worker takes the cache's // geometry and the CPU model's policies once, as it starts, and // without the windows the geometry has no inline load or store path. - // Workers read plain physical code addresses, which the cached plane - // serves; the uncached views would decode them as Hub registers. + // Workers read plain physical code addresses through the view the CPU + // fetched each page by: the cached plane for cached pages, the + // uncached attribute-0 view for the reset PROM's uncached fetches. + // Decoding plain addresses in the uncached views for cached pages + // would reach Hub registers. #[cfg(feature = "jitv2")] { let cpu_device: Arc = cpu.clone().as_device(); @@ -153,7 +156,7 @@ impl FuelMachine { #[cfg(not(feature = "jitv2_lockstep"))] { let stats = g.stats.clone(); - g.compile_queue.start(board.cached_bus(), stats); + g.compile_queue.start(board.code_bus(), stats); } // The compile pool keeps only a weak handle on the CPU. std::mem::forget(cpu_device); @@ -882,9 +885,19 @@ impl FuelBoard { self.bus.cached_view(self.md.dma_view()) } + /// The bus the JIT compiles code through; see [`Ip35Bus::code_view`]. + #[cfg(feature = "jitv2")] + pub fn code_bus(&self) -> Arc { + self.bus.code_view(self.cached_bus()) + } + pub fn build_cpu(&self) -> Box { let mut cpu = build_cpu(self.bus.clone()); cpu.cache = R14000FuelShadowCache::new(self.cached_bus()); + // The PROM fetches its own code uncached; with memory over its + // window the cached and uncached views of it differ. + #[cfg(feature = "jitv2")] + cpu.enable_jitv2_uncached_views(self.code_bus()); // The PROM keeps its stack in cache lines it tags for a window with no // memory behind it yet; the controller answers that window while they stand. let memory = self.md.clone(); From 51b199b30d75e3eeac605d4952edd6f748f9bfde Mon Sep 17 00:00:00 2001 From: James Braid Date: Fri, 30 Oct 2026 04:22:12 -0700 Subject: [PATCH 29/31] odyssey: let a widget-offset machine host the board, with credit accounting The board assumed the Octane: its interrupt packets went only to HEART at widget 8, and it took the processor's addresses in the IP30 widget windows. A Fuel's processor reaches widget 13 through Bedrock, which hands the board widget offsets, and the board's packets cross the XBridge's crossbar to Bedrock like any widget's. Add a constructor for such a machine: it takes a packet sender for the interrupts and decodes every address as an aperture offset. The Octane's constructor and behaviour are unchanged. A test routes a packet to a widget other than HEART and requires the sender to get it, not HEART's line. Graphics credits (odyssey): A Bedrock or Hub processor interface counts graphics credits: each counted store to a graphics head takes one, and the head returns it as the store leaves its command FIFO. The Octane's HEART has no such block, so the board left the accounting out. Bring it back for a machine that links a processor interface: the PiGfx register block and the PiGfxLink trait (return, owe and repay credits), a counted and an owed mark on each queued store, the credit loop in retire and clear, the one-store runs while a counted store waits, and the processor's one-shot store mark. With no link, as on the Octane, every credit call is a no-op. How long a V12 takes to return a credit is not published. The host's credit window is 100 doublewords (IP35) or 256 (IP30), a credit is inferred to come back when the board takes the doubleword into its FIFO, and the FIFO must hold more than the window. The model credits a store as it enters while fewer than 16,384 stores wait (cfifo::DEPTH). The initialise call stores its start-up program, about 12,291 words, with no level check before the low-water interrupt is routed; with a depth of 512 two boots in five timed out with "CFIFO timeout" and no screens, and 16,384 is the smallest power of two that covers the program even at one word per store. The depth is a model choice derived from that program. A deeper store owes its credit and hurries the board thread; a reset refunds what the abandoned stores owed. Inside the depth the board thread takes full runs, so a credited backlog does not drain one store at a time. The board thread reads the count of counted stores once per run taken and once per run retired, on every machine. Both reads are Relaxed: a run's length is only a hint, a stale count costs a longer run and never a lost credit, and the retire reads it under the lock a counted push takes to change it. Nothing else on the Octane's store path changed. The tests drive the credit rules with a stand-in link: a counted store is credited on entry, a reset cancels owed credits, the mark counts exactly one store, a widget that runs out of credits starves, the FIFO set-up words do not widen the credit depth, and an owed credit hurries the board thread. Others fill the largest host window plus one with the board taking nothing and require every credit back with none owed, and post 16,384 counted stores and require each credited at entry and the next one owed. The refused-push test used to sleep 150 ms and require a push thread to be unfinished, which a loaded host could break once the board thread's timed hold ran out first. The hold now lasts until the test releases it, and the test waits for the push to be refused before it asserts. --- clanker/odyssey/gotchas.md | 3 +- clanker/odyssey/threads-and-locks.md | 23 +- src/dev/odyssey/board.rs | 1 + src/dev/odyssey/consts.rs | 27 +- src/dev/odyssey/fifo.rs | 192 ++++++++++++-- src/dev/odyssey/interface.rs | 49 +++- src/dev/odyssey/mmio.rs | 45 +++- src/dev/odyssey/mod.rs | 76 ++++-- src/dev/odyssey/pi_gfx.rs | 210 ++++++++++++++++ src/dev/odyssey/tests/fifo.rs | 359 ++++++++++++++++++++++++--- src/dev/odyssey/tests/interrupts.rs | 23 ++ src/dev/odyssey/worker.rs | 11 +- 12 files changed, 918 insertions(+), 101 deletions(-) create mode 100644 src/dev/odyssey/pi_gfx.rs diff --git a/clanker/odyssey/gotchas.md b/clanker/odyssey/gotchas.md index 826d3672f..7d7db173b 100644 --- a/clanker/odyssey/gotchas.md +++ b/clanker/odyssey/gotchas.md @@ -40,7 +40,8 @@ Status: rule (2026-10-30). timing measured to better than that is noise. - **Model choices to remember.** The FIFO's depth is unknown in any unit. - The model clamps the level at 32,768 words, whatever the stored `0x6bX` + The model keeps 16,384 stores for + credits and clamps the level at 32,768 words, whatever the stored `0x6bX` values say, and uses a 16-word low-water threshold; `0x6b2` as the status-bit-15 threshold is a model choice too. The level is in words, a doubleword store counting two. The tests pin the model, not the diff --git a/clanker/odyssey/threads-and-locks.md b/clanker/odyssey/threads-and-locks.md index 0e2fc8c08..cd9485d26 100644 --- a/clanker/odyssey/threads-and-locks.md +++ b/clanker/odyssey/threads-and-locks.md @@ -37,7 +37,7 @@ apply to BUZZ). Nothing models more than one raster unit. - **Command FIFO** (`fifo.rs`, CPU to `odyssey-work`): the processor's stores to the two command-FIFO ports and the context-switch stores, each - as (offset, size, value). The level the guest sees + as (offset, size, value) with its credit state. The level the guest sees is in words, against the model's 32,768. - **Raster queue** (`raster_unit.rs`, `odyssey-work` to `odyssey-raster`): jobs, each owning what its raster work reads, captured when the front @@ -116,8 +116,8 @@ other, as MGRAS's producers take its `submit` lock (below). drawing. A processor access to a host-interface register does not take it. - **`HostInterface`** is behind the front lock, a register file's own lock held for a few instructions: identity, control, the interrupt bank, - the routing, the swap bank and the FIFO status, plus the interrupt state. - A front access (those registers) takes only it + the routing, the swap bank and the FIFO status, plus the interrupt state + and the credit return. A front access (those registers) takes only it and never waits for a draw. It is a leaf; the raster thread takes it for a fence's interrupt, and the display takes it to latch swaps. - **The raster unit's state** (`RasterCtx`: the stipple phase and the rows @@ -197,15 +197,15 @@ the room check and the push, is refused, and its store retries the push behind t (`BoardLocks::write_command`), not around it. The ring is deeper than the board's FIFO, so a second producer's overshoot of the level by a store is absorbed. The consumer (the worker, or a draining access with the worker stopped) looks at up to 32 stores, runs them, and takes the ones it ran out together -(`CommandFifo::take_run`, `retire`): the level and the store count -move once a run, not once a store, because each store costs the +(`CommandFifo::take_run`, `retire`): the level, the store count and the +credits move once a run, not once a store, because each store costs the worker little and the processor reads those words. A store that has run but not retired still counts in the level, so the level never reads below what the -board has left to take. A push counts its store (level, store count) -before the ring shows it and takes the counts back off if the ring +board has left to take. A push counts its store (level, store count, counted +stores) before the ring shows it and takes the counts back off if the ring refuses it: the worker can take a store and retire it the instant it lands, before the push returns, and a retire must find every store it takes already -counted or the level goes below zero. +counted or the level goes below zero. While a counted store waits a run is one store. While the worker runs every command store queues, so decoding and drawing never run on the processor's thread. @@ -225,6 +225,13 @@ A caller other than the bus, with the board's thread stopped, still runs the queue and then its store. The flow-control target word `+0x1074` is stored, not routed, so a store to it goes in at once even while the FIFO holds work. +Counted stores (machines whose processor interface counts graphics credits; +the Octane has none and the calls are no-ops) are credited as they enter the +FIFO's first 16,384 places (the model's depth, a model choice), counted in +stores: at the store when there is room, +or when the stores ahead have run, with the node holding the credit as owed +meanwhile. + ## The FIFO ordering rules These are guest-visible, and hold across both queues; any change to either diff --git a/src/dev/odyssey/board.rs b/src/dev/odyssey/board.rs index d0b72fe63..4314024e4 100644 --- a/src/dev/odyssey/board.rs +++ b/src/dev/odyssey/board.rs @@ -923,6 +923,7 @@ impl Board { // established, so it is zero. let identity = self.identity(); let mut f = self.front.lock(); + f.slot = self.slot; f.identity = identity; f.widget_control = self.slot & ODSY_CONTROL_WIDGET_MASK; f.xt_req_timeout = ODSY_XT_REQ_TIMEOUT_RESET; diff --git a/src/dev/odyssey/consts.rs b/src/dev/odyssey/consts.rs index 23d143938..96f2508f9 100644 --- a/src/dev/odyssey/consts.rs +++ b/src/dev/odyssey/consts.rs @@ -179,8 +179,9 @@ pub(super) mod reg { // the bit and raises the flag, and zero, as out of reset, is no threshold. // Neither the roles nor the link between bit 15, flag 25 and any 0x6bX value // is established. The level's unit (32-bit words, a doubleword store counting - // two) is a model choice, as is the FIFO's size (super::cfifo::WORDS), - // independent of the stored values, and so is the low-water threshold (super::cfifo::LOW_WATER). The model contract is pinned by + // two) is a model choice, as is the FIFO's size (super::cfifo::WORDS) and depth + // (super::cfifo::DEPTH), independent of the stored values, and so is the + // low-water threshold (super::cfifo::LOW_WATER). The model contract is pinned by // the_level_counts_words_against_the_model_fifo_and_the_6b2_threshold and // the_threshold_follows_6b2. pub const FIFO_SETUP_FIRST: u32 = 0x6b0; @@ -319,6 +320,28 @@ pub(super) mod cfifo { // deeper or shallower. Test, a model contract: // the_fifo_size_ignores_the_set_up_words. pub const WORDS: u32 = 32768; + + // The waiting stores whose credits have been returned, in stores (one + // credit per store). MODEL CHOICE: the FIFO's depth is UNKNOWN (see + // WORDS). It must exceed the host's credit window of 100 (IP35) or 256 + // (IP30) doublewords, and a credit is inferred to come back when the + // board takes the doubleword into its FIFO. Exceeding the window is not + // enough: the + // kernel's initialise call stores its start-up program, about 12,291 + // words, with no level check, before the low-water interrupt has a + // destination, and a credit owed during that load can leave the writer + // held until the 15-second FIFO timer. 16,384 is the smallest power of two + // that covers the whole program even at one word per store, so none of it + // waits on the board's thread for its credit. The figure is a model choice + // derived from the documented ~12,291-word start-up program, as evidence + // for a depth whose grade is unknown. Limitation: past this depth a credit + // waits for the board's thread again, the processor can run this many + // stores ahead of the board, and a writer that does is held until the + // backlog drains below the low-water mark. The stored 0x6bX values do not + // change it (fifo.rs). Tests: the_credit_depth_is_deeper_than_the_host_window, + // every_store_inside_the_credited_depth_is_credited_and_the_next_is_owed, + // the_fifo_set_up_does_not_widen_the_credit_depth (a model contract). + pub const DEPTH: u32 = 16384; } pub(super) const ODSY_FIFO_SETUP_7B5_INDEX: usize = 4; diff --git a/src/dev/odyssey/fifo.rs b/src/dev/odyssey/fifo.rs index 87b128b63..0813ac82c 100644 --- a/src/dev/odyssey/fifo.rs +++ b/src/dev/odyssey/fifo.rs @@ -40,11 +40,40 @@ //! //! The processor's stores are the producer; whoever runs the queue (the //! board's thread, or an access that drains it with that thread stopped) -//! is the consumer. +//! is the consumer. Graphics credits, which only machines whose processor +//! interface counts stores use, are settled as a store enters the FIFO's +//! first `cfifo::DEPTH` places: at the store when there is room, or +//! when the stores ahead of it have run. That depth (16,384 stores) is a +//! MODEL CHOICE, independent of the stored `0x6bX` values: the smallest +//! power of two that holds the kernel's whole start-up program (about +//! 12,291 words) even at one word per store, and far deeper than the +//! largest host credit window (256). A writer is then credited as its stores enter and does +//! not wait on the board's thread. That matters because IRIX's high-water +//! handler looks at the count for only about ten microseconds before it +//! holds the writer, and while the X server's initialise call is loading +//! that program, before the low-water interrupt has a destination, only +//! the kernel's 15-second FIFO timer wakes a writer held there. Limitation: a writer that runs more than the depth plus its +//! window ahead of the board waits for the board's thread again, and is +//! held until the level falls below the low-water mark. Whether a 32-bit +//! store costs the hardware a whole credit is UNKNOWN; the model credits +//! one per store. +//! +//! The deeper credited FIFO widens what the guest can see. A counted writer +//! can now leave up to the depth plus its window (about 16,640 stores) +//! waiting, where 64 plus the window was the bound before, so the level, +//! the unreturned-credit field of `0x106c` (which saturates), status bit 17 +//! (below the low-water mark) and flag 9 follow a backlog that long: bit 17 +//! stays clear, and flag 9 latches, only after it drains. Doubleword stores +//! that deep reach the `0x6b2` threshold the kernel stores, 32,192 words, +//! which the old bound never did, so status bit 15 and flag 25 can now +//! rise for counted stores too; the kernel leaves flag 25 masked and does +//! not read bit 15. The level the guest reads stops at the model's FIFO +//! size. use super::consts::{cfifo, intr, ODSY_WORK_QUEUE_MAX}; use super::interface::HostInterface; +use super::pi_gfx::{pi_gfx_owe_credits, pi_gfx_repay_credits}; use super::worker::GAP; use std::cell::Cell; use std::sync::atomic::{AtomicBool, AtomicU32, Ordering::{Relaxed, SeqCst}}; @@ -64,18 +93,28 @@ pub(super) struct QueuedStore { pub value: u64, pub addr: u32, pub size: u8, + /// The processor interface counted it: its credit is due. + pub counted: bool, + /// It waited outside the FIFO, so the node holds its credit as owed. + pub owed: bool, } // A ring entry is (address word, value). The address word carries the -// aperture offset in its low 24 bits and the store's size above them. +// aperture offset in its low 24 bits and the store's size and credit +// state above them. const ENTRY_OFFSET_MASK: u32 = 0x00ff_ffff; const ENTRY_SIZE_SHIFT: u32 = 24; const ENTRY_SIZE_MASK: u32 = 0xf; +const ENTRY_COUNTED: u32 = 1 << 28; +const ENTRY_OWED: u32 = 1 << 29; impl QueuedStore { fn entry(&self) -> u32 { debug_assert!(self.addr <= ENTRY_OFFSET_MASK && self.size as u32 <= ENTRY_SIZE_MASK); - self.addr | (self.size as u32) << ENTRY_SIZE_SHIFT + self.addr + | (self.size as u32) << ENTRY_SIZE_SHIFT + | if self.counted { ENTRY_COUNTED } else { 0 } + | if self.owed { ENTRY_OWED } else { 0 } } fn from_entry(addr: u32, value: u64) -> Self { @@ -83,6 +122,8 @@ impl QueuedStore { value, addr: addr & ENTRY_OFFSET_MASK, size: ((addr >> ENTRY_SIZE_SHIFT) & ENTRY_SIZE_MASK) as u8, + counted: addr & ENTRY_COUNTED != 0, + owed: addr & ENTRY_OWED != 0, } } @@ -108,7 +149,7 @@ pub(super) struct Run { impl Run { pub fn new() -> Self { - Run { stores: [QueuedStore { value: 0, addr: 0, size: 0 }; RUN_MAX], len: 0 } + Run { stores: [QueuedStore { value: 0, addr: 0, size: 0, counted: false, owed: false }; RUN_MAX], len: 0 } } /// The stores to run, oldest first. @@ -127,6 +168,8 @@ struct Levels { words: AtomicU32, /// Stores pushed and not yet taken out. stores: AtomicU32, + /// Counted stores waiting; while zero no store needs credit work. + counted: AtomicU32, } pub(super) struct CommandFifo { @@ -146,9 +189,9 @@ pub(super) struct CommandFifo { /// The board's thread is parked (`wait_for_work`): a producer unparks it /// only then, so a store reads a line nobody writes instead of locking. asleep: CachePadded, - /// The processor waits for the board's thread, for a refused access: its - /// next slice starts without the pause for a burst of stores, and a - /// pause under way ends. + /// The processor waits for the board's thread, for a refused access or + /// a counted store whose credit is owed: its next slice starts without + /// the pause for a burst of stores, and a pause under way ends. hurry: CachePadded, /// The board's thread is running stores. pub slice: CachePadded, @@ -168,7 +211,13 @@ pub(super) struct CommandFifo { /// counted after its last look at the hurry request. #[cfg(test)] pub test_parked: AtomicU32, - /// Tests: the next push calls this, once, as soon as its + /// Tests: pushes the full ring has refused. + #[cfg(test)] + pub test_refused: AtomicU32, + /// Tests: ends a long slice's hold (`Board::test_slice_sleep_ms`) early. + #[cfg(test)] + pub test_release: AtomicBool, + /// Tests: the next uncounted push calls this, once, as soon as its /// store is in the ring: a consumer that takes the store at that /// moment, before the push has returned. #[cfg(test)] @@ -182,7 +231,7 @@ impl Default for CommandFifo { // place because the ring is too big for the stack. ring: unsafe { Box::new_zeroed().assume_init() }, pop: Mutex::new(()), - levels: CachePadded::new(Levels { words: AtomicU32::new(0), stores: AtomicU32::new(0) }), + levels: CachePadded::new(Levels { words: AtomicU32::new(0), stores: AtomicU32::new(0), counted: AtomicU32::new(0) }), threshold_6b2: CachePadded::new(AtomicU32::new(0)), running: CachePadded::new(AtomicBool::new(false)), asleep: CachePadded::new(AtomicBool::new(false)), @@ -199,6 +248,10 @@ impl Default for CommandFifo { #[cfg(test)] test_parked: AtomicU32::new(0), #[cfg(test)] + test_refused: AtomicU32::new(0), + #[cfg(test)] + test_release: AtomicBool::new(false), + #[cfg(test)] test_after_ring_push: Mutex::new(None), } } @@ -251,9 +304,9 @@ impl CommandFifo { self.running.load(SeqCst) && !self.has_room() } - /// The processor waits for the board's thread to run what is queued, as - /// a refused access does. A thread already in its pause (`gap`) is woken - /// to run at once. + /// The processor waits for the board's thread to run what is queued: + /// a refused access, or a counted store whose credit is owed. A thread + /// already in its pause (`gap`) is woken to run at once. pub fn hurry(&self) { // A refused store retries in a loop; write the line only when it // changes, as IMPACT does for its asleep flag. @@ -310,20 +363,36 @@ impl CommandFifo { self.running.load(SeqCst) || self.pending() || self.held() } - /// Queue a store. False when the host ring has no room for it. - pub fn push(&self, front: &Mutex, addr: u64, value: u64, size: u32) -> bool { + /// The stores in the FIFO's first places, credited. + pub fn in_fifo(&self) -> u32 { + self.stores().min(cfifo::DEPTH) + } + + /// Queue a store. A counted store is credited at once when it lands + /// inside the FIFO's first places and owed otherwise. False when the + /// board's FIFO has no room for it. + pub fn push(&self, front: &Mutex, addr: u64, value: u64, size: u32, counted: bool) -> bool { let words = store_words(size); if !self.has_room() { + #[cfg(test)] + self.test_refused.fetch_add(1, SeqCst); return false; } - let q = QueuedStore { value, addr: addr as u32, size: size as u8 }; + let mut q = QueuedStore { value, addr: addr as u32, size: size as u8, counted, owed: false }; + // A counted store's place is settled against the consumer's pops. + let _c = counted.then(|| self.pop.lock()); // Count the store before the ring shows it. The board's thread can // take it and retire it the moment it is in the ring, and a retire // must find every store it takes already counted, or the level falls // below zero. Until the push lands the counts are a store high, // which reads as a store still waiting, never as one gone. - self.levels.stores.fetch_add(1, SeqCst); + if counted { + self.levels.counted.fetch_add(1, SeqCst); + } + let stores_before = self.levels.stores.fetch_add(1, SeqCst); let before = self.levels.words.fetch_add(words, SeqCst); + let credited = counted && stores_before < cfifo::DEPTH; + q.owed = counted && !credited; if !self.ring.try_push(q.entry(), q.value) { // With one processor this cannot happen: has_room counted the // ring's stores and nothing else pushes. With two, the other @@ -333,11 +402,32 @@ impl CommandFifo { // come back off. self.levels.words.fetch_sub(words, SeqCst); self.levels.stores.fetch_sub(1, SeqCst); + if counted { + self.levels.counted.fetch_sub(1, SeqCst); + } return false; } #[cfg(test)] - if let Some(consumer) = self.test_after_ring_push.lock().take() { - consumer(self, front); + if !counted { + if let Some(consumer) = self.test_after_ring_push.lock().take() { + consumer(self, front); + } + } + if counted { + let mut f = front.lock(); + if credited { + f.return_credit(); + } else { + if let Some(pi) = &f.pi { + pi_gfx_owe_credits(pi.as_ref(), f.slot, 1); + } + drop(f); + // The credit comes back only as the stores ahead retire, and + // the processor can be watching its credit count (a read the + // bus does not refuse), so the board's thread runs without + // its pause. + self.hurry(); + } } let after = before + words; let mark = self.threshold_6b2.load(SeqCst); @@ -353,17 +443,28 @@ impl CommandFifo { } /// Look at the oldest waiting stores, up to `RUN_MAX`, without taking - /// them out; the caller runs them, then `retire`s the ones it ran. + /// them out; the caller runs them, then `retire`s the ones it ran. While + /// stores wait past the credited depth, one of which may owe its credit, + /// the run is one store, so each owed credit is settled as the store + /// ahead of it leaves rather than after a whole run. Inside the depth + /// every counted store already has its credit, so the run is full. pub fn take_run(&self, run: &mut Run) { + // Only a hint for the run's length, so a stale count costs a longer + // run, never a lost credit (`retire` settles each one). The stores + // are read first and short-circuit it: a Relaxed read of `counted` + // after seeing more than DEPTH stores sees every counted push whose + // store count it read, since a push counts `counted` first. + let owing = self.stores() > cfifo::DEPTH && self.levels.counted.load(Relaxed) != 0; + let max = if owing { 1 } else { RUN_MAX }; let mut raw = [(0u32, 0u64); RUN_MAX]; - run.len = self.ring.peek_run_at(0, &mut raw); + run.len = self.ring.peek_run_at(0, &mut raw[..max]); for (store, (addr, value)) in run.stores.iter_mut().zip(&raw[..run.len]) { *store = QueuedStore::from_entry(*addr, *value); } } /// Take the first `n` stores of `run` out of the FIFO: they have run. The - /// level and the stores waiting move once for the run, not + /// level, the stores waiting and the credits move once for the run, not /// once per store, which is what the board's thread spends most of its /// time on when each store costs little to decode. A store the thread has /// run but not retired still counts in the level, so the level never @@ -379,6 +480,7 @@ impl CommandFifo { } let ran = &run.stores[..n]; let words: u32 = ran.iter().map(QueuedStore::words).sum(); + let counted_ran = ran.iter().filter(|q| q.counted).count() as u32; let _c = self.pop.lock(); self.ring.consume_n_at(0, n); if self.levels.words.load(SeqCst) < cfifo::LOW_WATER + words { @@ -393,17 +495,59 @@ impl CommandFifo { } else { self.levels.words.fetch_sub(words, SeqCst); } - self.levels.stores.fetch_sub(n as u32, SeqCst); + let stores_before = self.levels.stores.fetch_sub(n as u32, SeqCst); + if counted_ran != 0 { + self.levels.counted.fetch_sub(counted_ran, SeqCst); + } + // `pop` is held, and a counted push takes it to count `counted`, so + // this read needs no ordering of its own. Relaxed keeps the + // machines with no counted stores (every one but a linked + // processor interface) off a sequentially consistent load per run. + if self.levels.counted.load(Relaxed) == 0 { + return; + } + // The next waiting store moves into the FIFO's last credited place + // and is credited. + for k in 1..=n as u32 { + if stores_before - k < cfifo::DEPTH { + continue; + } + // The run is already out of the ring, so count back from its head. + let place = cfifo::DEPTH as usize - 1 - (n - k as usize); + if let Some((addr, value)) = self.ring.peek_nth_at(0, place) { + let entered = QueuedStore::from_entry(addr, value); + let mut f = front.lock(); + if entered.owed { + if let Some(pi) = &f.pi { + pi_gfx_repay_credits(pi.as_ref(), f.slot, 1); + } + } else if entered.counted { + f.return_credit(); + } + } + } } - /// Abandon every waiting store. - pub fn clear(&self) { + /// Abandon every waiting store; an owed one outside the FIFO gives its + /// owed credit back. + pub fn clear(&self, front: &Mutex) { let _c = self.pop.lock(); + let mut index = 0u32; while let Some((addr, value)) = self.ring.peek() { self.ring.consume(); let q = QueuedStore::from_entry(addr, value); self.levels.words.fetch_sub(q.words(), SeqCst); self.levels.stores.fetch_sub(1, SeqCst); + if q.counted { + self.levels.counted.fetch_sub(1, SeqCst); + } + if q.owed && index >= cfifo::DEPTH { + let f = front.lock(); + if let Some(pi) = &f.pi { + pi_gfx_owe_credits(pi.as_ref(), f.slot, -1); + } + } + index += 1; } } diff --git a/src/dev/odyssey/interface.rs b/src/dev/odyssey/interface.rs index 139129035..8790b8b5c 100644 --- a/src/dev/odyssey/interface.rs +++ b/src/dev/odyssey/interface.rs @@ -27,13 +27,22 @@ use super::consts::{ }; use super::fifo::CommandFifo; use super::mmio::{access_waits, command_store, fifo_port}; +use super::pi_gfx::{pi_gfx_return_credits, PiGfxLink}; use super::registers::reg; +use super::PacketSender; use parking_lot::Mutex; use std::sync::Arc; pub(super) struct HostInterface { - /// Where interrupt packets go (the iris glue's HEART). + /// The crosstalk widget number. + pub slot: u32, + /// Where interrupt packets go on the Octane (the iris glue's HEART). pub irq: Option>, + /// Where interrupt packets go on a machine whose fabric carries them to + /// any widget (the Fuel's XBridge crossbar); `irq` is unused then. + pub packets: Option, + /// The master node's processor interface, for graphics credits. + pub pi: Option>, /// The widget number of HEART, the packets' destination. pub heart_widget: u32, /// ODSY_IDENTITY, composed from the revision at reset. @@ -73,6 +82,9 @@ pub(super) struct HostInterface { /// The command FIFO (fifo.rs), whose level STATUS0 reports. pub fifo: Arc, + /// Tests: stores count as counted by a processor interface. + #[cfg(test)] + pub test_counted_stores: u8, /// The front accesses' own trace numbering; the board's accesses keep /// the state's. pub trace_sequence: u64, @@ -81,7 +93,10 @@ pub(super) struct HostInterface { impl Default for HostInterface { fn default() -> Self { HostInterface { + slot: 0, irq: None, + packets: None, + pi: None, heart_widget: 8, identity: 0, widget_control: 0, @@ -108,6 +123,8 @@ impl Default for HostInterface { displayed_buffer: 0, displayed_written: false, fifo: Default::default(), + #[cfg(test)] + test_counted_stores: 0, trace_sequence: 0, } } @@ -281,6 +298,15 @@ impl HostInterface { self.update_irq(); } } + + /// A counted store's credit, to the processor interface the node names. + pub fn return_credit(&mut self) { + if let Some(pi) = &self.pi { + pi_gfx_return_credits(pi.as_ref(), self.slot, 1); + } else { + crate::dlog!(LogModule::Odyssey, "sgi-odyssey: counted graphics store has no processor interface for its return"); + } + } } // The front registers. @@ -387,7 +413,7 @@ impl HostInterface { self.flctl_target_reported = true; crate::dlog!( LogModule::Odyssey, - "sgi-odyssey: flow-control interrupt target {} and credit target {} stored", + "sgi-odyssey: flow-control interrupt target {} and credit target {} stored; credits still return through the node interface", (value as u32) >> flctl::INTR_TARGET_SHIFT, ((value >> flctl::TARGET_SHIFT) as u32) & flctl::TARGET_MASK ); @@ -453,15 +479,20 @@ pub(super) fn front_mmio_read(f: &Mutex, addr: u64, size: u32) -> /// A processor's store through the front alone: true when it went to a /// front register or joined the command FIFO for the board's thread, false -/// when it needs the board's state. -pub(super) fn front_mmio_write(f: &Mutex, fifo: &CommandFifo, addr: u64, value: u64, size: u32) -> bool { +/// when it needs the board's state. `counted`: the processor interface counted +/// it. +pub(super) fn front_mmio_write(f: &Mutex, fifo: &CommandFifo, addr: u64, value: u64, size: u32, counted: bool) -> bool { if front_register(addr, size, true) { - f.lock().write(addr, value, size); + let mut f = f.lock(); + f.write(addr, value, size); + if counted { + f.return_credit(); + } return true; } // A queued store needs the board's thread to run it, and the queue to // have room; otherwise the caller runs it. - command_store(addr, size) && fifo.running.load(std::sync::atomic::Ordering::SeqCst) && fifo.push(f, addr, value, size) + command_store(addr, size) && fifo.running.load(std::sync::atomic::Ordering::SeqCst) && fifo.push(f, addr, value, size, counted) } /// The fabric packet's address bits; the widget sits above them. @@ -488,6 +519,12 @@ impl HostInterface { pub fn send_interrupt(&mut self, packet: u64, number: u8, set: bool) { let widget = (packet >> XIO_PACKET_ADDRESS_BITS) as u32; let address = packet & ((1u64 << XIO_PACKET_ADDRESS_BITS) - 1); + if let Some(send) = &self.packets { + if !send(widget as u8, address, number, set) { + crate::dlog!(LogModule::Odyssey, "odyssey: interrupt packet to widget {widget} address {address:#x} was refused"); + } + return; + } if widget != self.heart_widget || address != HEART_PACKET { crate::dlog!(LogModule::Odyssey, "odyssey: interrupt packet to widget {widget} address {address:#x} has no receiver"); return; diff --git a/src/dev/odyssey/mmio.rs b/src/dev/odyssey/mmio.rs index 59e3c7708..c009b1218 100644 --- a/src/dev/odyssey/mmio.rs +++ b/src/dev/odyssey/mmio.rs @@ -18,6 +18,7 @@ use super::consts::{ }; use super::fifo::CommandFifo; use super::interface::{front_mmio_read, front_mmio_write, front_register, HostInterface}; +use super::pi_gfx::pi_gfx_take_mark; use super::raster_unit::RasterQueue; use super::reader::DisplayShared; use super::registers::reg; @@ -132,6 +133,10 @@ impl Board { // see the drawing are answered busy until the queue is empty // (BoardLocks::bus_read) or, with no thread running (unit tests), run what // is queued first (Board::work_drain). + // + // The Octane has no PI link (only Hub and Bedrock machines have graphics + // credits), so the front's `pi` stays None there and every credit call is + // skipped. } @@ -221,13 +226,31 @@ impl Board { } } +/// Whether this store is counted: the processor interface's mark. +pub(super) fn store_counted(front: &parking_lot::Mutex) -> bool { + // The Octane has no PI link, so nothing sets the mark there. + #[cfg(test)] + if front.lock().test_counted_stores != 0 { + return true; + } + let _ = front; + pi_gfx_take_mark() +} + impl Board { /// A store to the board's state. pub fn mmio_write(&mut self, addr: u64, value: u64, size: u32) { + let counted = store_counted(&self.front); + self.mmio_write_counted(addr, value, size, counted); + } + + pub fn mmio_write_counted(&mut self, addr: u64, value: u64, size: u32, mut counted: bool) { let fifo = self.fifo.clone(); - let queued = command_store(addr, size) && fifo.queues() && fifo.push(&self.front, addr, value, size); + let queued = command_store(addr, size) && fifo.queues() && fifo.push(&self.front, addr, value, size, counted); if queued { - // With no board thread running the storing caller runs it. + // Credited as it enters the FIFO (CommandFifo::push). With no + // board thread running the storing caller runs it. + counted = false; if !fifo.running.load(std::sync::atomic::Ordering::SeqCst) && !fifo.held() { self.work_drain(addr); } @@ -242,6 +265,10 @@ impl Board { } self.write_dispatch(addr, value, size); } + if counted { + // It went through the FIFO inside the store. + self.front.lock().return_credit(); + } } } @@ -487,7 +514,8 @@ impl BoardLocks { /// answer busy, when it waits for queued stores the board's thread will /// run, or when it is a command store and the host ring is full while /// the board's thread runs: the processor retries the store, taking its - /// interrupts meanwhile, while the board's thread makes room. + /// interrupts meanwhile, while the board's thread makes room. The + /// processor interface's mark stays untaken until the store goes in. /// /// A store that waits is held back for the command FIFO only, not the /// raster queue: the raster unit never shared a lock with the board's @@ -524,25 +552,27 @@ impl BoardLocks { /// A command store the caller has classified, holding `submit`: it joins /// the FIFO for the board's thread, else, with no thread, runs here. fn write_command(&self, cpu: crate::locks::SpinGuard<'_, ()>, addr: u64, value: u64, size: u32) { + let counted = store_counted(&self.front); while self.fifo.running.load(std::sync::atomic::Ordering::SeqCst) { // The ring refuses a push that finds the last place taken between // the room check and the push: the board's thread owns the state, // so the store goes in behind it rather than around it. - if self.fifo.push(&self.front, addr, value, size) { + if self.fifo.push(&self.front, addr, value, size, counted) { return; } std::thread::yield_now(); } // SAFETY: no board thread is running. - unsafe { self.board(cpu) }.mmio_write(addr, value, size); + unsafe { self.board(cpu) }.mmio_write_counted(addr, value, size, counted); } /// A store to the board's state that is not a command store, holding /// `submit`. fn write_board(&self, cpu: crate::locks::SpinGuard<'_, ()>, addr: u64, value: u64, size: u32) { + let counted = store_counted(&self.front); // SAFETY: the waiting accesses have drained the FIFO, or the board's // thread is stopped. - unsafe { self.board(cpu) }.mmio_write(addr, value, size); + unsafe { self.board(cpu) }.mmio_write_counted(addr, value, size, counted); } /// A store for a caller other than the bus: through the front alone when it @@ -551,7 +581,8 @@ impl BoardLocks { /// for a store that waits. pub fn write(&self, addr: u64, value: u64, size: u32) { if front_register(addr, size, true) { - front_mmio_write(&self.front, &self.fifo, addr, value, size); + let counted = store_counted(&self.front); + front_mmio_write(&self.front, &self.fifo, addr, value, size, counted); return; } let cpu = self.submit.lock(); diff --git a/src/dev/odyssey/mod.rs b/src/dev/odyssey/mod.rs index c7edb4bf6..e87fd90e1 100644 --- a/src/dev/odyssey/mod.rs +++ b/src/dev/odyssey/mod.rs @@ -41,7 +41,8 @@ //! - `memory`, `scanout`, `reader`, `display`: board memory and planes; the //! marks writers make and the display thread's own reader, with the scanned //! frame and retrace; what reaches the window. -//! - `pixel_dma`: DMA between guest memory and planes. +//! - `pixel_dma`, `pi_gfx`: DMA between guest memory and planes, and graphics +//! credits for machines whose processor interface counts them. //! - `depth_word`: the board's 24-bit float depth word and its offset. //! - `consts`: register, command and field constants. //! @@ -73,6 +74,8 @@ mod interface; mod memory; mod mmio; mod objects; +mod pi_gfx; +pub use pi_gfx::{PiGfx, PI_GFX_REGS}; mod pixel_dma; mod pixel_write; mod raster; @@ -129,9 +132,15 @@ pub struct Odyssey { rec: Option, } +/// Carries a board interrupt packet into a machine's crosstalk fabric: the +/// destination widget, the address within it, the vector, and whether the +/// source rose or fell. False when nothing at the destination takes it. +pub type PacketSender = Arc bool + Send + Sync>; + impl Odyssey { - /// A board of `product` (v12, v10, v8, v6) at `widget`: its interrupt - /// packets reach `irq` (HEART), its DMA reaches `mem`. + /// A board of `product` (v12, v10, v8, v6) at `widget` on the Octane: + /// the processor reaches it through the IP30 widget windows, its + /// interrupt packets reach `irq` (HEART), its DMA reaches `mem`. pub fn new( widget: u8, product: &str, @@ -140,25 +149,62 @@ impl Odyssey { status: Arc, ) -> Result, String> { let mut s = Board::new(product, widget as u32)?; - let rec = match std::env::var("IRIS_ODSY_REC") { - Ok(path) => Some(record::Recorder::create(&path).map_err(|e| format!("IRIS_ODSY_REC {path}: {e}"))?), - Err(_) => None, - }; - let mem: Arc = match &rec { + let rec = Self::recorder()?; + s.guest = Some(Self::recorded_mem(mem, &rec)); + s.front.lock().irq = Some(irq); + Ok(Self::with_board(s, widget_small_base(widget), widget_large_base(widget), status, rec)) + } + + /// A board of `product` at `widget` on a machine that hands the board + /// widget-relative addresses, as Bedrock's I/O interface does, and + /// carries its interrupt packets through `packets`. + pub fn new_widget_relative( + widget: u8, + product: &str, + packets: PacketSender, + mem: Arc, + status: Arc, + ) -> Result, String> { + let mut s = Board::new(product, widget as u32)?; + let rec = Self::recorder()?; + s.guest = Some(Self::recorded_mem(mem, &rec)); + s.front.lock().packets = Some(packets); + // Every address is already an aperture offset: no large window. + Ok(Self::with_board(s, 0, u64::MAX, status, rec)) + } + + /// `IRIS_ODSY_REC`'s recorder, when it is set. + fn recorder() -> Result, String> { + match std::env::var("IRIS_ODSY_REC") { + Ok(path) => Ok(Some(record::Recorder::create(&path).map_err(|e| format!("IRIS_ODSY_REC {path}: {e}"))?)), + Err(_) => Ok(None), + } + } + + /// `mem`, recorded when a recording is running. + fn recorded_mem(mem: Arc, rec: &Option) -> Arc { + match rec { Some(r) => Arc::new(record::RecordingMem { inner: mem, rec: r.clone() }), None => mem, - }; - s.guest = Some(mem); - s.front.lock().irq = Some(irq); - Ok(Arc::new(Self { - small: widget_small_base(widget), - large: widget_large_base(widget), + } + } + + fn with_board( + s: Box, + small: u64, + large: u64, + status: Arc, + rec: Option, + ) -> Arc { + Arc::new(Self { + small, + large, state: BoardLocks::new(s), display: Arc::new(Display::new(status)), running: Arc::new(AtomicBool::new(false)), thread: Mutex::new(None), rec, - })) + }) } pub fn display(&self) -> Arc { diff --git a/src/dev/odyssey/pi_gfx.rs b/src/dev/odyssey/pi_gfx.rs new file mode 100644 index 000000000..cf1bb67a3 --- /dev/null +++ b/src/dev/odyssey/pi_gfx.rs @@ -0,0 +1,210 @@ +//! Graphics flow control in the Hub and Bedrock processor interfaces. The +//! Octane has no processor-interface link (its board's `pi` stays `None`), +//! so on that machine the credit calls are no-ops. `PiGfx` is the register +//! block a node's processor interface embeds; its functions are pure state. +//! The board reaches the link through the `PiGfxLink` trait. +//! +//! Ported from hw/mips/sgi/pi-gfx.c in jamesbraid/qemu-sgi-origami. + +use crate::devlog::LogModule; +use std::cell::Cell; + +pub(super) const PI_GFX_SLOTS: usize = 2; +pub const PI_GFX_REGS: u32 = 10; + +/// The five registers of one slot, in address order. +pub(super) const PI_GFX_PAGE: u32 = 0; +pub(super) const PI_GFX_CREDIT: u32 = 1; +pub(super) const PI_GFX_BIAS: u32 = 2; +pub(super) const PI_GFX_INT_COUNT: u32 = 3; +pub(super) const PI_GFX_SLOT_REGS: u32 = 5; + +/// Page: bit 40 enables it, bits 39:17 name it; the rest is reserved. +pub(super) const PI_GFX_PAGE_ENABLE: u64 = 1 << 40; +pub(super) const PI_GFX_PAGE_ADDRESS: u64 = 0xfffffe0000; +pub(super) const PI_GFX_PAGE_MASK: u64 = PI_GFX_PAGE_ENABLE | PI_GFX_PAGE_ADDRESS; +pub(super) const PI_GFX_CREDIT_MASK: u32 = 0xfff; +pub(super) const PI_GFX_BIAS_MASK: u32 = 0xfff; +pub(super) const PI_GFX_INT_MASK: u32 = 0x3ffffff; + +#[derive(Clone, Copy, Default)] +pub struct PiGfx { + pub page: [u64; PI_GFX_SLOTS], + pub credit: [u32; PI_GFX_SLOTS], + pub bias: [u32; PI_GFX_SLOTS], + pub int_count: [u32; PI_GFX_SLOTS], + pub int_compare: [u32; PI_GFX_SLOTS], + /// The slot's graphics interrupt has been raised for the counter falling + /// below the bias, and not yet re-armed by the counter rising above it. + pub starved: [bool; PI_GFX_SLOTS], + /// Host-only: the zero-length stall has been logged for this slot. + pub stall_logged: [bool; PI_GFX_SLOTS], + /// Bedrock: a counted store left the counter below the bias, and the + /// slot's next counted store waits until the processor has taken the + /// graphics interrupt or the counter has risen above the bias. + pub stopped: [bool; PI_GFX_SLOTS], + /// Bedrock: the interrupt counter runs (the credit counter is at zero). + pub int_running: [bool; PI_GFX_SLOTS], + /// The clock tick at which int_count held its value. + pub int_epoch: [u64; PI_GFX_SLOTS], +} + +/// The interface a node device implements for the graphics heads behind it +/// (return_credits, owe_credits and repay_credits). `widget` is the head's crosstalk widget number, the +/// value the driver writes into the node-widget map. +pub(super) trait PiGfxLink: Send + Sync { + /// A graphics head took `n` counted stores off its port. + fn return_credits(&self, widget: u32, n: u32); + /// Credits a head owes for counted stores it has not yet taken off its + /// port; a negative count cancels owed credits, as a board reset that + /// abandons the words does. A node without a ledger ignores the debt. + fn owe_credits(&self, widget: u32, n: i32) { + let _ = (widget, n); + } + /// Owed credits returned as the head takes the words. A node without a + /// ledger routes a repayment as an ordinary return. + fn repay_credits(&self, widget: u32, n: u32) { + self.return_credits(widget, n); + } +} + +/// Return `credits` to `node` for stores a head took off its port. +pub(super) fn pi_gfx_return_credits(node: &dyn PiGfxLink, widget: u32, credits: u32) { + node.return_credits(widget, credits); +} + +/// Record `credits` a head owes `node` for stores it has not yet taken. +pub(super) fn pi_gfx_owe_credits(node: &dyn PiGfxLink, widget: u32, credits: i32) { + node.owe_credits(widget, credits); +} + +/// Repay `credits` that `node` held as owed. +pub(super) fn pi_gfx_repay_credits(node: &dyn PiGfxLink, widget: u32, credits: u32) { + node.repay_credits(widget, credits); +} + +impl PiGfx { + /// The counter above the bias re-arms the slot's graphics interrupt. + fn rearm(&mut self, slot: usize) { + if self.credit[slot] > self.bias[slot] { + self.starved[slot] = false; + } + } + + /// Register n of the block, 0 to PI_GFX_REGS - 1, counted in + /// doublewords from PI_GFX_BASE. + pub fn read(&self, n: u32) -> u64 { + let slot = (n / PI_GFX_SLOT_REGS) as usize; + + assert!(n < PI_GFX_REGS); + match n % PI_GFX_SLOT_REGS { + PI_GFX_PAGE => self.page[slot], + PI_GFX_CREDIT => self.credit[slot] as u64, + PI_GFX_BIAS => self.bias[slot] as u64, + PI_GFX_INT_COUNT => self.int_count[slot] as u64, + _ => self.int_compare[slot] as u64, + } + } + + /// A write keeps the field's bits; a write to the credit counter replaces + /// the count. + pub fn write(&mut self, n: u32, value: u64) { + let slot = (n / PI_GFX_SLOT_REGS) as usize; + + assert!(n < PI_GFX_REGS); + match n % PI_GFX_SLOT_REGS { + PI_GFX_PAGE => self.page[slot] = value & PI_GFX_PAGE_MASK, + PI_GFX_CREDIT => { + self.credit[slot] = (value as u32) & PI_GFX_CREDIT_MASK; + self.rearm(slot); + } + PI_GFX_BIAS => self.bias[slot] = (value as u32) & PI_GFX_BIAS_MASK, + PI_GFX_INT_COUNT => self.int_count[slot] = (value as u32) & PI_GFX_INT_MASK, + _ => self.int_compare[slot] = (value as u32) & PI_GFX_INT_MASK, + } + } + + /// Whether the slot's page is enabled and covers physical address `addr`. + pub fn covers(&self, slot: usize, addr: u64) -> bool { + assert!(slot < PI_GFX_SLOTS); + let page = self.page[slot]; + (page & PI_GFX_PAGE_ENABLE) != 0 + && (addr & PI_GFX_PAGE_ADDRESS) == (page & PI_GFX_PAGE_ADDRESS) + } + + /// An uncached I/O store by the processor in `slot` reached physical + /// address bits 39:0 `addr`. Charges one credit when the slot's page is + /// enabled and covers it, and returns whether it did. `who` names the + /// device in log messages. + pub fn count_store(&mut self, slot: usize, addr: u64, who: &str) -> bool { + if !self.covers(slot, addr) { + return false; + } + // Whether the twelve-bit counter wraps, saturates or goes negative is + // unknown. It wraps, and says so. + if self.credit[slot] == 0 { + crate::dlog!(LogModule::Odyssey, "{}: graphics credit counter {} wrapped below zero", who, (b'A' + slot as u8) as char); + } + self.credit[slot] = self.credit[slot].wrapping_sub(1) & PI_GFX_CREDIT_MASK; + true + } + + /// After a counted store: whether the slot's graphics interrupt must be + /// raised now. True once when the counter has fallen below the bias; the + /// counter rising above the bias again re-arms it. The stall that + /// accompanies it on the chip is modelled as zero length. + pub fn take_starved(&mut self, slot: usize, who: &str) -> bool { + assert!(slot < PI_GFX_SLOTS); + if self.starved[slot] || self.credit[slot] >= self.bias[slot] { + return false; + } + self.starved[slot] = true; + if !self.stall_logged[slot] { + self.stall_logged[slot] = true; + crate::dlog!( + LogModule::Odyssey, + "{}: graphics credit counter {} fell below its bias; interrupt raised at once, the write stall and the interrupt counter are not modelled", + who, + (b'A' + slot as u8) as char + ); + } + true + } + +} + +thread_local! { + /// The mark on the store a processor path is forwarding, per host + /// thread: the forward and the head's handler run on the issuing + /// vCPU's thread. + static PI_GFX_FORWARDING_COUNTED: Cell = const { Cell::new(false) }; +} + +/// A processor interface marks each store it forwards down its I/O path; +/// returns the previous mark, to be restored with pi_gfx_unmark_store. +pub(super) fn pi_gfx_mark_store(counted: bool) -> bool { + PI_GFX_FORWARDING_COUNTED.with(|m| m.replace(counted)) +} + +pub(super) fn pi_gfx_unmark_store(previous: bool) { + PI_GFX_FORWARDING_COUNTED.with(|m| m.set(previous)); +} + +/// The one-shot read: the call clears the mark. +pub(super) fn pi_gfx_take_mark() -> bool { + PI_GFX_FORWARDING_COUNTED.with(|m| m.replace(false)) +} + +impl PiGfx { + /// `credits` credits returned to `slot`. + pub fn add_credits(&mut self, slot: usize, credits: u32, who: &str) { + assert!(slot < PI_GFX_SLOTS); + let sum = self.credit[slot] as u64 + credits as u64; + if sum > PI_GFX_CREDIT_MASK as u64 { + crate::dlog!(LogModule::Odyssey, "{}: graphics credit counter {} wrapped above {:#x}", who, (b'A' + slot as u8) as char, PI_GFX_CREDIT_MASK); + } + self.credit[slot] = (sum & PI_GFX_CREDIT_MASK as u64) as u32; + self.rearm(slot); + } +} + diff --git a/src/dev/odyssey/tests/fifo.rs b/src/dev/odyssey/tests/fifo.rs index 1ba7f683b..7f088ebf0 100644 --- a/src/dev/odyssey/tests/fifo.rs +++ b/src/dev/odyssey/tests/fifo.rs @@ -1,4 +1,4 @@ -//! The command FIFO: queueing, the board's thread, status. +//! The command FIFO: queueing, the board's thread, status, credits. use crate::dev::odyssey::board::Board; use crate::dev::odyssey::command::{CFIFO_PREAMBLE, CfifoPreamble}; @@ -8,6 +8,10 @@ use crate::dev::odyssey::consts::{ use crate::dev::odyssey::fifo::{CommandFifo, RING_STORES, RUN_MAX, Run}; use crate::dev::odyssey::interface::HostInterface; use crate::dev::odyssey::mmio::BoardLocks; +use crate::dev::odyssey::pi_gfx::{ + pi_gfx_mark_store, pi_gfx_take_mark, pi_gfx_unmark_store, PI_GFX_BIAS, PI_GFX_CREDIT, + PI_GFX_PAGE, PI_GFX_PAGE_ENABLE, PiGfx, PiGfxLink, +}; use crate::dev::odyssey::tests::{ board, board_with_ram, cmd, cmd64, reg_read, reg_write, route_to_heart, select, FLAG_VECTOR, Ram, VPRO_WIDGET, @@ -210,41 +214,151 @@ fn a_waiting_bus_read_is_busy_until_the_board_thread_has_run_the_stores() { h.work_stop(); } +/// Records every call a board makes on its processor-interface link. +#[derive(Default)] +struct PiStub(Mutex>); + +impl PiGfxLink for PiStub { + fn return_credits(&self, widget: u32, n: u32) { + self.0.lock().push(("return", widget, n as i64)); + } + fn owe_credits(&self, widget: u32, n: i32) { + self.0.lock().push(("owe", widget, n as i64)); + } + fn repay_credits(&self, widget: u32, n: u32) { + self.0.lock().push(("repay", widget, n as i64)); + } +} + +impl PiStub { + fn count(&self, kind: &str) -> i64 { + self.0.lock().iter().filter(|c| c.0 == kind).map(|c| c.2).sum() + } + /// Credits given back so far, returned or repaid. + fn settled(&self) -> i64 { + self.count("return") + self.count("repay") + } +} + +/// A board whose stores are all counted and queued, with a stub link. +fn credit_board() -> (Box, Arc) { + let (s, _) = board("v12"); + let pi = Arc::new(PiStub::default()); + { + let mut f = s.front.lock(); + f.pi = Some(pi.clone()); + f.test_counted_stores = 1; + } + // No work thread runs: hold the queue and step by hand. + s.fifo.test_words.store(1, SeqCst); + (s, pi) +} + +/// Command work credits at FIFO entry: a counted store's credit +/// comes back when the board takes it into its FIFO, the oldest +/// `cfifo::DEPTH` waiting stores at once; each store behind them is owed, then +/// repaid as the work ahead runs and it moves up. A store that does +/// not wait is credited inside the store. +/// +/// QEMU: command-work-credits-at-fifo-entry. +#[test] +fn a_counted_store_is_credited_as_it_enters_the_fifo() { + let (mut s, pi) = credit_board(); + let stores = cfifo::DEPTH + 8; + + for _ in 0..stores { + s.mmio_write(consts::cmd::FIFO, 0, 8); + } + // Credits count stores; the level counts words, two a doubleword, and + // reads no more than the model's FIFO size. + assert_eq!(s.fifo.stores(), stores); + assert_eq!(s.fifo.level(), (2 * stores).min(cfifo::WORDS)); + assert_eq!(s.fifo.in_fifo(), cfifo::DEPTH); + assert_eq!(pi.count("return"), cfifo::DEPTH as i64); + assert_eq!(pi.count("owe"), (stores - cfifo::DEPTH) as i64); + assert_eq!(pi.count("repay"), 0); + assert!(pi.0.lock().iter().all(|c| c.1 == VPRO_WIDGET)); + + // One store runs per slice: the next one moves up and is repaid. + s.work_step(&mut Budget::stores(1)); + assert_eq!(pi.count("repay"), 1); + assert_eq!(pi.settled(), cfifo::DEPTH as i64 + 1); + + s.work_step(&mut Budget::unlimited()); + assert!(!s.fifo.pending()); + assert_eq!(pi.settled(), stores as i64); + + // A store that does not wait is credited inside the store. + s.fifo.test_words.store(0, SeqCst); + s.mmio_write(consts::cmd::FIFO, 0, 8); + assert!(!s.work_pending()); + assert_eq!(pi.settled(), stores as i64 + 1); + assert_eq!(pi.count("owe"), (stores - cfifo::DEPTH) as i64); +} + /// A run of stores the board's thread has looked at and then runs leaves -/// the FIFO together. The stores run in order; the level and the store count -/// end where taking the stores out one by one leaves them, including when -/// more stores join the queue after the run was taken. +/// the FIFO together. The stores run in order; the level, the store count +/// and the credits end where taking the stores out one by one leaves them, +/// including when a counted store joins the queue after the run was taken. #[test] -fn a_run_retires_with_the_levels_of_one_by_one() { +fn a_run_retires_with_the_levels_and_credits_of_one_by_one() { fn queue(s: &mut Board) { - for _ in 0..70 { + s.front.lock().test_counted_stores = 0; + for _ in 0..cfifo::DEPTH + 6 { s.mmio_write(consts::cmd::FIFO, 0, 8); } } - fn join(s: &mut Board) { + fn counted_join(s: &mut Board) { + s.front.lock().test_counted_stores = 1; for _ in 0..10 { s.mmio_write(consts::cmd::FIFO, 0, 8); } } - // One at a time: the store count and the level are the reference. - let (mut one, _, _) = queued_board(); + // One at a time: the store count and the credits are the reference. + let (mut one, pi_one) = credit_board(); queue(&mut one); - join(&mut one); + counted_join(&mut one); for _ in 0..RUN_MAX { step(&mut one); } - let (mut run, _, _) = queued_board(); + let (mut run, pi_run) = credit_board(); queue(&mut run); let mut taken = Run::new(); run.fifo.take_run(&mut taken); - assert_eq!(taken.stores().len(), RUN_MAX, "a full run"); - join(&mut run); - assert_eq!(run.fifo.stores(), 80); + assert_eq!(taken.stores().len(), RUN_MAX, "no counted store waits yet: a full run"); + counted_join(&mut run); + assert_eq!(run.fifo.stores(), cfifo::DEPTH + 16); run.fifo.retire(&run.front, &taken, RUN_MAX); assert_eq!(run.fifo.stores(), one.fifo.stores()); assert_eq!(run.fifo.level(), one.fifo.level()); + assert_eq!(pi_run.count("repay"), pi_one.count("repay")); + assert_eq!(pi_run.count("return"), pi_one.count("return")); + assert_eq!(pi_run.count("owe"), pi_one.count("owe")); + assert!(pi_run.count("repay") > 0, "the run moved owed stores up"); + // The counted stores left all sit inside the credited depth, so the + // next run is full. + assert!(run.fifo.stores() <= cfifo::DEPTH); + run.fifo.take_run(&mut taken); + assert_eq!(taken.stores().len(), RUN_MAX); +} + +/// The board's thread takes stores one at a time only while some wait past +/// the credited depth, where one may owe its credit; a backlog of counted +/// stores inside the depth, all credited, goes in full runs. +#[test] +fn a_run_is_one_store_only_past_the_credited_depth() { + let (mut s, _) = credit_board(); + for _ in 0..cfifo::DEPTH { + s.mmio_write(consts::cmd::FIFO, 0, 8); + } + let mut run = Run::new(); + s.fifo.take_run(&mut run); + assert_eq!(run.stores().len(), RUN_MAX, "every waiting store is credited"); + s.mmio_write(consts::cmd::FIFO, 0, 8); + s.fifo.take_run(&mut run); + assert_eq!(run.stores().len(), 1, "one store owes its credit"); } /// A slice cut short by its budget retires only the stores it ran: the rest @@ -265,6 +379,76 @@ fn a_slice_cut_short_leaves_the_rest_of_its_run_queued() { assert_eq!(ram.half(HOST_WRITE_BASE + 6), 0xa5a5); } +/// A reset abandons the waiting stores and cancels each owed credit +/// (one owe of -1 per owed store); stores not owed cancel nothing. +#[test] +fn a_reset_cancels_the_credits_owed_for_abandoned_stores() { + let (mut s, pi) = credit_board(); + + for _ in 0..cfifo::DEPTH + 8 { + s.mmio_write(consts::cmd::FIFO, 0, 8); + } + assert_eq!(pi.count("owe"), 8); + s.work_reset(); + assert!(!s.fifo.pending()); + assert_eq!(s.fifo.in_fifo(), 0); + assert_eq!(pi.count("owe"), 0); + assert_eq!(pi.0.lock().iter().filter(|c| c.0 == "owe" && c.2 == -1).count(), 8); + assert_eq!(pi.count("repay"), 0); +} + +/// The processor interface's mark makes a store counted, once, and a +/// counted store with no link logs and returns nothing. +#[test] +fn the_processor_mark_counts_exactly_one_store() { + let (mut s, pi) = credit_board(); + + { + let mut f = s.front.lock(); + f.test_counted_stores = 0; + } + s.fifo.test_words.store(0, SeqCst); + s.mmio_write(consts::cmd::FIFO, 0, 8); + assert_eq!(pi.settled(), 0); + + let previous = pi_gfx_mark_store(true); + s.mmio_write(consts::cmd::FIFO, 0, 8); + pi_gfx_unmark_store(previous); + assert_eq!(pi.0.lock().as_slice(), &[("return", VPRO_WIDGET, 1)]); + + // One-shot: the mark was taken by the first store. + s.mmio_write(consts::cmd::FIFO, 0, 8); + assert_eq!(pi.settled(), 1); + + // The Octane: no link, so the credit is dropped with a log line. + s.front.lock().pi = None; + pi_gfx_mark_store(true); + s.mmio_write(consts::cmd::FIFO, 0, 8); + assert_eq!(pi.settled(), 1); + assert!(!pi_gfx_take_mark()); +} + +/// The register block's pure state (PiGfx). +#[test] +fn the_credit_block_starves_a_widget_that_runs_out_of_credits() { + let mut g = PiGfx::default(); + + g.write(PI_GFX_PAGE, PI_GFX_PAGE_ENABLE | 0x2_0000); + g.write(PI_GFX_CREDIT, 3); + g.write(PI_GFX_BIAS, 2); + assert!(g.covers(0, 0x2_1000)); + assert!(!g.covers(0, 0x4_0000)); + assert!(g.count_store(0, 0x2_0000, "t")); + assert!(!g.take_starved(0, "t")); + assert!(g.count_store(0, 0x2_0000, "t")); + assert!(g.take_starved(0, "t")); + assert!(!g.take_starved(0, "t")); + g.add_credits(0, 5, "t"); + assert_eq!(g.read(PI_GFX_CREDIT), 6); + assert!(!g.starved[0]); +} + + /// The model's low-water threshold, in waiting words (a model choice). const LOW_WATER_MARK: u32 = cfifo::LOW_WATER; const HOST_WRITE_BASE: u64 = 0xb70000; @@ -508,6 +692,8 @@ fn the_threshold_follows_6b2() { /// MODEL CONTRACT ONLY: the FIFO's size is unknown and the model's size is /// its own choice. Whatever `0x6b1` the set-up stores, the level clamps at /// cfifo::WORDS however many stores wait. +/// +/// QEMU: command-work-credit-depth-ignores-setup covers the credit depth. #[test] fn the_fifo_size_ignores_the_set_up_words() { for word_6b1 in [1u32, 256, 0x7fff_ffff] { @@ -555,35 +741,129 @@ fn low_water_is_raised_as_the_queue_falls_below_the_mark() { assert_eq!(rec.events(FLAG_VECTOR), [true]); } -/// A hurry ends the board thread's pause before a slice. The pause is set -/// far longer than the wait below, so only the hurry runs the stores in -/// time; the stores wait while the thread sits in it. +/// MODEL CONTRACT ONLY: the FIFO's depth is unknown and the model's depth is +/// its own choice. The stored `0x6b1` and `0x6b2` values (256 and 32192, as +/// the kernels write them) do not widen the credits: only the oldest +/// `cfifo::DEPTH` waiting stores, the model's depth, have their credits back, so the +/// processor interface still holds back a writer that runs ahead of the slow +/// board. +/// +/// QEMU: command-work-credit-depth-ignores-setup. +#[test] +fn the_fifo_set_up_does_not_widen_the_credit_depth() { + let (mut s, pi) = credit_board(); + s.fifo.test_words.store(0, SeqCst); + cmd64(&mut s, 0x6b1, 256); + cmd64(&mut s, 0x6b3, 24144); + cmd64(&mut s, 0x6b2, 32192); + finish(&mut s); + pi.0.lock().clear(); + + s.fifo.test_words.store(1, SeqCst); + let stores = cfifo::DEPTH + 8; + for _ in 0..stores { + s.mmio_write(consts::cmd::FIFO, 0, 8); + } + assert_eq!(s.fifo.stores(), stores); + assert_eq!(pi.count("return"), cfifo::DEPTH as i64); + finish(&mut s); + assert_eq!(pi.settled(), stores as i64); +} + +/// MODEL CHOICE: the FIFO holds far more than the host's credit window, so +/// a writer that fills the whole window, 256 doublewords on the IP30 and at most 256 on +/// the IP35, gets every credit back as its stores enter, with the board's +/// thread taking none of them, and owes nothing. A credit that waited on +/// the board's thread could miss the high-water handler's ten-microsecond +/// poll. +#[test] +fn the_credit_depth_is_deeper_than_the_host_window() { + const LARGEST_HOST_WINDOW: u32 = 256; + let (mut s, pi) = credit_board(); + for _ in 0..=LARGEST_HOST_WINDOW { + s.mmio_write(consts::cmd::FIFO, 0, 8); + } + assert_eq!(s.fifo.stores(), LARGEST_HOST_WINDOW + 1, "the board took none"); + assert_eq!(pi.count("return"), LARGEST_HOST_WINDOW as i64 + 1); + assert_eq!(pi.count("owe"), 0); + finish(&mut s); +} + +/// MODEL CHOICE: the credited depth is 16,384 stores, the smallest power of +/// two that holds the kernel's start-up program of about 12,291 words at +/// one word per store. With the board taking nothing, every one of the +/// first 16,384 counted stores is credited as it enters, and +/// the next is owed until the board takes a store. #[test] -fn a_hurry_ends_the_board_threads_pause() { - let (s, _rec) = board("v12"); +fn every_store_inside_the_credited_depth_is_credited_and_the_next_is_owed() { + const DEPTH: u32 = 16_384; + let (mut s, pi) = credit_board(); + for _ in 0..DEPTH { + s.mmio_write(consts::cmd::FIFO, 0, 8); + } + assert_eq!(s.fifo.stores(), DEPTH, "the board took none"); + assert_eq!(pi.count("return"), DEPTH as i64); + assert_eq!(pi.count("owe"), 0); + s.mmio_write(consts::cmd::FIFO, 0, 8); + assert_eq!(pi.count("return"), DEPTH as i64); + assert_eq!(pi.count("owe"), 1, "the store past the depth owes its credit"); + finish(&mut s); + assert_eq!(pi.settled(), DEPTH as i64 + 1); +} + +/// A counted store whose credit is owed has the board's thread run without +/// its pause, so the credit comes back as soon as the stores ahead retire. +/// A guest that watches its credit count reads a register the bus does not +/// refuse, so nothing else hurries the thread; with the pause the kernel saw +/// its count fall below the bias in a burst of a few hundred stores and gave +/// the FIFO up as stuck. The stores inside the model's depth are credited +/// at once and do not hurry. +#[test] +fn a_counted_store_owed_its_credit_hurries_the_board_thread() { + let (mut s, pi) = credit_board(); + for _ in 0..cfifo::DEPTH { + s.mmio_write(consts::cmd::FIFO, 0, 8); + } + assert!(!s.fifo.take_hurry(), "a credited store leaves the pause alone"); + s.mmio_write(consts::cmd::FIFO, 0, 8); + assert_eq!(pi.count("owe"), 1); + assert!(s.fifo.take_hurry(), "an owed credit hurries the board's thread"); + finish(&mut s); + + // On the board's thread, with a pause far longer than the wait below: + // only the hurry runs the burst in time. + let stores = cfifo::DEPTH + 136; let h = BoardLocks::new(s); h.fifo.test_gap_ms.store(60_000, SeqCst); + // credit_board holds the queue for stepping by hand. + h.fifo.test_words.store(0, SeqCst); + pi.0.lock().clear(); h.work_start(); - // One store starts the thread, and the burst waits until the thread is - // in its pause: a burst that lands before it would find the request - // already made and prove nothing about waking a sleeper. + // One credited store starts the thread, and the burst waits until the + // thread is in its pause: a burst that lands before it would find the + // request already made and prove nothing about waking a sleeper. h.write(consts::cmd::FIFO, 0, 8); let deadline = std::time::Instant::now() + std::time::Duration::from_secs(10); while h.fifo.test_parked.load(SeqCst) == 0 { assert!(std::time::Instant::now() < deadline, "the board's thread never reached its pause"); std::thread::yield_now(); } - for _ in 1..200 { + for _ in 1..stores { h.write(consts::cmd::FIFO, 0, 8); } - assert_eq!(h.fifo.stores(), 200, "the stores wait out the pause"); - h.fifo.hurry(); + // The last few stores may find the queue run dry and wait out a pause + // of their own, but inside the depth they are credited at the store. let deadline = std::time::Instant::now() + std::time::Duration::from_secs(10); - while h.fifo.stores() != 0 { - assert!(std::time::Instant::now() < deadline, "the board's thread kept its pause: {} stores wait", h.fifo.stores()); + while pi.settled() != stores as i64 { + assert!(std::time::Instant::now() < deadline, "the board's thread kept its pause: {} stores wait, {} of {stores} credits back", h.fifo.stores(), pi.settled()); std::thread::yield_now(); } + assert!(pi.count("owe") > 0, "the burst ran past the model's depth"); + assert_eq!(pi.count("repay"), pi.count("owe")); + // Cut a pause still under way short, so the stop does not wait it out. + h.fifo.hurry(); h.work_stop(); + assert_eq!(h.fifo.stores(), 0); } /// A stopping machine finishes the work waiting; a reset abandons it. @@ -628,7 +908,7 @@ fn two_processors_storing_to_the_fifo_lose_nothing_and_keep_each_ones_order() { let pushed = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { for seq in 0..PER { let size = if seq % 3 == 0 { 8 } else { 4 }; - while !h.fifo.push(&h.front, consts::cmd::FIFO, id << 32 | seq, size) { + while !h.fifo.push(&h.front, consts::cmd::FIFO, id << 32 | seq, size, false) { std::thread::yield_now(); } } @@ -676,7 +956,7 @@ fn a_store_run_before_its_push_returns_is_already_in_the_level() { let h = BoardLocks::new(s); h.front.lock().intr_flags &= !intr::FIFO_LOW_WATER; for seq in 0..20 { - assert!(h.fifo.push(&h.front, consts::cmd::FIFO, seq, 4)); + assert!(h.fifo.push(&h.front, consts::cmd::FIFO, seq, 4, false)); } fn run_everything(fifo: &CommandFifo, front: &Mutex) { let mut run = Run::new(); @@ -687,7 +967,7 @@ fn a_store_run_before_its_push_returns_is_already_in_the_level() { assert_eq!((fifo.stores(), fifo.level()), (0, 0), "the run took out what the level counted"); } *h.fifo.test_after_ring_push.lock() = Some(run_everything); - assert!(h.fifo.push(&h.front, consts::cmd::FIFO, 20, 8)); + assert!(h.fifo.push(&h.front, consts::cmd::FIFO, 20, 8, false)); assert!(h.fifo.test_after_ring_push.lock().is_none(), "the consumer ran inside the push"); assert_eq!((h.fifo.stores(), h.fifo.level()), (0, 0)); assert_ne!(h.front.lock().intr_flags & intr::FIFO_LOW_WATER, 0, "the level fell through the mark"); @@ -790,7 +1070,7 @@ fn two_processors_racing_for_the_last_place_in_the_ring_leave_one_store_in() { // The ring has room for one more store when it holds RING_STORES - 1. let fill = |h: &BoardLocks| { for _ in 0..RING_STORES - 1 { - assert!(h.fifo.push(&h.front, consts::cmd::FIFO, 0, 4)); + assert!(h.fifo.push(&h.front, consts::cmd::FIFO, 0, 4, false)); } }; fill(&h); @@ -805,7 +1085,7 @@ fn two_processors_racing_for_the_last_place_in_the_ring_leave_one_store_in() { for _ in 0..ROUNDS { start.wait(); // A panic in a push must fail the round, not leave the others waiting. - let pushed = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| h.fifo.push(&h.front, consts::cmd::FIFO, 0, 4))); + let pushed = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| h.fifo.push(&h.front, consts::cmd::FIFO, 0, 4, false))); match pushed { Ok(true) => { accepted.fetch_add(1, SeqCst); @@ -875,7 +1155,7 @@ fn processors_wait_for_each_other_and_the_board_thread_does_not_wait_for_them() // before the lock was held. Nothing is queued here, so queue through the // unlocked ring and see it drain. for w in &words[..4] { - assert!(h.fifo.push(&h.front, w.port, w.value, w.size as u32)); + assert!(h.fifo.push(&h.front, w.port, w.value, w.size as u32, false)); } let deadline = std::time::Instant::now() + std::time::Duration::from_secs(20); while h.fifo.pending() { @@ -922,11 +1202,14 @@ fn a_refused_command_push_retries_behind_the_board_thread() { let (h, _rec) = threaded_board(); let words = preamble_twice(); - start_long_slice(&h, &words, 1000); + // The slice holds the board's thread until the test releases it, so the + // ring stays full however long the host takes to get here. + start_long_slice(&h, &words, 600_000); while h.fifo.stores() < RING_STORES { assert!(h.bus_write(consts::cmd::FIFO, 0, 4)); } let stores = h.fifo.stores(); + let refused = h.fifo.test_refused.load(SeqCst); let done = Arc::new(std::sync::atomic::AtomicBool::new(false)); let store = { let (h, done) = (h.clone(), done.clone()); @@ -935,9 +1218,15 @@ fn a_refused_command_push_retries_behind_the_board_thread() { done.store(true, SeqCst); }) }; - std::thread::sleep(std::time::Duration::from_millis(150)); + // The store is refused and retries: wait until it has been. + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(20); + while h.fifo.test_refused.load(SeqCst) == refused { + assert!(std::time::Instant::now() < deadline, "the store was never refused"); + std::thread::yield_now(); + } assert!(!done.load(SeqCst), "the store was dropped or ran around the board's thread"); assert_eq!(h.fifo.stores(), stores); + h.fifo.test_release.store(true, SeqCst); store.join().unwrap(); assert!(h.fifo.level() <= cfifo::WORDS); h.fifo.test_words.store(0, SeqCst); diff --git a/src/dev/odyssey/tests/interrupts.rs b/src/dev/odyssey/tests/interrupts.rs index 6bbb67ca9..8f89604cf 100644 --- a/src/dev/odyssey/tests/interrupts.rs +++ b/src/dev/odyssey/tests/interrupts.rs @@ -437,3 +437,26 @@ fn a_held_flag_is_sent_with_the_source_enables_clear() { reg_write(&mut s, route::DEST_UPPER, HEART_UPPER); assert_eq!(rec.events(FLAG_VECTOR), [true]); } + +/// A board built for a machine whose fabric carries packets to any widget +/// hands them to that machine's sender, to the widget the routing names, +/// and never to HEART's line: the Fuel's widget 13 reaches Bedrock this way. +#[test] +fn a_packet_sender_takes_the_interrupt_to_the_routed_widget() { + let (mut s, rec) = board("v12"); + let seen = Arc::new(Mutex::new(Vec::new())); + let log = seen.clone(); + s.front.lock().packets = Some(Arc::new(move |widget, address, vector, level| { + log.lock().push((widget, address, vector, level)); + true + })); + s.mmio_write(ROUTE_DEST_UPPER, 1 << 25, 4); + s.mmio_write(ROUTE_DEST_LOWER, 0x1230, 4); + s.mmio_write(ROUTE_VECTORS, TEST_VECTOR as u64, 4); + s.mmio_write(INTR_ENABLE_SET, INTR_FIFO_LOW_WATER, 4); + assert_eq!(*seen.lock(), vec![(1, 0x1230, TEST_VECTOR as u8, true)]); + assert!(rec.silent(), "the packet did not go to HEART"); + + s.mmio_write(INTR_CLEAR, INTR_FIFO_LOW_WATER, 4); + assert_eq!(seen.lock().last(), Some(&(1, 0x1230, TEST_VECTOR as u8, false))); +} diff --git a/src/dev/odyssey/worker.rs b/src/dev/odyssey/worker.rs index 0bb30032c..8ac0a4677 100644 --- a/src/dev/odyssey/worker.rs +++ b/src/dev/odyssey/worker.rs @@ -24,7 +24,8 @@ pub(super) const ODYSSEY_WORK_THREAD: &str = "odyssey-work"; /// The pause before the board's thread runs a burst, so the stores of the /// burst land first and one run takes them together. A processor that /// waits on the thread cuts it short (`CommandFifo::hurry`): an access the -/// bus refused. +/// bus refused, or a counted store whose credit stays owed until the stores +/// ahead of it retire. pub(super) const GAP: Duration = Duration::from_micros(100); /// How much one run may do: for tests a count of stores. The budget is @@ -142,7 +143,11 @@ impl Board { // A run that takes as long as a long draw would. let sleep_ms = std::mem::take(&mut self.test_slice_sleep_ms); if sleep_ms != 0 { - std::thread::sleep(std::time::Duration::from_millis(sleep_ms as u64)); + // Until the time is up or a test releases the hold. + let end = std::time::Instant::now() + std::time::Duration::from_millis(sleep_ms as u64); + while std::time::Instant::now() < end && !self.fifo.test_release.load(SeqCst) { + std::thread::sleep(std::time::Duration::from_millis(1)); + } } } @@ -195,6 +200,6 @@ impl Board { /// Reset clears the FIFO and abandons the waiting stores. pub fn work_reset(&mut self) { - self.fifo.clear(); + self.fifo.clear(&self.front); } } From 7dedfddc079d450d106a7b8529c5c29e89521b3a Mon Sep 17 00:00:00 2001 From: James Braid Date: Fri, 30 Oct 2026 04:22:15 -0700 Subject: [PATCH 30/31] ip35: fit the VPro board in the Fuel's graphics slot Every Fuel has a VPro graphics slot: a Fuel IRIX inventory lists its board under xtalk/13 and OpenBSD finds Odyssey at crossbar widget 13. Build the board there when the machine is given a V12 or a V10, through the widget-offset constructor, and keep none, the serial machine, as the default. The XBridge now reports link 13 alive, the board's DMA enters Bedrock's inbound path as widget 13, and a link reset of port 13 resets it. Its interrupt packets cross the crossbar to Bedrock like the PCI heads'. The status bar's activity (SCSI, IOC3 and Ethernet) and the CPU's Compare ticks reach its LEDs as on the Octane. IRIX's VPro driver writes each processor interface's graphics flow-control registers at attach and tears the board down when the credit counter reads zero, so the interfaces keep those registers; they count no stores and return no credits, so no processor interface is linked to the board's credit accounting. With PROM 6.210 the board passes the graphics diagnostic, and IRIX 6.5.30 with the VPro driver lists Graphics board: V12 and draws the X login screen on it. Input comes from the PS/2 pair the Fuel already has. Configuration accepts the board on the Fuel too, see the config note below. Tests build a Fuel board with each board and check that the processor reaches the board at widget 13 and finds its link alive, that its interrupt packets set and clear a processor interface's pending bit, that a link reset of its port resets it, that a headless Fuel has nothing there, and that the processor interfaces keep their graphics registers. Config (config): A Fuel is reported with both the V12 and the V10, so take either with [graphics] board; the other boards stay refused, with a message that names the Fuel's choices, and none, the serial machine, stays the default. The doc comments for the two boards now say they fit the Fuel as well as the Octane. The Fuel's validation case now accepts both boards and refuses the V6. --- src/config.rs | 28 ++++++--- src/ip35/tests.rs | 115 +++++++++++++++++++++++++++++++++++ src/ip35_machine.rs | 144 +++++++++++++++++++++++++++++++++++++++++--- 3 files changed, 270 insertions(+), 17 deletions(-) diff --git a/src/config.rs b/src/config.rs index d8540d25d..6b55e623c 100644 --- a/src/config.rs +++ b/src/config.rs @@ -832,14 +832,14 @@ pub enum GraphicsBoard { /// VPro V8: BUZZ A, 128 MB. Octane only. #[clap(name = "v8", alias = "vpro:v8", alias = "vpro_v8")] V8, - /// VPro V10: BUZZ B, 32 MB. Octane only. It shipped only in the Octane2 - /// and needs Crossbow 1.4, so the machine takes the Octane2's Crossbow and - /// Bridge revisions. + /// VPro V10: BUZZ B, 32 MB. Octane and Fuel. On the Octane it shipped + /// only in the Octane2 and needs Crossbow 1.4, so the machine takes the + /// Octane2's Crossbow and Bridge revisions. #[clap(name = "v10", alias = "vpro:v10", alias = "vpro_v10")] V10, - /// VPro V12: BUZZ B, 128 MB. Octane only. It shipped only in the Octane2 - /// and needs Crossbow 1.4, so the machine takes the Octane2's Crossbow and - /// Bridge revisions. + /// VPro V12: BUZZ B, 128 MB. Octane and Fuel. On the Octane it shipped + /// only in the Octane2 and needs Crossbow 1.4, so the machine takes the + /// Octane2's Crossbow and Bridge revisions. #[clap(name = "v12", alias = "vpro:v12", alias = "vpro_v12")] V12, /// No graphics board: the machine runs on its serial console. Octane @@ -905,7 +905,7 @@ impl GraphicsBoard { pub fn supports(self, profile: MachineProfile) -> bool { match profile { MachineProfile::OctaneIp30 => self.is_impact() || self.is_vpro() || self == Self::None, - MachineProfile::FuelIp34 => self == Self::None, + MachineProfile::FuelIp34 => matches!(self, Self::V10 | Self::V12 | Self::None), MachineProfile::Indigo2Ip22 => !self.is_vpro() && self != Self::None, _ => !self.is_vpro() && !matches!(self, Self::Extreme | Self::None), } @@ -2445,6 +2445,8 @@ impl MachineConfig { if !board.supports(self.machine.profile) { return Err(if self.machine.profile == MachineProfile::OctaneIp30 { format!("graphics.board \"{name}\" is not modelled on the Octane (valid: solidimpact, highimpact, maximpact, v6, v8, v10, v12, none)") + } else if self.machine.profile == MachineProfile::FuelIp34 { + format!("graphics.board \"{name}\" is not a Fuel board (valid: v12, v10, none)") } else { format!("graphics.board \"{name}\" is only valid on the Octane") }); @@ -2527,8 +2529,8 @@ impl MachineConfig { if self.banks[..2] != [256, 256] { return Err("Fuel currently requires banks = [256, 256] (512 MiB)".into()); } - if board != GraphicsBoard::None { - return Err("Fuel currently supports graphics.board = \"none\" only".into()); + if !board.supports(MachineProfile::FuelIp34) { + return Err(format!("graphics.board \"{}\" is not a Fuel board (valid: v12, v10, none)", board.as_str())); } if let Some(text) = &self.fuel.l1_revision { crate::dev::l1::parse_revision(text).map_err(|e| format!("fuel.l1_revision: {e}"))?; @@ -4172,7 +4174,7 @@ mod fuel_input_tests { } #[test] - fn fuel_requires_its_fixed_memory_population_and_no_graphics() { + fn fuel_requires_its_fixed_memory_population_and_a_vpro_or_no_graphics() { let mut cfg = MachineConfig::default(); cfg.machine.profile = MachineProfile::FuelIp34; cfg.machine.cpu = CpuModel::R10000; @@ -4187,6 +4189,12 @@ mod fuel_input_tests { cfg.validate().unwrap(); cfg.graphics.board = Some(GraphicsBoard::Newport); assert!(cfg.validate().unwrap_err().contains("none")); + for board in [GraphicsBoard::V12, GraphicsBoard::V10] { + cfg.graphics.board = Some(board); + cfg.validate().unwrap(); + } + cfg.graphics.board = Some(GraphicsBoard::V6); + assert!(cfg.validate().unwrap_err().contains("not a Fuel board")); cfg.graphics.board = None; cfg.banks[1] = 0; assert!(cfg.validate().unwrap_err().contains("256, 256")); diff --git a/src/ip35/tests.rs b/src/ip35/tests.rs index 26583801b..94d1768c9 100644 --- a/src/ip35/tests.rs +++ b/src/ip35/tests.rs @@ -221,6 +221,7 @@ fn the_configured_station_address_reaches_the_l1_and_the_ethernet() { timekeeper: Arc::new(crate::dev::timekeeper::Timekeeper::fuel().unwrap()), ps2_input: true, usb_input: true, l1: Default::default(), + graphics: crate::config::GraphicsBoard::None, }).unwrap(); assert_eq!(board.l1.station_mac(), mac); assert_eq!(board.eth.mac(), mac); @@ -236,6 +237,7 @@ fn the_l1_identity_overrides_reach_the_controller() { station_mac: [8, 0, 0x69, 0x12, 0x34, 0x56], timekeeper: Arc::new(crate::dev::timekeeper::Timekeeper::fuel().unwrap()), ps2_input: true, usb_input: false, l1, + graphics: crate::config::GraphicsBoard::None, }).unwrap().l1.config(); let stock = build(L1Identity::default()); assert_eq!(stock.revision, Some([1, 48, 1])); @@ -603,6 +605,7 @@ fn the_fuel_plugs_a_usb_keyboard_and_mouse_only_when_configured_on() { timekeeper: Arc::new(crate::dev::timekeeper::Timekeeper::fuel().unwrap()), ps2_input: true, usb_input, l1: Default::default(), + graphics: crate::config::GraphicsBoard::None, }).unwrap(); // Power each port: one with a device shows it connected. board.usb.bar_write(0, 0x54, 0x100, 4); @@ -861,3 +864,115 @@ mod code_views { }); } } + +fn fuel_with_graphics(graphics: crate::config::GraphicsBoard) -> crate::ip35_machine::FuelBoard { + crate::ip35_machine::FuelBoard::with_config(crate::ip35_machine::FuelBoardConfig { + payload: &[], clock: Arc::new(|| 0), flash_path: None, + station_mac: [0x08, 0x00, 0x69, 0x21, 0x43, 0x65], + timekeeper: Arc::new(crate::dev::timekeeper::Timekeeper::fuel().unwrap()), + ps2_input: true, usb_input: true, + l1: Default::default(), + graphics, + }).unwrap() +} + +/// The VPro's link status in the crossbar, in its slot's data half. +const VPRO_LINK_STATUS: u64 = 0x100 + 5 * 0x40 + 0x10 + 4; +/// A processor access to widget 13 through Bedrock's small window. +const VPRO_PIO: u64 = 13 << 24; + +// The VPro cases follow tests/qtest/sn1-fuel-vpro-test.c in +// jamesbraid/qemu-sgi-origami: widget-identity, profile-identity-and-routing +// and headless-fuel-has-no-board. + +// VPro host-interface registers the tests below touch, as byte offsets in +// the board's widget space. The board's own constants are private to it. +const ODSY_IDENTITY: u64 = 0x00; +const ODSY_CONTROL: u64 = 0x20; +const ODSY_CONTROL_WIDGET_MASK: u32 = 0xf; +const ODSY_INT_DEST_LOWER: u64 = 0x38; +const ODSY_INTR_CLEAR: u64 = 0x101c; +const ODSY_INTR_ENABLE_SET: u64 = 0x1034; +const ODSY_INTR_FIFO_LOW_WATER: u32 = 0x200; +const ODSY_ROUTE_DEST_UPPER: u64 = 0x1044; +const ODSY_ROUTE_DEST_LOWER: u64 = 0x104c; +const ODSY_ROUTE_VECTORS: u64 = 0x1054; +const ODSY_ROUTE_DEST_WIDGET_SHIFT: u32 = 25; + +/// A Fuel IRIX inventory lists its VPro under xtalk/13: the processor must +/// reach the board there, learn its widget number from it, and find the +/// link alive in the crossbar. +#[test] +fn the_vpro_answers_at_widget_13_with_its_link_alive() { + for (graphics, revision) in [(crate::config::GraphicsBoard::V12, 0xa), (crate::config::GraphicsBoard::V10, 0xa)] { + let board = fuel_with_graphics(graphics); + let identity = board.io.pio_read(VPRO_PIO | (ODSY_IDENTITY + 4), 4); + assert_eq!(identity.status, BUS_OK); + assert_eq!(identity.data as u32, (revision << 28) | 0x0c01_3047, "{graphics:?}"); + let control = board.io.pio_read(VPRO_PIO | (ODSY_CONTROL + 4), 4).data as u32; + assert_eq!(control & ODSY_CONTROL_WIDGET_MASK, 13); + assert_ne!(board.xbridge.crossbar().read32(VPRO_LINK_STATUS).data & (1 << 31), 0); + } +} + +/// The board's interrupt packets cross the XBridge to Bedrock as the PCI +/// heads' do: a raised source sets its vector's pending bit in the +/// processor interface the destination names, and a cleared one clears it. +#[test] +fn vpro_interrupt_packets_set_and_clear_a_processor_interface_pending_bit() { + let board = fuel_with_graphics(crate::config::GraphicsBoard::V12); + // Register space (bit 47), the node's hub window and PI0's pending + // modify register, at the Bedrock's crossbar port 0xa. + let packet = (1u64 << 47) | 0x180_0000 | 0x90; + let vector = 5; + let write = |offset: u64, value: u64| { + assert_eq!(board.io.pio_write(VPRO_PIO | offset, 4, value), BUS_OK, "{offset:#x}"); + }; + write(ODSY_ROUTE_DEST_UPPER, (0xa << ODSY_ROUTE_DEST_WIDGET_SHIFT) | (packet >> 32)); + write(ODSY_ROUTE_DEST_LOWER, packet & 0xffff_ffff); + write(ODSY_ROUTE_VECTORS, vector); + write(ODSY_INTR_ENABLE_SET, ODSY_INTR_FIFO_LOW_WATER as u64); + let pending = || board.pi[0].read64(0x98).data & (1 << vector); + assert_ne!(pending(), 0); + write(ODSY_INTR_CLEAR, ODSY_INTR_FIFO_LOW_WATER as u64); + assert_eq!(pending(), 0); +} + +/// A link reset of the graphics port resets the board behind it, as the +/// PCI port's does for the functions behind its head. +#[test] +fn a_graphics_link_reset_resets_the_vpro() { + let board = fuel_with_graphics(crate::config::GraphicsBoard::V12); + let destination = VPRO_PIO | (ODSY_INT_DEST_LOWER + 4); + assert_eq!(board.io.pio_write(destination, 4, 0x55), BUS_OK); + assert_eq!(board.io.pio_read(destination, 4).data, 0x55); + let link_reset = 0x100 + 5 * 0x40 + 0x30 + 4; + assert_eq!(board.xbridge.crossbar().write32(link_reset, 1), BUS_OK); + assert_eq!(board.io.pio_read(destination, 4).data, 0); +} + +/// Without graphics nothing answers at widget 13 and its link is down, as +/// on a headless Fuel. +#[test] +fn a_fuel_without_graphics_has_no_widget_13() { + let board = fuel_with_graphics(crate::config::GraphicsBoard::None); + assert!(board.vpro.is_none()); + assert_ne!(board.io.pio_read(VPRO_PIO | 4, 4).status, BUS_OK); + assert_eq!(board.xbridge.crossbar().read32(VPRO_LINK_STATUS).data & (1 << 31), 0); +} + +/// IRIX's VPro driver writes each processor interface's graphics credit +/// counter at attach and tears the board down if it reads back zero; the +/// registers must keep what the guest writes, per interface. +#[test] +fn processor_interface_graphics_registers_keep_their_writes() { + let board = fuel_with_graphics(crate::config::GraphicsBoard::V12); + // Slot A's credit counter and bias; PI1 keeps its own block. + assert_eq!(board.pi[0].write64(0x308, 100), BUS_OK); + assert_eq!(board.pi[0].write64(0x310, 0x1000 | 68), BUS_OK); + assert_eq!(board.pi[0].read64(0x308).data, 100); + assert_eq!(board.pi[0].read64(0x310).data, 68); + assert_eq!(board.pi[1].read64(0x308).data, 0); + board.pi[0].power_on(); + assert_eq!(board.pi[0].read64(0x308).data, 0); +} diff --git a/src/ip35_machine.rs b/src/ip35_machine.rs index 0df55974f..1c75b2d45 100644 --- a/src/ip35_machine.rs +++ b/src/ip35_machine.rs @@ -4,8 +4,11 @@ //! Selected with `[machine] profile = "fuel_ip34"`. Shared keys select the //! PROM, NVRAM, memory and SCSI devices; `[fuel]` selects the serial console //! and writable flash. `ip35::Ip35Bus` maps the processor views over the -//! same Physical memory banks. Graphics is not attached, and the IOC3 has -//! no keyboard or mouse ports. +//! same Physical memory banks. The IOC3 carries PS/2 keyboard and mouse ports +//! (default) and the OHCI controller can take a USB pair; without graphics the +//! machine runs on a serial console. With `[graphics] board = "v12"` (or +//! `"v10"`) a VPro board answers at crossbar widget 13, the Fuel's graphics +//! slot. //! //! During PROM cache-as-memory bootstrap, cached startup bytes live in a //! Fuel cache adapter until a successful cached fetch leaves the PROM homes. @@ -18,10 +21,16 @@ //! at relative offsets 0 and 0x40 before DRAM discovery. Their hardware //! meaning is unknown; Bedrock logging records accesses to these windows. //! +//! The processor interfaces' graphics flow-control registers store what +//! IRIX writes, which is what its VPro driver checks, but count no stores: +//! no uncached store into the graphics page takes a credit and the board +//! returns none, so the credit-starvation interrupt never fires. The board's +//! DMA reaches memory through Bedrock's inbound path as widget 13. +//! //! Board composition ported from hw/mips/sgi/sn1-node.c //! in jamesbraid/qemu-sgi-origami. -use crate::config::{scsi_label, FuelConsole, MachineConfig}; +use crate::config::{scsi_label, FuelConsole, GraphicsBoard, MachineConfig}; use crate::cpu::mips_exec::{CpuDevice, MipsCpu, MipsCpuConfig, MipsExecutor}; use crate::cpu::mips_tlb::MipsTlb; use crate::dev::bedrock_flash::{Flash, PART_SIZE}; @@ -33,6 +42,7 @@ use crate::dev::bedrock_pi::{reg as pi_reg, BedrockPi, BedrockPiConfig, SIZE as use crate::dev::bedrock_xb::{BedrockXb, SIZE as XB_SIZE}; use crate::dev::heart::{host_clock, Clock}; use crate::dev::ioc3::{Ioc3, Ioc3Config}; +use crate::dev::odyssey::{Odyssey, PacketSender, PiGfx, PI_GFX_REGS}; use crate::dev::ioc3_eth::Ioc3Eth; use crate::dev::isp12160::Isp12160; use crate::dev::l1::{L1, L1Config, L1Event}; @@ -42,7 +52,7 @@ use crate::dev::prom_image; use crate::dev::ps2::Ps2Controller; use crate::dev::serial_host::{null_serial_backend, open_log, tcp_serial_backend, CiSerialBackend, LogMode, SerialBackend, TeeBackend}; use crate::dev::timekeeper::Timekeeper; -use crate::dev::xbridge::XBridge; +use crate::dev::xbridge::{InterruptPacket, XBridge}; use crate::devlog::{init_devlog, LogModule}; use crate::gdb_stub::CpuDebug; use crate::gfx_display::GfxDisplay; @@ -107,9 +117,12 @@ impl FuelMachine { revision: cfg.fuel.l1_revision.as_deref().map(crate::dev::l1::parse_revision).transpose()?, brick_serial: cfg.fuel.brick_serial.clone(), }, + graphics: cfg.graphics_board(), })?); let timers = Arc::new(TimerManager::new()); let mut exec = board.build_cpu(); + // The CPU's Compare matches blink the graphics status bar's LED. + exec.core.fasttick_count = board.fasttick.clone(); exec.core.set_timer_manager(timers.clone()); exec.core.set_count_hz( cfg.clock @@ -121,6 +134,8 @@ impl FuelMachine { let poll_board = board.clone(); exec.batch_hook = Some(Box::new(move |_| poll_board.poll())); let cpu: Arc = Arc::new(MipsCpu::<_, R14000FuelShadowCache>::new(*exec)); + board.status.set_cpu_cycles(cpu.cycles_ptr()); + board.status.set_count_hz(cpu.count_hz_atomic()); if let Some(space) = board.md.physical().ppmem_space() { use crate::ppmem::MappedMemory; unsafe { @@ -221,7 +236,8 @@ impl FuelMachine { board.eth.attach(link); } eprintln!( - "iris: Fuel IP34, R14000 500 MHz, 512 MiB, serial console {:?}", + "iris: Fuel IP34, R14000 500 MHz, 512 MiB, {}, serial console {:?}", + if board.vpro.is_some() { cfg.graphics_board().label() } else { "no graphics" }, cfg.fuel.console ); Ok(Self { @@ -367,6 +383,9 @@ impl Workstation for FuelMachine { self.board.eth.power_on(); self.board.scsi.power_on(); self.board.usb.power_on(); + if let Some(vpro) = &self.board.vpro { + vpro.power_on(); + } self.timekeeper.power_on(); self.restart_peripherals(); } @@ -403,7 +422,7 @@ impl Workstation for FuelMachine { self.lifecycle = state; } fn get_display(&self) -> Option> { - None + self.board.vpro.as_ref().map(|vpro| vpro.display()) } fn register_monitor_device(&self, device: Arc) { self.monitor.register_device(device); @@ -526,6 +545,15 @@ pub(crate) struct FuelBoard { pub eth: Arc, pub scsi: Arc, pub usb: Arc, + /// The VPro board in the graphics slot, widget 13. + pub vpro: Option>, + /// The status bar's activity bits and the CPU's Compare matches, which + /// the graphics board shows. + pub heartbeat: Arc, + pub fasttick: Arc, + /// What the graphics board's status bar shows: the activity bits, the + /// Compare matches and the CPU's cycle count and Count rate. + pub status: Arc, pub serial: [Arc; 3], #[cfg(test)] pub irq: Arc, @@ -546,6 +574,8 @@ pub(crate) struct FuelBoardConfig<'a> { pub usb_input: bool, /// Replacements for the L1's reported identity; unset keeps its defaults. pub l1: L1Identity, + /// `None`, `V12` or `V10`; validation refuses the others. + pub graphics: GraphicsBoard, } /// What a Fuel config may replace in the L1's identity replies. @@ -570,11 +600,12 @@ impl FuelBoard { ps2_input: true, usb_input: false, l1: L1Identity::default(), + graphics: GraphicsBoard::None, }) } pub fn with_config(config: FuelBoardConfig<'_>) -> Result { - let FuelBoardConfig { payload, clock, flash_path, station_mac, timekeeper, ps2_input, usb_input, l1: l1_identity } = config; + let FuelBoardConfig { payload, clock, flash_path, station_mac, timekeeper, ps2_input, usb_input, l1: l1_identity, graphics } = config; let md = BedrockMd::with_clock(clock.clone()); let phys = md.physical().clone(); let pi: [Arc; 2] = std::array::from_fn(|interface| { @@ -767,6 +798,39 @@ impl FuelBoard { Isp12160::new_bus(), Isp12160::new_bus(), )); + let heartbeat = Arc::new(AtomicU64::new(0)); + let fasttick = Arc::new(AtomicU64::new(0)); + scsi.set_heartbeat(heartbeat.clone()); + ioc3.set_heartbeat(heartbeat.clone()); + eth.set_heartbeat(heartbeat.clone()); + let status = crate::disp::StatusSource::new(heartbeat.clone(), fasttick.clone()); + let vpro = if graphics == GraphicsBoard::None { + None + } else { + // Interrupt packets leave through the crossbar as the heads' do; + // DMA enters Bedrock as this widget's inbound traffic. + let crossbar = Arc::downgrade(&xbridge); + let packets: PacketSender = Arc::new(move |widget, address, vector, level| { + crossbar.upgrade().is_some_and(|x| x.send_interrupt(InterruptPacket { + source_widget: WIDGET_VPRO, + widget, + address, + vector, + level, + })) + }); + let dma = Arc::new(WidgetDma { io: Arc::downgrade(&io), widget: WIDGET_VPRO }); + let board = Odyssey::new_widget_relative( + WIDGET_VPRO, graphics.as_str(), packets, dma, status.clone(), + )?; + xbridge.attach_widget(WIDGET_VPRO, board.clone()); + io.connect_widget(WIDGET_VPRO, xbridge.widget(WIDGET_VPRO).expect("attached widget")); + Some(board) + }; + // Each processor interface keeps its own graphics registers. + for pi in &pi { + pi.set_node_hooks(Arc::new(PiGraphics::default())); + } xbridge.attach(15, 1, scsi.clone()); xbridge.attach(15, 4, ioc3.clone()); let usb = Ohci::new(); @@ -791,6 +855,7 @@ impl FuelBoard { let reset_eth = eth.clone(); let reset_scsi = scsi.clone(); let reset_usb = usb.clone(); + let reset_vpro = vpro.as_ref().map(Arc::downgrade); xbridge.set_reset_output(Arc::new(move |port| { if port == 15 { reset_ioc3.power_on(); @@ -798,6 +863,12 @@ impl FuelBoard { reset_scsi.power_on(); reset_usb.power_on(); } + // A link reset of the graphics port resets the board behind it. + if port == WIDGET_VPRO { + if let Some(vpro) = reset_vpro.as_ref().and_then(|v| v.upgrade()) { + vpro.power_on(); + } + } })); let mut bus = Ip35Bus::new(phys.clone()); #[cfg(feature = "jitv2")] @@ -872,6 +943,10 @@ impl FuelBoard { eth, scsi, usb, + vpro, + heartbeat, + fasttick, + status, serial, #[cfg(test)] irq, @@ -919,6 +994,9 @@ impl FuelBoard { self.eth.start(); self.scsi.start(); self.usb.start(); + if let Some(vpro) = &self.vpro { + vpro.start(); + } } pub fn poll(&self) { if self.running.load(Ordering::Acquire) { @@ -938,6 +1016,9 @@ impl FuelBoard { self.running .store(false, Ordering::Release); self.md.stop_clock(); + if let Some(vpro) = &self.vpro { + vpro.stop(); + } self.usb.stop(); self.scsi.stop(); self.eth.stop(); @@ -949,6 +1030,55 @@ impl FuelBoard { } } +/// The crossbar widget of the Fuel's graphics slot: a Fuel IRIX inventory +/// lists its VPro under xtalk/13, and OpenBSD finds Odyssey at widget 13. +pub const WIDGET_VPRO: u8 = 13; + +/// One processor interface's graphics flow-control registers (0x300 to +/// 0x348): stored as written, with no store counting (see the module notes). +#[derive(Default)] +struct PiGraphics(Mutex); +impl crate::dev::bedrock_pi::NodeHooks for PiGraphics { + fn gfx_read(&self, index: usize) -> Option { + (index < PI_GFX_REGS as usize).then(|| self.0.lock().read(index as u32)) + } + fn gfx_write(&self, index: usize, value: u64) -> bool { + if index >= PI_GFX_REGS as usize { + return false; + } + self.0.lock().write(index as u32, value); + true + } + fn gfx_reset(&self) { + *self.0.lock() = PiGfx::default(); + } +} + +/// A widget's DMA as Bedrock's inbound path takes it: crosstalk addresses, +/// with the widget's inbound permissions applied. +struct WidgetDma { + io: std::sync::Weak, + widget: u8, +} +macro_rules! widget_dma_access { + ($r:ident,$w:ident,$ty:ty,$result:ident,$size:expr) => { + fn $r(&self, a: u64) -> $result { + let Some(io) = self.io.upgrade() else { return $result::err() }; + let r = io.inbound_read(self.widget, None, a, $size); + $result { data: r.data as $ty, status: r.status } + } + fn $w(&self, a: u64, v: $ty) -> u32 { + self.io.upgrade().map_or(BUS_ERR, |io| io.inbound_write(self.widget, None, a, $size, v as u64)) + } + }; +} +impl BusDevice for WidgetDma { + widget_dma_access!(read8, write8, u8, BusRead8, 1); + widget_dma_access!(read16, write16, u16, BusRead16, 2); + widget_dma_access!(read32, write32, u32, BusRead32, 4); + widget_dma_access!(read64, write64, u64, BusRead64, 8); +} + // XIO register packets reach only the two PI interrupt-modify registers. // The processor's register projection has a separate, wider interface. struct IoHub([Arc; 2]); From 0a6ed7d96da6101a6e467b22c996dcadcfe18068 Mon Sep 17 00:00:00 2001 From: James Braid Date: Fri, 30 Oct 2026 08:13:37 +0000 Subject: [PATCH 31/31] docs: describe the Fuel machine Document the supported serial configuration, image inputs, the host input options and the explicit limits for graphics, USB devices and snapshots, in the feature list, hacking notes, debug notes, the sample config and a Fuel page. Startup storage remains a PROM-specific compatibility approximation. --- CHANGELOG.md | 8 ++++ FEATURES.md | 33 +++++++++++---- HACKING.md | 59 +++++++++++++++++--------- debug.md | 9 +++- docs/fuel-ip34.md | 103 ++++++++++++++++++++++++++++++++++++++++++++++ fuel.toml.example | 44 ++++++++++++++++++++ 6 files changed, 228 insertions(+), 28 deletions(-) create mode 100644 docs/fuel-ip34.md create mode 100644 fuel.toml.example diff --git a/CHANGELOG.md b/CHANGELOG.md index 8e3c43f73..659073085 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -264,6 +264,14 @@ hashes are given where a change is easiest to understand by reading the commit. - A store-conditional on memory outside the window is now atomic with two CPUs. ### SGI Fuel (IP34 board, IP35 software platform) +- **The Fuel takes a VPro board**: `[graphics] board = "v12"` or `"v10"` + fits the Octane's VPro model at crossbar widget 13. IRIX 6.5.30 with the + VPro driver brings the X login screen up on it. + +- Added a headless Fuel profile with the 500 MHz R14000, 512 MiB RAM, + Bedrock, dual-head XBridge, IOC3 networking and two-channel ISP12160 SCSI. + The normal interpreter boot reaches the IRIX 6.5.30 root shell with PROM + 6.210. See the [Fuel guide](docs/fuel-ip34.md). - **jitv2 compiles uncached code from the uncached view.** The PROM runs from the reset window uncached; once memory covers that window the cached view returns RAM there, and the compile pool used to compile RAM's bytes in place of the diff --git a/FEATURES.md b/FEATURES.md index 487c73b05..0c7179342 100644 --- a/FEATURES.md +++ b/FEATURES.md @@ -170,15 +170,34 @@ the first `[section]`, because TOML tables continue until the next header. | Key | Default | Values and behavior | |---|---|---| -| `machine.profile` | `indy_ip24` | indy_ip24, indigo2_ip22, indigo2_ip28, o2_ip32 (configured by `[o2]`, see [the O2 guide](docs/o2-ip32.md)), octane_ip30 (configured by `[octane]`, see [the Octane guide](docs/octane-ip30.md)). | -| `machine.cpu` | `r4400` | r4400, r5000, r10000; CPU/ISA choice is independent of build features. An O2 takes r5000 or r10000 (an r4400 is refused when the machine is built); the Octane is r10000 only. | +| `machine.profile` | `indy_ip24` | indy_ip24, indigo2_ip22, indigo2_ip28, o2_ip32 (configured by `[o2]`, see [the O2 guide](docs/o2-ip32.md)), octane_ip30 (configured by `[octane]`, see [the Octane guide](docs/octane-ip30.md)), fuel_ip34 (see [the Fuel guide](docs/fuel-ip34.md)). | +| `machine.cpu` | `r4400` | r4400, r5000, r10000, r12000 (Octane, O2), r14000 (Octane, Fuel), r16000 (no machine yet). CPU/ISA choice is independent of build features. A Fuel always runs its one R14000: set `r14000` (`r10000`, the family, is also accepted); any other value is refused. | | `machine.cpus` | `1` | Processors fitted. 2 only on the Octane; see [its guide](docs/Octane/multiprocessor.md). | +### [fuel] + +Fuel-specific settings apply when `machine.profile = "fuel_ip34"`. + +| Key | Default | Values and behavior | +|---|---|---| +| `flash` | unset | Persistent 16 MiB boot-flash file. Unset, the flash is volatile. | +| `console` | `l1` | `l1`, `ioc3a` or `ioc3b`. Selects the guest console connection. | +| `l1_port` | `8882` | Host TCP port for the L1 console. Used when `console = "l1"`. | +| `l1_log` | unset | Optional serial log path for L1 output when an IOC3 port is selected as console. | +| `l1_revision` | `1.48.1` | L1 firmware revision as `major.minor.patch`, decimal. The guest reads it with L1 request `0x0005`. | +| `brick_serial` | `EMUN00` | L1 brick serial, at most 37 bytes. The guest reads it with L1 request `0x0004`. The default is a marked synthetic serial, not a real one. | + +The machine uses one 500 MHz R14000 and fixed 512 MiB RAM in two 256 MiB +banks. Its IOC3 has no keyboard or mouse ports, so it runs on a serial +console. A `[fuel]` table is written to a saved config only when one of +these keys is set. The sample configuration and supported PROM inputs are in the +[Fuel guide](docs/fuel-ip34.md). + ### [graphics] | Key | Default | Values and behavior | |---|---|---| -| `graphics.board` | `newport`; `solidimpact` on the Octane | newport, xz, extreme, solidimpact, highimpact, maximpact, v6, v8, v10, v12, none. Extreme is Indigo2 only. The Octane takes only `solidimpact` (SI), `highimpact` (SI with texture option), `maximpact` (MXI), `v6`, `v8`, `v10`, `v12` (the VPro boards, also `vpro:v12` and so on; need an IRIX install with the VPro driver, IRIX 6.5.10 or later and PROM 4.0, or 4.5 for the V10 and V12, which shipped only in the Octane2 and so give the machine the Octane2's Crossbow 1.4 and Bridge revision) or `none` (no board: the machine runs on its serial console, with no window). | +| `graphics.board` | `newport`; `solidimpact` on the Octane | newport, xz, extreme, solidimpact, highimpact, maximpact, v6, v8, v10, v12, none. Extreme is Indigo2 only. The Octane takes only `solidimpact` (SI), `highimpact` (SI with texture option), `maximpact` (MXI), `v6`, `v8`, `v10`, `v12` (the VPro boards, also `vpro:v12` and so on; need an IRIX install with the VPro driver, IRIX 6.5.10 or later and PROM 4.0, or 4.5 for the V10 and V12, which shipped only in the Octane2 and so give the machine the Octane2's Crossbow 1.4 and Bridge revision) or `none` (no board: the machine runs on its serial console, with no window). The Fuel takes `v12` or `v10` in its graphics slot, or `none`, its default. | | `graphics.heads` | `1` | 1 or 2; dual-head requires Newport. | | `graphics.resolution` | `guest` | guest, 1024x768, 1280x960, 1280x1024; presets require Newport. | @@ -209,7 +228,7 @@ An explicit non-disk kind takes precedence over the legacy cdrom flag. | Key | Default | Values and behavior | |---|---|---| -| `network.mode` | `nat` | nat, pcap, stream or off (no Ethernet; O2 and Octane only). Stream needs `network.stream`. PCAP needs build support and host capture permissions. PCAP works on every Ethernet; stream on every onboard one, not on a DaynaPort. | +| `network.mode` | `nat` | nat, pcap, stream or off (no Ethernet; O2, Octane and Fuel only). Stream needs `network.stream`. PCAP needs build support and host capture permissions. PCAP works on every Ethernet; stream on every onboard one, not on a DaynaPort. | | `network.stream` | `unset` | `tcp:IP:PORT` or `unix:PATH`: the server of the private segment for `mode = "stream"` (QEMU `-netdev stream` framing). See NETWORKING.md. | | `network.pcap_interface` | `auto-pick when unset` | Interface name or numeric list index; PCAP only. | | `network.nfs_pcap_ip` | `unset` | Virtual LAN IP for the built-in NFS responder in PCAP mode. | @@ -373,8 +392,8 @@ IP22/IP24 four-bank IRIX 6.5 behavior remains separate from IP28's verified two-bank 1 GB layout. Extreme requires IP22. All non-Newport boards require `heads=1` and `resolution="guest"`. IP28 IRIX needs IMPACT; its PROM has an embedded fallback. -The config validator permits SCSI controller 1 only on IP22, the O2 and the -Octane, the machines with two controllers. +The config validator permits SCSI controller 1 only on IP22, the O2, Octane +and Fuel. These machines have two controllers. Standalone CLI options update the loaded config where supplied; boolean enable switches do not provide a general way to clear every saved true setting. @@ -427,7 +446,7 @@ iris-gui does not parse this standalone option set; use its named-machine UI. | `--list-net-interfaces` | Print the host network interfaces libpcap can bridge onto, then exit. Requires a build with --features pcap. | | `--no-scsi-deferred-int` | Disable deferred SCSI status interrupts (default: enabled for OpenBSD/NetBSD compatibility). | | `--cpu` | Emulated CPU: `r4400` (default), `r5000`, or `r10000`. Overrides `[machine] cpu`. | -| `--graphics` | Graphics board: `newport` (default), `xz`, `extreme`, `solidimpact`, `highimpact`, `maximpact` (or `impact:solid`, `impact:high`, `impact:max`), `v6`, `v8`, `v10`, `v12` (`vpro:v12` and so on, Octane only). | +| `--graphics` | Graphics board: `newport` (default), `xz`, `extreme`, `solidimpact`, `highimpact`, `maximpact` (or `impact:solid`, `impact:high`, `impact:max`), `v6`, `v8`, `v10`, `v12` (`vpro:v12` and so on; Octane, and `v10`/`v12` on the Fuel). | | `--gdb-port` | Enable GDB stub on the given TCP port (e.g. --gdb-port 1234). Connect with: target remote localhost: | | `--monitor-port` | Monitor console port on 127.0.0.1 (default 8888). Give each iris running at once its own. | | `--vnc [ADDR]` | Serve the screen, keyboard and mouse over VNC on ADDR (`host:port` or a port; default 127.0.0.1:5900). Sets `vnc.listen`. | diff --git a/HACKING.md b/HACKING.md index bfce0fbab..67e0114a0 100644 --- a/HACKING.md +++ b/HACKING.md @@ -16,13 +16,18 @@ Other documents: ## 1. What IRIS is -IRIS emulates five SGI workstations: the Indy (IP24), the Indigo2 (IP22), the -Indigo2 IMPACT (IP28), the O2 (IP32) and the Octane (IP30). The CPU is an -R4400, R5000, R10000, R12000, R14000 or R16000 (two parts), chosen at runtime (the IP28 takes the R10000; -the Octane the R10000, R12000 or R14000; the O2 the R5000, R10000 or R12000). -Graphics is Newport (REX3), GR2 (XZ/Extreme) or IMPACT on the Indy and -Indigo2, GBE on the O2 and Solid IMPACT on the Octane. It boots the real PROMs -and IRIX 5.3 and 6.5 to a desktop with networking. +IRIS emulates the Indy (IP24), Indigo2 (IP22), Indigo2 IMPACT (IP28), O2 +(IP32), Octane (IP30) and Fuel (IP34 board, IP35 software platform). The CPU +is an R4400, R5000, R10000, R12000, R14000 or R16000 (two parts), chosen at runtime (the +IP28 takes the R10000; the Octane the R10000, R12000 or R14000; the O2 the +R5000, R10000 or R12000; the Fuel its 500 MHz R14000). Graphics is Newport +(REX3), GR2 (XZ/Extreme) or IMPACT on the Indy and Indigo2, GBE on the O2, and +Solid IMPACT on the Octane. Fuel has no graphics board and runs headless on +its serial console. It does have a PS/2 keyboard and mouse on its IOC3, fed +from the host's input and the default as in the Fuel's manual, and `[fuel] +usb_input = true` adds a USB keyboard and mouse on its OHCI controller; +`ps2_input = false` turns PS/2 off. The graphical machines boot their real +PROMs and IRIX 5.3 or 6.5 to a desktop with networking. It is **not cycle-accurate**, anywhere. IRIX is interrupt-driven and does not care how many cycles an instruction takes, so IRIS models behaviour @@ -83,17 +88,18 @@ image. A CD-ROM's node is a `ChangerNode`: the queue of discs and the open active one, with the `Removable` trait (present, changed, eject, load) separate from `BlockNode` so that removable media which is not a block device, a tape, can use it too. The WD33C93A (Indy, Indigo2), the Adaptec -7880 (O2) and the ISP1040 (Octane) each make an empty bus of their own kind -(`new_bus`); the machine fills it from config and gives it to the controller's -constructor, so a controller exists with its targets and keeps only its -registers and DMA. What a `BlockNode` must guarantee about durable writes, -and why a disk's `write_cache` setting matters, is in the "Disk write cache" -section of [STORAGE.md](STORAGE.md) and the `block_node.rs` module docs: read -them before writing a backend or changing a flush. `src/scsi_monitor.rs` implements the monitor's `scsi` and `cow` +7880 (O2) and ISP1040 (Octane) each make an empty bus of their own kind +(`new_bus`). Fuel uses an ISP12160 with two SCSI buses behind one PCI +function. The machine fills each bus from config and gives it to the controller +constructor, so the controller keeps its registers and DMA. What a `BlockNode` +must guarantee about durable writes, and why a disk's `write_cache` setting +matters, is in the "Disk write cache" section of [STORAGE.md](STORAGE.md) and +the `block_node.rs` module docs: read them before writing a backend or changing +a flush. `src/scsi_monitor.rs` implements the monitor's `scsi` and `cow` subcommands and the run-time disc changes once, over the bus a controller hands -out through `ScsiTargets::scsi_bus`; chip-specific subcommands (`regs`; the -WD's `wdt`, `defer`, `debug`) stay with the chip, and `DiscChanger` is -implemented once for every `ScsiTargets`. Configuration is shared the same +out through `ScsiTargets::scsi_bus`. Chip-specific subcommands stay with the +chip, including `regs` and the WD's `wdt`, `defer` and `debug`. `DiscChanger` +is implemented once for every `ScsiTargets`. Configuration is shared the same way: `scsi::attach_from_config` walks the `[scsi.N]` tables for every machine, where the key is a slot (controller × 16 + id, see `config::scsi_slot`), and the machine supplies only a route from controller number to bus. @@ -101,8 +107,9 @@ the machine supplies only a route from controller number to bus. `ScsiTarget::bus_reset` defaults to no action. Disk targets record reset as unit attention. Only the ISP12160 calls it; the other controllers keep their earlier behaviour of not reporting resets to targets. -The ISP12160 releases its controller state lock before walking either bus. -Target callbacks run under that bus lock and must not reacquire it. +The ISP12160 defers target resets and releases its controller state lock +before walking the bus. It calls targets while holding the bus lock, so a +target callback must not take that lock again. Flash command engines share `FlashBacking` file storage while retaining their own command state. @@ -310,9 +317,12 @@ Every device that has real work to do runs on its own thread: | `VNC-Grab`, `VNC-Pointer`, `VNC-Client`/`VNC-Read` | with `--vnc`: frame diffing, pointer steering, one pair per viewer | | `machine-events` | power-off and reset requests from devices | | `ISP1040-0`, `ISP1040-1` | Octane SCSI request rings and disk transfers | -| `IOC3-RX-A`, `IOC3-RX-B`, `IOC3-NVRAM` | Octane serial input per port; the DS1685 NVRAM file writer (IOC3's other deadlines are `hptimer` one-shots) | +| `ISP12160-0` | Fuel SCSI request ring for both ISP12160 channels | +| `IOC3-RX-A`, `IOC3-RX-B`, `IOC3-NVRAM` | Octane and Fuel IOC3 serial input and NVRAM writer (IOC3's other deadlines are `hptimer` one-shots) | +| `l1-serial` | Fuel L1 console UART worker. Its other serial ports are IOC3 A and B | | `ip30-eth-rx`, `ip30-eth-net` | Octane ethernet receive pump; the network backend | | `odyssey-work`, `odyssey-raster`, `odyssey-display` | VPro: the front end decodes the queued command-FIFO stores (the processor's stores only queue), the raster unit draws what the front end sets up, and the display copies the planes out, counts the retrace and presents the window at 60 Hz. No board-wide lock: the front end owns its state, an access that needs the drawing is answered busy until it is done, and the display reads the planes while they are drawn | +| `ioc3-eth-rx`, `ioc3-eth-net` | Fuel IOC3 Ethernet receive pump and network backend | Threads that have nothing to do park instead of spinning: the graphics consumers when their FIFO is empty, and the CPU thread in `idle-pause` builds @@ -611,6 +621,15 @@ introspection commands are in [debug.md](debug.md#jit-v2). Crossbow and the Bridge with IOC3 and two ISP1040 SCSI controllers, an R10000, and IMPACT graphics (see the [Octane guide](docs/octane-ip30.md) for the boards). +- **IP34 board, IP35 software platform (Fuel)**: one 500 MHz R14000, 512 MiB + in two 256 MiB banks, Bedrock, two XBridge heads, IOC3 and a two-channel + ISP12160. It runs headless on the L1 or IOC3 serial console. The IOC3 uses + the shared `net::Link` for NAT or PCAP. The writable Bedrock flash and + serial-flash devices share `FlashBacking` storage, with separate command + handling. A VPro board (the Octane's Odyssey model) can sit at crossbar + widget 13; its packets and DMA go through the XBridge and Bedrock's inbound + path, and the PI graphics registers are stored but count no credits. + Snapshots are unsupported. See the [Fuel guide](docs/fuel-ip34.md). The Indy and Indigo2 profiles each have an embedded PROM fallback; `src/platform.rs` and `src/machine.rs` wire their differences, and diff --git a/debug.md b/debug.md index f444d05c4..d75d0875c 100644 --- a/debug.md +++ b/debug.md @@ -367,6 +367,13 @@ widget's link, refused Crossbow accesses, and accesses to physical addresses nothing is mapped at, with the PC; a device's own refusal is not listed). The `phys` commands work on the Octane as well. +On the Fuel the one controller is a two-channel ISP12160: `scsi` (or +`scsi0`) and `cow` act on channel 0, `scsi1` and `cow1` on channel 1, and +ids run 0 to 15 on each (`[scsi.17]` is id 1 on channel 1). `scsi notes` +shows the whole chip's command total and anything odd the driver did, and +has no `scsi1` form; `regs`, `wdt`, `defer`, `debug` and `dayna` are not +there. + Queue position `[0]` is the mounted disc, `[1]` the next on eject. A disc change reaches IRIX as a Unit Attention ("medium changed") on its next TEST UNIT READY; IRIX's `mediad` may still need a manual remount @@ -522,7 +529,7 @@ a forced termination; a normal stop saves them automatically. |---|---| | `phys mem [n]` / `mm` | Dump physical memory | | `phys dis [n]` / `md` | Disassemble physical memory | -| `phys map` | The decode table: pages by kind (untouched, filled but not yet mapped, shared, private), memory in use, slots faulted in, and the table as runs of devices. Every machine on `Physical` has the `phys` commands (Indy, Indigo2, O2, Octane) | +| `phys map` | The decode table: pages by kind (untouched, filled but not yet mapped, shared, private), memory in use, slots faulted in, and the table as runs of devices. Every machine on `Physical` has the `phys` commands (Indy, Indigo2, O2, Octane, Fuel) | | `phys trace ` / `trace ` | Log physical bus accesses | | `phys error ` | Stop on bus errors | | `phys hole ` | Stop on accesses to unmapped addresses | diff --git a/docs/fuel-ip34.md b/docs/fuel-ip34.md new file mode 100644 index 000000000..90d6d677e --- /dev/null +++ b/docs/fuel-ip34.md @@ -0,0 +1,103 @@ +# SGI Fuel (IP34 board, IP35 software platform) + +Fuel is the IP34 board running the IP35 software platform. The emulated +configuration has one 500 MHz R14000 and 512 MiB of RAM in two fixed 256 MiB +banks. It includes Bedrock, two XBridge heads, an IOC3, and an ISP12160 with +two SCSI channels. The machine runs headless on its L1 or IOC3 serial console. + +## Configuration + +Copy [`fuel.toml.example`](../fuel.toml.example) to `fuel.toml`, provide the +PROM and disk files named there, then run `iris --config fuel.toml` from the +repository root. The sample selects `machine.profile = "fuel_ip34"` and +`machine.cpu = "r10000"`. The machine always runs its one R14000; `cpu` +must name the R10000 family, and any other value is refused. The sample +sets `banks = [256, 256]`, `headless = true`, `no_audio = true`, serial +ports 18880 and 18881, and `graphics.board = "none"`. The `[fuel]` settings +select the flash backing file, console, and optional L1 port and log. The +console options are `l1`, `ioc3a` and `ioc3b`. The Fuel's IOC3 has no +keyboard or mouse ports, so the console is always serial. + +The L1 answers firmware revision requests with 1.48.1 and brick serial +requests with the synthetic `EMUN00`; both are model choices, not values read +from a Fuel. `[fuel] l1_revision = "major.minor.patch"` and +`[fuel] brick_serial = "..."` replace them. Its system serial request returns +the machine's Ethernet address (`[network] mac`) as twelve uppercase +hexadecimal digits, the text the PROM converts back into that address. +IRIX's `numastatd` reads the revision at startup; without one it logs +`SNDRV_GET_ELSC_VERSION: I/O error`. + +The PROM decoder accepts a raw payload or a supported version-six, +single-segment stored or gzip container. The sample attaches a raw IRIX disk +at SCSI ID 1 and writes guest changes to its overlay. `[network] mode` defaults +to `nat`. `off` disconnects the link. `pcap` uses the shared `net::Link` backend +with a PCAP-enabled build and host capture support. The example sets `off`. +Set it to `nat` for the built-in gateway. + +## Observed boot and limits + +With the normal interpreter configuration, PROM 6.210 reports +`ALL DIAGS PASSED` and IRIX 6.5.30 has reached the root shell. The guest +reports a 500 MHz R14000 with 512 MiB, detects `ef0`, and passes the observed +NAT networking checks. During initial boot, IRIX's file indexer can keep the +guest busy while the serial console shows little activity. + +The PROM uses the R10000-family primary data cache as temporary memory before +it configures DRAM: it tags cache lines for the 32 KiB window +`0x100000`-`0x107fff` and installs them without a fill. The memory controller +models this from the cache's tag stores. PROM 6.210 stores, in order, test +patterns, an invalidate sweep, a valid tag (state 0xc0 or 0xc1) for each of the +window's eight pages, then an invalidate sweep. A valid tag for a page of the +window starts a private zero-filled page that answers the window on every path +(the cached path, uncached accesses and DMA), and keeps that 64 MiB region out +of the direct window; the first invalid tag ends it, and the window is memory +again. Power-on clears the page. This is the same mechanism the O2 uses for its +POST stack. Two things about it are approximated, not traced: the end is the +first invalid tag of the final sweep, not the last, which is right only if the +PROM does not use its stack between them; and the page answers uncached and DMA +accesses too, where the hardware has nothing behind the window to answer them. +A different PROM revision could tag another window or in another order. + +The ordinary interpreter configuration is the one with recorded Fuel boot +evidence. `lightning` and experimental `jitv2` builds are available, but no +Fuel guest boot with either is claimed here. + +## Graphics + +`[graphics] board = "v12"` (or `"v10"`) fits a VPro board in the graphics +slot, crossbar widget 13: a Fuel IRIX inventory lists its board under +xtalk/13, and Fuels have been reported with both the V12 and the V10. It is +the Octane's VPro model. Its interrupt packets cross the XBridge to Bedrock +like the PCI heads', its DMA enters Bedrock's inbound path as widget 13, and +a link reset of port 13 resets it. The processor interfaces' graphics +flow-control registers keep what IRIX writes but count no stores, so the +credit-starvation interrupt never fires. + +With PROM 6.210 the graphics diagnostic finds the board and passes. An IRIX +6.5.30 disk whose kernel includes the VPro driver lists `Graphics board: V12` +in `hinv` and brings the X login screen up on it. A disk installed for a +machine without graphics has no VPro driver in its kernel and runs as before. + +## Input + +The IOC3 carries a PS/2 keyboard and mouse, fed from host input; they are the +default, as in the Fuel's manual. `[fuel] usb_input = true` also plugs a USB +keyboard and mouse into the OHCI controller, and `ps2_input = false` leaves the +PS/2 pair out. Machine snapshots are unsupported. + +## Source basis + +The literal `0x53474972756c6573` is named `IIO_PROTECT_OVRRD_KEY` in public +Linux revision `db2ddb87143519e20a95aa36c60b36107b736a58`, +`arch/mips/include/asm/sn/sn0/hubio.h`. This establishes the literal's public +provenance, not the mapping of each Fuel register offset that uses it. Those +offsets remain approximate. The board composition port follows the project's +BSD-licensed QEMU SGI model in `hw/mips/sgi/sn1-node.c`. That source is a BSD-licensed implementation +reference, not proof of hardware behavior. + +The Hub-special head windows are compatibility approximations. Reads return +zero and writes are acknowledged without side effects. PROM 6.210 writes the +64-bit word `0x2000` at relative offsets `0x0` and `0x40` before discovering +DRAM; their hardware meaning remains unknown. Bedrock logging records +accesses. This keeps the existing model behavior explicit rather than +claiming a register implementation. diff --git a/fuel.toml.example b/fuel.toml.example new file mode 100644 index 000000000..1c1325694 --- /dev/null +++ b/fuel.toml.example @@ -0,0 +1,44 @@ +# SGI Fuel (IP34 board, IP35 software platform), serial-only setup. +# PROM 6.210 and IRIX 6.5.30 have been observed reaching the root shell. +# Copy to fuel.toml and run from the repository root: iris --config fuel.toml +# Create ignore/fuel/ and supply your decoded raw PROM payload and IRIX disk. +# PROM also accepts supported version-six single-segment stored/gzip containers. +# No PROM or disk images are included. Convert QCOW2 disks to raw first. + +prom = "ignore/fuel/ip35prom.payload.bin" +nvram = "ignore/fuel/nvram.bin" +banks = [256, 256] +headless = true +no_audio = true +monitor_port = 18888 +serial_port_a = 18880 +serial_port_b = 18881 + +[machine] +profile = "fuel_ip34" +# Required: names the R10000 family. The machine always runs its one +# R14000 at 500 MHz with a 2 MiB secondary cache; other values are refused. +cpu = "r10000" + +[graphics] +# "v12" or "v10" fits a VPro in the graphics slot; IRIX needs a kernel with +# the VPro driver. The console stays on serial either way. +board = "none" + +[fuel] +flash = "ignore/fuel/flash.bin" +console = "l1" +l1_port = 18882 + +[network] +# Disconnected cable. For NAT, use mode = "nat". +# Keep the installed disk's working ef0 address and default route. +# NAT can adopt its static gateway before routing begins. +# Initial defaults: guest 192.168.0.2/24, gateway 192.168.0.1. +mode = "off" + +[scsi.1] +path = "ignore/fuel/irix.raw" +cdrom = false +# Keep the base disk unchanged. Guest writes go to irix.raw.overlay. +overlay = true

`cq>$hJOM+4*i1Ri&3aC8pRzsACocs4Dc!VtvwxUqflcnM&AII@p#^F ze-vtL#qdkOtI+2RSEEp4CWaZm8XqzITT!U75W`!EW0aHfvfljt82&$|>Pk~eGbqDM3kJ0Z2 z+5d?AFGBww{29Yv0f*pIhR=eZOws=}3NW#rrXQt0#X5~FYO@e;%IkH$+3(;pfaG0c3baS+4f;3o3V7``J4HTGe6j`)W7 zL%-qo`gn)og(%b*hv5%Jp~g52v;WZ;hhpjPQ?%EwSA~bUcbE2e;j=#GVdYOo*Iy%a z>#xE;gYzl;HTcE;Cyf6b^kRzsB)B_;e-wop8!`T;pwr)8{uLeI^(ov5UYo+t0QaUa z^Pk31RNfWzm&Q#DGrwu9#PH2gsId{ll_=DBh~Zz4!b6D96b%nYp~gH6k42%zGz`BP zeL@PDzh1$7rg03TzuU(w41WMz6Mn-7Dc_$e--JGvqBCBusif%4XByM6@_$7vLhN(Q z@JSz=F#L~Es4)n`kAfeE-}X~id?N}qwqW#?DAd@2VdguH6&U`HDAf3X;v)W}@c_l* zU;i+N{VI%4p0A4WGo<=gk@@auNc}c{71{3`ZGvt6D84WXH3nevZ;3*E<~RJS;1&37 zzAv)==rg?0ncwu8J<;b&r{P~VeHdTgcK-EDH@_5j^S+PRZ=}aP9~S4>zb}I&$i)0p zq`ywh0lVRs_B`mH$Ds>md|fwb{8U+Fzj1a1JZSQn&+>J!m;X;U|;|Dqj24lVUdshmpxrD<7aEd=#0xXfyD=k$QRrLcKZXv*CSu#6cd%a` z6FU6ZZ_RzY7c&QrjGy)W+n6@i?+EgnDg5!MAH)Td$on+!x5#|s%P+%U0egM0_d2n+ z@MEtv--@1hP0g^BXT4bWIsDbP@85a8mhn+EeV(HIec)lmodx1E5&S0%Url`?&+OTW zKZCfa50_i>Ny8W62Y2G%IpLqO@&o?88pE&TeKBcI*)aAlh<^?lX8*me0v<5D9eZs6 z`}(I?&qco3x0Cf--&ZsK_Yn{2hh8y^z1K^7tA>w8;g^oR-}{pUmg_MtyInP0A|0KGmx!9L4i8!!C3N%(^X*xJ|C ziM@wY_}bR~?Ro+B5PNz4E%cw*)B3mT705dcHvj3A@o@||ss7ycS{q+d9{=pZzfT^A zZuabA{?K>5v^`yQ=DT`|j(?}eYuEk${VU^NqI?s6>))=U%C^Qo^C!WKr>@W7 zk1LQjSowQf{kQ9j{=G?~pTVC*pGm`K(Fgzb_U`((lzjZZ>jLG4zgPLY^U#li)qWbn z+)bV&X^6bKIzeXFsAN_8WHk3us%E$*ncbeqTf&+P>VME2Pu>vrKlm(2pIew@CAw6{ zjhkG49|0G#He3#)eD8D#gPJC(xra&aqfau_Y`}D?;pIZKsqDWvy-#jjTA#}-^zSzN z9JM5=EJ>i_E|?%;gd{P2!aaT&|!`u=6RT-VEuquQ*pKod%@_h_k9B z@hi#xa%pheTq2lDMU!+H&3ubH!&zS}39$&H9`zi5t;rJp59PcF&HlArhz_V&y%%P-?g zlDqgKyR0SmNRrT83X-&2-@Z>$r68y~aO2-?_qlONmK;)*%T*=`sj2iW%d0{?pES13(r{Q6J$p&DCBGpckycnsLFh>`AHIj@vWTdj| zEF%mj_5$Jzzc@LJ8|o09yx`cu;?xnBP^Itk$B9{+T;7w*?AWY%KXDS0q=&c^l_a+* zl?;^R$fT03lC+j2dCM|qELkR%pwc#YENdo}sS_vExco1d6+^;STQQ%2lq7U%K2AD& zrLBl%L&a%PNuHF|=zfTEA>tI2)^rc^K1=+FQ%>TX3(LW1$!Z}{MT>Eh&Sr5bAwGpA z&Sr7HYNCis_t0DSt0k(y>h98+Dwr z>B(Y*_}~F4tf~I8({d+Ms1)`pdZLlAx13Nky3-lVX`}O_2yy$fou-16(L-t6ojYI0 zjwE91=g#`E-o11D4)%QWhw#TDXgN6xL`262_PFCx{DDi_0@o}A;kSB7 z2YUcnFgbh2DZH1{FU$qwbA*H-hdqdFG>K0T(xUO>&x z)ppxTU%U5kRFs1^V(s+w9~6ntAECG6(G*_d%sf+uqXLkn3Fnmr8-{H>;ndy#T*iV*)F z*E)FR!ItNHy9->-6~JBYp_jYs`4}Hx5|s}ArpX_F@!00B8~7_ko?n-nL55GTE^-w8 zn>JN%9y`1pj9++WZ4LF4Opzvc(OgS@2u+Fs%PP>a0$8~LgCtSh5`c-D5; zL#^_rg-80xUn0wYh+o*1-$RVM_c*A`=OQ1GC--ak5J#{pe;?1{-SV5iM3!8H8`6A> NUjYB55bSb%{|ofM{5=2w literal 308876 zcmeFae|%KcnLqyAdnc0#1h@$U3_^7-bMIsjkU>L@c0Tqx2}3}wLBUOb>~@+2h}E`n z>vq$%pYmP(53O8|EV5BH$w6BH$w6 zBH$w6BH$w6BH$w6BH$w6BH$w6BH$w6BH$w6BH$w6BH$w6BH$w6BH$w6BH$w6BH$w6 zBH$w6BH$w6BH$w6BH$w6BH$w6BH$w6BH$w6BH$w6BH$w6BH$w6BH$w6BH$w6BH$w6 zBH$w6BH$w6BH$w6BH$w6BH$w6BH$w6BH$w6BH$w6BH$w6BH$w6BH$w6BH$w6BH$w6 zBH$w6BH$w6BH$w6BH$w6BH$w6BH$w6BH$w6BH$w6BH$w6BH$w6BH$w6BH$w6BH$w6 zBH$w6BH$w6BH$w6BH$w6BH$w6BH$w6BH$w6BH$w6BH$w6BH$w6BH$w6BH$w6BH$w6 zBH$w6BH$w6BH$w6BH$w6BH$w6BH$w6BH$w6BH$w6BH$w6BH$w6BH$w6BH$w6BH$w6 zBH$w6BH$w6BH$w6BH$w6BH$w6BH$w6BH$w6BH$w6BH$w6BH$w6BH$w6BH$w6BH$w6 zBH$w6BH$w6BH$w6BH$w6BH$w6BH$w6BH$w6BH$w6BH$w6BH$w6BH$w6BH$w6BH$w6 zBH$w6BH$w6BH$w6BH$w6BH$w6BH$w6BH$w6BH$w6BH$w6BH$w6BH$w6BH$w6BH$w6 zBH$w6BH$w6BH$w6BH$w6BH$w6BH$w6BJlq&0-iFGm;z{Q&nMpG^ zJT?(wH{n^@5i&XhDe0@t+J=XTc7C16*hwD5d8XZLzgL*{vA-jFtOfBmk&3v{g1A;G zNj~xJtE9y~8a6wU{3<@C)tq~Fk!fur-DlfoM?JpRityN{IzknX1yU7{*~3a|#E8xA z&?;z_FUA^8HxH~XXVnX=OsuFENlzj3{w zW68mr#WQiRC0Bkk6;;Kkxatb>24i$r(1DBmaAYb}sDovyB(rd_{{wf91H`45M6Y1$@ny(I%tQV1rzhJZmX;xE2wZ_9@9+Igh zsL#KXB*aBgMy21PBBF>v^0c;SoD|MaOCp}w#9yX>cM9pM(n*Tp&%|F7C6Sh*h90Mk zK7piI$!9ZYm&dr72BN!30>*?g&yUswiDbXb&vc4vKt~wQmB2*?9v6`OnKrat0xlAH zf}nDmMMr`jTBhphtyGiVOe;IrlG)inP5K_BIZ~yePAQErb%4r%MU%dbJpO0w!;+;M z8yYB6P?y4WfuD{<8Kq^N!5;qeLMw=r#wFPx`JxwOi%5*%siU=5} z{>}FL%xj&E2b&6;Nja=8RmK6UWPxM42B|o?ax=WFL*jJfJ~*mm>TfpDBmX zZtl~H$4DLLn`c$#xwLtpJKx_q9vpuMujma2y3=J%>0irb~Exw{Jn~0 z!|H>#^7qL;&wc5JgG~+&uOGpzHXb;oDB(QDZ`Z(SHDJGMB(iNgCPe(LXELr+t zdlci3=VL84;R1VS`+*aSmqS*x1+`eowr%uJQ9U*>MT_T-OiF(jJcYJ$U(>Tuqq7h5 z-!hDU9~BvxTg!;f!`u?zfHFk6NXzAL8JxE;-^yk73C?r3d@w0+S$klFpJ?~gbn;*; z_$CmOi~dv@|4-5GL+H2Bh67fPmx3`FGXJnTPc`ik5$=t~r`ROVk`Xs07uUgIDC6q+)ihyck|fkSf%_b^E=@Na=;X5VG<2>5hHd0W zzj!uogBV)$UHU0dQ`wbpLlZpNkQfpAOE0J~4r+t28MXFrWh?tZrWV7!m$F`19efw9K?mBEPDyMVU{DJSDUy z=+LsXPNU&~fq2B#&MQOtJ8Pl)A|A587`m5@c59S~YAK#OQ1{Ay`xJj?9&lQNf7b`X z=mUk@;ZIT}x51yG3DF~@6sEfetrGO3J>0I;5!>c8e#?G`wwPK$yUMs-9L9BQKtAZh zDz_tj#2(o&!9J01bNY4r1l~_F9{7F|-$zx;{v*DR7PPrcw0Q^Sz#GxNr6_l--P~6D zot-64yMd{Xwp3|_baP)gnAp#8oWNvzMC*DD?Lk|(P5vGBXljjZpO%mNmGzVq`ZR+% zbiFxm8u46;e0!jO73K8$*8*Bs69N+fjl=y<+V7?Co-b>HPd>tVP{t&>GYVcNPgfh> z?{s7bfA89YGNCuB)tG;`ktFg~Ll?uxDaX;3_+7|<=kMQtoxbLLdi{%nSlwFcuLRhwE0xZL4|E|38& zzz2E@+Lh_1qPeW&P9b1NcLuetZJ@gyd_0zHch{o+|j>UGRNZuDtq!u;zmANl@?1`#&Pp zeuR9O`-yb`vN{Z_9g(Rn17oR#$TBeh4un+<`>3i}H9}W-n|Us` z|3FFAki*+=7-tWssu@tQ`(-3Y`aj1qT_Y z;dUKK)33MR=P+egm!?m(ZN{?=c;67D3Vsi(wmp5S{ciX=+Xk-nhkzNdtp&Cwu&q+S zXTUbexaoheZS^qMgSMT~KhvX5J!dNeX%|dDy<85h&*K;<7r6324Gz|r8y;kAIjB79 zj5DUMe5$>l=^LXxYQM+f{Eo6OYO6szqRu;~LpdW{27mCqgK5X}PCT3NY{zo~_;3d6 zG`wcOe9yYa4DgI^ewzmhE9%C)h4q)oa~Im=<2e<&&M~y(4YWgb;^sVF5I3BU<9azg zza!6He#h!p<~-5%DU5Tzyq62}a59z;oqew#0}o7sZY0Hue)(8sm;+4!@a(22D9aYug^Wi-5ggwu1mxjJOdldf^r9URA>Ts*e}#uuRvjklv+ zw8I;dr9~{S3){hAdVB%K1!TWzpW?Fev=x-gZOhkNias5)35oWiK0#~s>&J3*42v~2 zWCiyz@?w6*_%TmH7GZ3GAIfFu)gEAubp-h8RvBx?&cg!3BJQt?QHf}09eMb@K<=Px zgq*sS^E!E0NB*S=88azejXom#IH%!HAstME)eatoe!*pOJo7ZmMX(|HEqnCLx{!Y_ zidG zcJkPxEDtyybw?l%f{+IR$O9E^&aW+_|A%zM3h0>rF?3A-SUNx_Fw}rcFjoA>OPY>C zHW$aLa!(k!e+9>|lF z{T7))qYdA47}De$agD0$H$HR7U#|JNtF5I z%(6JU$0CLEOki1r50_oYLxsGI@)zXlSukD?3I5I1vmjRwG|1pW=3y?E)M7tCt@eXt*$+_%S(^keS8S$w z^hJc%ZGz{@$UtA$`R}2p!!6+J29hGx^liwoNqE*{zI}?@h4J1EIbcKg(h$cMIE(qR zQ0L-w`SE-PzN}9$zB#+DSkM24+sysd)+EN0#Q8C%eiNQb60oj4gmxZ6I}f3q;PVLh zJP1Ay02ejapIyB6j7`PZLEm+*6l)$Iw=Mk5mn<%i%MX7O7-!Qk z&ISv{8FWb2bwr*QG0yg3oFN|aVF=@0(8u-tux7j-G0&hLr1>?c;rPz$$K$!)I?H}0 zpTCeEgzx6?!Z5V|ZKfOLJ;rh94;gZ*73bduKM!JzaX)ci=lWCTcoBEK z6X#9Z-{3e-BPU@Wrc>{5n!Ic!%M!MD$U~IFdfZDL{A9g=bvq~QxeT6`x z1>Dc;2v&Sy)^m#K%;^Y|&l0Ab8HRVQ;tQ{j*jGYN0sS`iv0@Trk9MHDbB>@}O5>US z43&at!gr6UG*dc6J~Pxpqag~IA(Nj$Gc>}_h#9KmXVeS@`B`O#Qv5Vyvw}olges79 zXsoF|$>l&-vqvi%r`R>|lWcR-Yx+)-L)U@S#xD>}W!(v%eA{wyTRCoqNw8yMsLyGk zPS8Zk1$~g^9u8|ld!Q)Ex$hoxb4#w=*GCSay@wEg2<_$Ug@17@o;Z(F&vEq->M{SV zg@eQ=kJ%?FgEN?yn-rVXp>4eFc#zKfkAI-^_O>ct+`E9z`*rW7U!V)je^Y7g7sxdK zO;juD01ZM{;+H%T-QI+hJjIZNeJ;N$`3mXmEL=Z+1jAIRrZv=#q8Bx09saY4-~{JM4b_71YxdndeX0 zf2JgCNBBcsf0Wn`*H1n_+qTvMTWUod_EE8(;WV4ka>yD9X^(N*JZ5l zX!>rFQagYV@)E)R*km8(vXJi1e1AK9OQ!De2pbLbJYloR@hAWnjOWZs2cLA-l% z?;V&kij!no2KNuoOX=&5SBd9}AK@l=u@N&3;j4K=CH)^s*1<5Bb2E^ zdm$b&s)2nb@qyg-tgNOh|JvF~G1Y0^@5z&@K!V%>?d*8zh7d3L^u3|P1C{GN|`WP3Dy+($-M ztV5px1`^Z7<>tzKmdgZhyw3IHX)cKG&&8vLoZfZS@k)KbBk+}yZ2N>x+6RPcSpEmZ zJNJ>Yne}%c`D=hR`YqBxGojB&;O78r((f{Z}|6TAuKw0SoFu`~Qot{;){VvCG8U8oy6W9$go&G`}kZI_XZTp@O zaBqa2wH?GN0ZZBf`}Z1rzdPW;dxg}V)UZ#aQ-|r9d#4zKDm9x1Z4PRZ1wLqmUA?)= z8(BlX)Lke&1G@`&k4gi5Y=0pom@zuBzW`iOR*3Dz*oTl-U1%TXy35edXt$4U#$MEB z*aVwRPpTUnCo9KOl?E-?8#WFuo!lU+I4=bNtt;U}QHg?W0%Wu(k* zCDSyT8slZKKTG6;zX^wVFLwM+pszGFgp8&=(vNuz(_H~a9|itc?}lj&c)6W*0JYq| z9{X#EXPMX8h(6}>P>#MD;tKf%#ytMof?>MTvDv{U`5uR}jfeY1QUdcZH^>tFMPk?o z!ycsy8R6spq~gnC%pXlhr5`JyAmkefn>Xj9)zNvP+`F7|VK0A&({Nq~AE6tF`4)34 zv3_Pl_ORRnCX+Gm`pPjc@|u%qz=)M$U9u!w$9UbXXIEBS23zP%dz8aiW?|Vc_9pXk zie(i1OY-*4Ao^h!V~%xjv~$&Q6Kmm!{W$Ezsgz;=9P8tsVSSvlx$vA1ntU?&G`!8^ zeQg`-8S!EMT)71D4DX^3>yO8P6?||YV;~z8C7`kYfk@YYbod^d!EI%E(`@MFm^S|m z+v@qYMH}k0d#)wAr<$aDmJlpf|6q|a)0AFm4NI@Kh8JZqA5CeskbSvO>Tl#I4 z0lmm(+6~z5nSLsZ%c-Y`QUgM6DXCjf&wA`xKHRKDx1g^em(-ux&nZe&>v{$F9wVvi zcTCH97Wu9;sTt$-M&4gUKO?QoGOQVM+W`L1J{|Q;+Xf#c$U(^Se`j16cg9hlEb+Sf zAk!xBCHnymAJ+rS57h(MCoHcAIP*SqS(Zn2<8q1LWsG^~0VaQ+>?^q;y@GlDFDa)! zROYZBOyEqJ1BMpz|HI@B$8)|ynUIq`j^6UQ>`Kw@<=HwS=q1ppVx`cdy>z=i2{H^H z|$uJmm821peSLRo5*1kR6$Ox0*pLvw);j~;X`;eg?tPdQ0rE7=Kd!({r z`ZDtecw0i9Uex)It8TVWc6B>p`V#CRKzBDHkm1;ayq`;hFNsvk`&X!YEqDlHhs%II zIhxvG+tT4yO_FOh=?!(>O4IJ=wlh9uLAtuLHhZ;01LlEZQl%x1{ID?R>RUO1N~u{s!7`-q0|zzWgW`EJ7bxGEM@w!*KiMGiZQa6+aT4T z@9qSBH4?X3hE15|g2Z*JZT2f%_sXTvYnZkNBJ)?2$!(;(WxelK{>*AKnN!+p;< zqmJ#2b?*BX{*JH{jF*`IcRBkMVyt1F!@TALUqWJLpm$He3K$w*rk9%VIh zyHM5|_#h)(lo8L?@j8b*b7J}um73b2dsK^7^E@dL)6v9wf>pk9GxQI{F$U;vqhPIp zVNj?~6({*!l)o&y61o^@0`FjMi(xG+d8HaSJ1>z6-}60Mc>2T?sE7Nd9*5$;aBuM>?174KCfpE z-pdQ(gD25{C&%N*(&zGrPLVwI6nVagzPz5KruvW+uM5${&%u;qk(6{0_-H`pEIgE8f_UA$So8md$uo!*4B{-e; z5%c;T_#SUWn_(-iv08<%08B0_hw8Zsx%*F7R#D062b&2#V(DeuG6PU}yxY@~NV_dQh zEO-k#7W;n)nW$VSvz)-1C(L^YDA$taVC@0E@b8Abdvzc@h6}eJ`u@3K!tFaJTs}B{ zVM_(=S7l*AP67}3e(~BCA2CLVF*5N7xl~&lS`Y#sCc%d#;KOFfy%5^jgtU;q?DvbY zGAjIvi<+V1!@p`$GyB+GL6a`2BG0TUdOo@UJh6bhi>ja>^7}%(FGRoN{k($s=t*F4 zay))4eJ=kcr%0J~io7>r%@iVOdvl28Ao!bT(v2jQm_iqnwu2uhz>jKsB!oWv6>L8t z?7tCcG>lHjs8V|mEmgy3kyCkn8(3iMf}h6pa$s1XkMsW2USMUiuFd;x5%weU!R98RZ$hvG>9BRhg)bC*h-h^r%p@-@V?DyY(f+3~Zgxb% zv{?=F*qZKve~PSDd{iHOTpx1#xF7R4y%6n%+)<}i+Pbk9w~ z1~vW;9kfVZ*|ZJz&7%3eyk>3McKw0DHBZ@{P5c95f9XI#^TV zlC_ob-+YRcd6z?$hpFg%$Vl;ojN|rVUMYf26qu0m=~mbT@Vwz&`YPnoQSt$A$i!$l z#?lovcj7C2R}z-*{zOoPb`~ke9C}5~dLNxQY@bBuO0uf8`X7CIJ#YJTI<%#Xcq8Qhna5f!1|H8@MEhh64+G;a z=;(cePY?|zoqP#R1;3Clfz~~HiVVBq@lm9{cEv6synQ3(5{Tllf{iw`#G3kb@ z?B6mi;Gy#hWu=gl;IGPJ%b8D<^Kg!(N)z)&BXk5GnFqhi6h#esIx?Mg8Ri+P#@r4X z(6(~eQ{Uw{@R4uj5&KV^{uJmJDPc3*#SEOr_nu% z;6E}`#;0@n9RBOh*+7*b+dVmqZ49uHc5yx@%`W5aP&SD*ApB<`@7HB-Rd1!0%$wq` z2+y3nf$jEY4ja6Ge=>bv_ST30F5B|(4cTVcahqf-YRDG%a~0ZIv_LPZxRp|>p?9Nh z57wp7MMzm;fhX|{Sk8O?SpF{S)`2&$$C9u}17FDgTKmltjz8>b z@aHjon-U3cQ-W;67&V==Y`5U=1f5VX1`Qebr=@cImIS|D>F}E4#C4#wH()WBO#qF64-%Z{>gG_CKjBVz3-KzvyFMfDVHVJ;H!a6&Ob#?^n>>yZ>`5u}=gHdELd6BQ9kgd{T&jK&g%uR1X&Xf45LNX+qBiD0dF zxmBoG||=zAF|Bk@W0nW(*T(sgr}=JAwuR*b zhlj7LYUS{|FG6|OX9YhG*6cr~m)LeqDnN2FV;Ik2etsS@ED8P4i?e@IlmO-@%s~T? zrFCrgY^0KFN2UMOrTykE`((`y?h(-^V|+-zs|rh25a}_;fNu}BIca$As@d#(2aZm9 zOgFSF6_zd4;JBI{s0ts<`A)nHNsKWg&O@4z<@mmGjr}Ho)I?*?{EVhjr%YoY>>``4ts|}wLHL~_eqZ7}aN(6S(=UF$ryBIPMXDz|!jf}^` z4N@l!V@)xPc0ixhk0=3S*=IIw#F*kbah6BJUan^_4mqA8I$`1uZAgT2a!bLP(>+R5 z+5;LfCk%cC=}e(JK(By)0sVye3g-yl;k=xd^$6a7g6{a5j5t}pFk7FYqbPkL5l?aOgqYy zp@)q!-I&WWRMef3`U9HLu0-S?b!%MCV1p9jHl0W+krQnwXS|Ol>y1hTR%b)noeI&$ zDot+NrO5-kv=^T!tizE3e(0oP&PSfs8L0YEt5D*t@Y_ z!d{eR)T97R?lajpc`JWSROVy}=0hQ4340*U+UaePw5%gWWgS;w{nSe9C=$-l*VLe<(1xbf|XUnxQ&wOHGPPX*L*d_Wc&}Z!(_n zC63L+AFtyA_?yz9y{POzHrThnW#0=!Y9cr*kq8D7z`T@8U|ocLrV|TP3;c0M;v%&T z3k8^#AivoRI_F#6u1=%+l zWqVP^2g{v<^3J8)h0Y%7%B1}Um-&*hzj>c$|A1sgiMh9 z@PB~JF+pdWz72gERNLVz@+6*#ebWAs!56>-Zw{p(Pp;%WFzBRaLdPHXvr`ZEkLllF ze-rYBcs#ML#D2b_tqns#q|41wb|3SUs4pKTF)qSR{qG#dSn0~>WTugE6m|*FD#Dpw z>{~JxkR^jmGlwS@=E}z$y6=vm=6x+gbv3CD9G~;8ODxLcPZu%@{J|fqi+ly|PWbm`u8}z|PKcXw1%z^)t&M#23mQQQxloKEae> zf}M$VJpQ21^ZK$Zcna$e&Rb|(8{-Gy`^nZv&>w?xLRat&k-83i zX`(M>;Atj!eU!|vk9_n2T(EW!xWFdObAO?HCirZfW_J~P1@Kc?%XYBf^g{gMEiQOz z@Y-$_@b)S}7M}8f$yxAt9{?=mM%X0*Lvt!foHv@m?i*}U0 z7$53QEX3XuKG37QY=>YvFm_rJb8gGv?+FeY+%uFJe06BxC4K1Fi^|Z*;9WyOu?7y` z)|Xf~@rlI3NUUZKAAOcC;`R%<@r|LqHJdFi8sqr3CpexzHJe*$-@QYviG|<6nz%K# z?{9}X_N^G&l+f4pB|@k(iZU;`?FsR`IQPEjwkMWI(w8`l%N%@bXdtl=>7%!02KNqS zYC2j_Bq|yF^-yN4?IbM}?XE++Q6~PL z;P}B2)U$}&crV&$<=Z%4_)EfPa$nDfwb7!N77iUt%&}sLXuJI4w4o7h^H^J%_QfC7 zRHo${4=*jiHk@TRMQFg>bZY5z4=3D2cY^+DmUFg6)EzAcvY0i*h(%TqlCx zNo?Fq0p_>C&kxmcn-iCPuRjqS!|Ia3zXApe$>9l{uS#5szPR+Z%svbB{~h}3di2$$ z|NV)57X$Ch9*-w3>%1}%eC&BHt48U?xbKirh9&Uqf!w_m^e@8rUW__d4Rz(ei{~Y| zI7lGz4l4{lJxR15-(ef`fnNV0$~kSmTL|oMB9i0w{R(q%5InJBe>d>RBo>u2UtLiX zvS^7SO~m~tD61tukKW7WY}`q+_btL4ziS9%iRn7Yl%p^oVes)pc_5Zmza+Hpte%6`izIP5pUYcHQ4#tKb{VYahxiB8nPXC=LucT8i7RalY$K+L>7wFVrPL>UG89e^r`~NG^3HT>bk?$w_*1<0D z3G-#X@5c3_uQM-k9?*XWbe$^)e>IdrIVtdKKZgrC%^~=_+e=u#W1mu*^o>!vrZrsC zV{sDEmt*ve(}FZ=*wHu8M&>K-Gsr8ZmE{!Yg`E0e+vHO(vy9?2h%ap4Ue3dxaXJEF z?#sN~Eu`aPv6}Aa-1AN*Z2e0OM$6mu_!%wUkequ@3uj^sIu7mx$7WQlN z5axNTW!VQhlGC|bSXSqCm(%sQviK*{;~>{m(5GTeU8gQU*`mDMIlkQbxGb)FWbk3m zA0WRT{=ck;3T*y@c2gam$}LQedmVfkh2fK=sI$^Gm$;X z_;%K3-w@Ahv&Y2qbJ-&pe;Hxxopkm0pr@a{Kgc#}*sa-?ZDWj8mRdL;Ft}Hr@DT2D zB1!T?hXbxy-~<7xrKo`!HC`DbG^kfiL(b)>@ zw9;m{6>S!+*i=r}gVq(GZ7H6zU-5U_Pp8^oLjX3PWN%L3x;%SAJeOsMV-pGfz0xUT zsdP%Zs$P?Vrg0A>vw{6v39|Tp?t6T?8i3y#JXd2Lz|BYEUW+A6gSrm(U5RPHUiP1L z_#$*2hMoJUOvZAV?w2Kd+FJX#0q-wSXKnVNVL-78!X}FSzr6nzm*HOmzr`l_`r_P^ zQGv4s6J%O^(olYJ6jhWe?<`y>eTVUrI{2OjX|&FydH2Z5du78^!8Qho z_mE5AkN*J9AsvRV9QJgD4`v(~0uy=?JOL9gY{T!+`XHZKD4KQ=`(2^^p1F$0q9UFT zBU>ca-Py)#^PNri$GKKvAD5<0w1;_*f_?0bi86&%nPyyUzgQ7CMVSB2dVY&(4v-U{h#ry#;G+MZ89Vu8g#%UhDZW%JX)c2(KjXcded>sS3WInT-!q z4Dlg(d$%TsJKz_W%+SO-$nQvoWxDhoX`xsW^z{+qrFb@zf#1}ypM`VjD`R3`6YdOdu$ ztS60q&PmPY#x$Amoz|2WS~RS)V@Owpa^{oo0Sok-%qvv-6%&4dA?2AmPE(3BLGYa> zeK#4BySqahA3&MYx-FbfGU*4Xzq>0$$EviAI8#A^Crn^q%IBjUpb2?rD<$Ba=YZV< z=(p{d=Z@EFIP;bPpZk>snDbu7ctn1|C+Z#GLGY})PQl!1vMiwD%S^c=rOC~kz?b(R z{#l$l>sNX(M@xSQY7f5y@%}(}DE%_{cS|6II#iTp(NyquEqG#bOGfzD9;*t`J?OWG z!DF`uLejlSP5RqXETXXIp^fiR8>Ny7t14}5mWhcWR}lvA?H#5nUv&EVyycWFJn zkXH{-5OPZwe305@(3D#Uvdu>mAtR5qqD+h%rPoCMQphF0{7Nr$m=2Tg|3Ez|1x8GH z3FOFJU_K*~K{$HDWEHQ+@qO9kz-QHP~GGs1aiZ$w*2 z4rZk5AuoOmTA(Y!ZyI`8FeCqQQwX%_cn{I^M@W8Y7i7bLg)r<4v|_v@Gm1VC;(3G9 zb3SPu`dUtzJZG?MKpj3|7lw?Gjs>x=o6;aZ=L_Ah1akcWib(6KLW2K`ElZqpVtKY4 zy4~+unMS`{Y$5)BlgACpiUQZlja$*49x@?ItDH01sdCICF)nB04&Z$LNJhpeVEHNi z4aSdEXDD+Z`#q@;%Vw0pd28rMtA;X(rdL@g(?c_NnUJdi&~ElTg0{~bu{J_)Mpz}u zU$Izb$%++{+LGKSm}8nV(j(29kh3Kn87kLIv0SpvLjLiYMmk=nNt2pGbR1!8 zqT0jIt$WKK^po;Tue_qoYNq2tH1Cn_F$zsvrS9PQaGi#yIbugebR;`Z7< zm+S3c%FT%LahM)|itNW_?7_%&_D|(n`zNw#|5&cFACV*WE;-nflKmAwmx4&IZrm*U z`K|^%E)(8M@$NydQN>Rsl7}sWR#a&+d}`R=m3?1vU*s?i2Ms<`guPpjj`a!l#4M~e z-b+`*?hGGT)}?Wl;ZOP!_@!=^NP+yT_%X_VOolxwg|dLE9CcyyVa$ry_>z(2I6X{}#8CF_DwI zjPeF54gVD5l*=hd$7wh%?pYG}aoXt(_S>8m7-7>>tZ__QVQOK;9+n7qw>5An@O^58 zY?9S}U}$+v@afCh^5w^+w4!RGFItSeiv-r;_Of4AO3LHNV|p4XEoyEMFreADOH z7QTD%=hv~z%{Igye8>4RWo3hXD~BHfza0XP9Rlwi0-qfMpB)099Ri;n0-qiNuN?xf zDPoS7TgKLb*EqHbtV4V_Px)L7>nE`eWM3T0ppRRP%E_=hl;9q)Qj!)Cs}eCAE*YV#`eD#rKU20~(u zuLP=rO*P(m&9A`+2l_hRad&_YyFjWB*gRlo85^n8)UeKb4HzjE0f{8YhZ!^N-x^uv ztZO_m7OQ!j+9SY8<~_^G_%N|8rSp;OfR3LyV4I3wBDGLMMx$>JX!->BVojDwS8*p} z31m%agQ=7@YiN?8@EQ;P^TtA&0-uK|(lG8;cqpPt@RjDC#yI=~04t(nXbZn%eXAz$ zj*qYbG)g=s-%T~0vg&?EUihlv4qeoTap!*>_YdrWJ%NAk;_LSB3}1kZx%SAasC|g( zfPVIY`~F1xuF38e&#QCxrK_@!iudK&Ol&glhVYWY=R5Q%I9K@sf5+M7rI;gvIM;a; zYsxLE))mM5I-E6ZN90n>WsMO{tjjUJIKB9E@OTTB!)F+>j4YDpD7do{eO<2LEYR;$ zo6+7iu(Mzd?tj5PiZ#5#e8zbNJy&Iaf_~VG^{?aieuiDk_?pKfB~@dqNwn4U@EsZQ zD*S!`zl;2z%h_}4v%8q*SvDHa;O`r-EA+Jh@1$Yep=1*OWxcaVx2~A*reU>K|`kf zpr7Q5e)3=1Pg586-**eqeJvik@7o^vz8gIrct^@q{sVXyU5ek$SST|-@~SS42BRAN z7;|d*Jqmdj!Sh&CV3Ski6_{7|BJQ6tPO!e+it@Iu@zB;LkG!?P;{i|XWBZN%x=hnP zXFtv7C7`!|lwW!jR62N7XVdKSsW*dg;F@;u@T{!O-D|?O$QodZF-_mad<-7$>H{Ww zfCqGYYQWhw@CoAsU&?pV_|E093>Uu(fO!V~`|rb6!guA#JZJFzFgf1q=5&0{hw15L z-axq+&mQc}!R80H;&Hwda(@_dpXGSB{pVC4>|#t;b#^P~>uM8yB#E()xq$i9|6Poq zhH+aKrxSVN*&l+xQoIk+`PFO+@+-pMW97HoolV(Iu(`0mZxptede~y>V2h~*FY;Ng zAY=e|o6i(f8K1`8t>8UXpDK~=rwrpF#?NugqiM8G#lKuX=(dOCD#z>31wWfuhgM-78pS#^0y!7NIy9h& zISNSQBl`EMY-f(QI>hsuY`b{YX1iFvgE^sFK%PTR3Yk7xvZ%DG8uxvWl;S>wzEp&H zWvQx3MX(7pV18nr>H?pA9)B2nxX;(BxXiY~i%?!!CPOoE&P7#C`Znyq;zt5F`rSc5FT1b?Ytfmuur5AvBQ zS;sX>kYjdgH%Xo(hpWa^P7he34G8apHT?nmag0U9z?n#6rXl(U~BlgAFNtK{y7E0Z8r8a7WRj~M-O66NphwNW!2%#lYIlh#s`}MaBNnw z)>$WQ!&)NFvLXYxF=sY`mzt8;Bf+}7w5OjY#H(|>hBJ8TXX$G2MLW_%_oG$lZ$6vX z8_^=^3$~WQipKpeug01R_Gv1*w4Vj6v>xU0=MkDorE~kCe<*yWm7asWx5(6RP8RXq zDouMZjxxb_e1?$E2VOLOhLFzqK%!|xHOxnm?GpJjm?007#QJL&(qYczxe{wO z@H)4ddH!m&ciNRGJ&M$@9ZfS~ON@p@yw5br8@2GRQ7zKLro_B2?T%_X^gR5|f($)^ z`Sn46m%W|!JwA`N5&Z+(jK_FEtVty3-uCo=weJqU2<1E&G1EjypjS%Y znLUZ~SR>3|kUMpdJD=pv)g;!_gu9gb8NU^l{bR;W-%jUYT)x4-13?+S361oZ5NcA`|C!;di&8 zA#0&;JnVnBA)f70SI{+RPejPSrK}e@`jGyr48p+MhpRMwGlXy$G6D>(bZpMUcv|#tul+q0J zX4J!Sjcr>qA@c%H0QhFuC0jE}2y@Rr2Sd`PWQgyz;d9&4?=g-_lcx0hR?cVgId19< zYOtl4NTbolAm%HUgLBxo)56&ZE2lqrQ<&FMyKz@fkv+t^i5>?(F0%&>U#XZU($_ol zMEbg{xO08yifq?T;Fre>^Gp!)jMxuK*Jax=*W~?@qmXyQm_NkXZuBGM*({U$4l)%CqkSdfYhy0ocS!s`FOrU`B*2< zGWng$72O6{O$mg6ND8ij4UfidW^W9}zM&56C>`Xr2hlRT=(0}DP-y`nB z0xkKrv7*8C4l|!IA9DM^qi5W==foBAD9)nhc(#Bi9X`BI@Fe_EH_XYVz>~AM?RlPr zkT)vj4D`oqKC8b1x2y>4V72IDe7e>s@ALXXjP>pO&TAm(nQQHLQg`Bv4DM){Cfjd` zeNE_t=*u7SSl+oPo7x%A-j}Zj{g}JoFS`j?Qy$~9^yBNN_v6geS^9C)eFgnUWw<+I zh4ULcTp#x*u!q?3b8Y2&5JrF}^ALD=_8PLp{V=#ozXSbSk8v`yiO*q8pNQXoV|_lJ-73b2>i7(C znb50R*>4E#uPW%fJfG(1M%nmqdHg&L$Hr{;7>*bp0!R3@>uYdLO3tU9#|D3x*I{5KA!*c$`C~@q7Z>1U=lhpr?fC z%+tj*<=%615bic`?uG3Yx*B`~a3}0;wFUY*kMk1eURU6jIif$AN#1ZXsi>k{17_TYGJybPPU&hE?zr*T=2a~f#2rtOUjRnNB@iC;x*=LE{~sQjEhGL$Ho7pd1FkD zfQQETn|WuPmjm0u%LC`i%S`_{@iTCbm62?N2hZf`ux?`7aqc<+HeUxl&wYR%$lOmG zNXKIfa#t>jICClPmgF(SpD}uJ_ePPZ1f z19YCF3|P6>q4DlWc93n=u#Ze0llH{3&GG>GF4&%%reWP4@Y>rmxN({5aaAJICekJ?*?MI9>j( zy3>A-!&(03WWgEp*y-~x)5S8RQ2s)Wym6MWV_arVCLPI_E%*)NfX@LrKkXs8`w6TGiD#Fr=C6!9|IoeG!HWkn7@yl zYreDe(=g+h*N0igv20;{xsP)5C1hQf;2Ypv44rg8>ADf4*)<){oWA z?SFZ^{r)%3z5Y+u{(Swq$0x=CzcVd4{S>;?7)^`^kYzk(`u@`NfCleb#w+CdY4q%M z=$Y~8xzkf#*grzPjq|0WtL6FfEObD|H9NctkDBx~*@JBBVR^^03wNQ3_3J&G!H1tf zCl}|yWfMN8kXf&eMEQJRQ=E$QbtB+o^ptjNDC0Y$IIqKb(q_OEc~Ks|$N0OxXC#?21DEB` z5vEf3UD)A~D8j1r_8BdDKMm+lR=k|HP=145#d{t6L0x0_xa-w4fV+X-)_Fe}zqQ4( zG~MU@5~lAWbuUL`Q^o!7 zWO6$ACzw`suI<}Mf?tQ}PhuZq3(hxN^dnJs8A+X21!`eO7+o_5db*#A*LBmJb=y#v ze^V^RXDzYTwl`BP=r4kuPBLDY(PGRUNg7|H;F@p3wl@vyD(oF_-CXZj{o4!b--G(` zF6!^M?RA?;DdTff_^2tmO-aH2nNnffB5%d^Ts!-S>fpg+y@I_2v&z6z?B}QrxF_~? zlA>|4H-x=kI<$jo-~SKz3ZRbnE4;~_2-~@jHtnpOP8+@o-93r@QWL)+dCsI>DP_#dAne`l5goqvG7eiXkU56w3HYy4*XOZe^77tvRH@jKA_;q&n_eoy{a z6!5=k{~q-OH|(I&4X{mZ=q1kv_;#kB66px(JV@t*J*<3Z0%_KO=3*+tciOoY?@P&} z4pW?I(x0JVC+h1&U7e__6Loc>u1?g|iMl#L2iKJb-D%L52JLCko(An{(4GeEY0!?} zc>g`qj(miCp3Y}b)*hreiF{e)c^!FP!~PHJ51r9@*c0z(MBs9G>Mnm%{2x_OpyDmxHw${@6Dq zXq+oO7>7AM?t)e>ZL;rzU9dkL%j;*{Zq_Y+xHe#lb+W>;1G)y^2ZuVvx}WcZ^I{b2 zC^)yh4*i$AN7TO^=U))VWucrHu#?8}pF`fRwSmjV^Wq$RKJQ(T>9_+mOgK{wSp)vV z-X`=zLO9ond{yV15BG1NuO-7bqlIl0MhOk%&+Pw;myv=#kF)s>ADw<~U!3s0$FuO8@fg0RQY6na1#cAj8n$yA2uLx8_{bN^&YqC4 zcZj=eaR==%+e3KX!`^$ClVRMx4f}fqo`1qTHWAN1;yD%1KU6isij8v-u+#hzwi65A z-w^dX`F^W!vEM=1QN;f)hW2u~+*U4kd>m_Wg72olY=Uzi8s_@EUgOvi1;!fA2?k-4 z#2PX8dlKTeB*60|M{&~adP*P~Z3ujlQ1%&Q;9_h-AWLi7*rr&_)I@BcmK&DgQh zf4!6@tn7_VymAlk@nfIQsND1WGNbd`k1SsP+SiPmJ_nCMe4d34?+rNj6)T^r;(Iuu zh?k2v{^vQwW1Q=(2bkEq!riDsuj!*%WJ)J0m!KWF--S1P(aRQN>><4hEW~}T@H;4n zf2{Lc8TsG%EuPzoyH6K_Hx}S5?gF-1F`q!bs_=n0Kn=|cfCqd`fywi1Cky89_T%66 zT_E}y_ZBgE80$}v_tRBPXxm4n13s8*NRRpcxF=SeeK98EY?-%cPE%v^&c(j$rA%1#h;cj5iaUf3H;HFkdc7mGWexryncoe}ud{?9)6 z423BT-r_wm-uG4R?IZ8hKKROoS%yhptA}6l_xhw;>cJOkxpYk(&MB?yqnqKwRaY05 z7S!Sl&a-{eWwl{x684UL{e4ov4AYxc@V~vSPhgv^3PUc%CFH?AF!qtB#`*j_e8qr$ zeK}2>8kZXC!_w8v<8|e@V?2&~z{Apfey=U361>kr`KA1hG2z3zj5^-ncjjx9cNld% z$M1Y+?bP_R-S$&L25$jBUxmNr_*;g*TKw@H&A=|dwEIjFqlh6)U;qlGwy4Q@|ULpDON=&iocW$#8yMwaUQn zlN&c7t{D5AYtaT}eva}@^2NkyY0`zx81vptVq6vB>HV}U^7bLV=pq?=5SoMWMWCT9!+wp^YV7B6SG?iy z-x+);=CL`WWVPa4No!2-;S}5%F$H^u*YZABS=Cwi5WgkQdvSl+dG9ot?uAVEoFUWq zK&H2!A=AB;RahyXF4OlwrnjFV)17p!ysvzkOy2{U-VT}0A0gLg{F~r2A=i1m3Exur zeB8BQvIG+yPr4;(kZ0(`a)}AE?3i+#DtJ0oHRqOdn{1j%UF~vRG@S zyYO2MZQ!M=c^(UvjwOu#c6st;!eX!hAf)^SylpB=7TINhAi_!mKA+*gypFohb&tTSvD84%;*1g`D*OK{gl5wO^{gMWjfwP{=az6A6ccm?;dW6c=2;xwK)hx7eB z?=ZjHV$6%ZREzCjo&AA2H3{2SF}}YNSgAJ9Bq0Y(tk19(z;BVktU@%pj&%^oxz+eh zEvymKhXX5FUrhG}_@4i&j z2Y1qYbVo014=rihbNvDS4V-)!nOLJQ#U1MHYrk8$2Y1o)$AdfP#cyxse}gmpF8mwC z@2rV+6Xp{CtMJo>-D>BYz?YcjopPYQkY+B1^`x9XlWDVE9eWjQtAza$`_ID9N!YVt zmCE})<=(O9?6fcGPtq)21B!dX?!ev0usx<*0yhs@`$srDeM_KOJR1T{XqU>?2llhDR=s&;m;EZ&gPuLfX-?NK&{NPa z*j5L{KhU`(AnvdJ-^{&vd{@Qw|3CM=WFusO00CJpki;Ncz=)_IZ$p5ffDsVU+LDk2 zNC`JMVj!7oi8Z}AI`j&*>`&h`(M1b zN6)Yw-_L~fE+MWLyyw@BZ~b;3bX_;hNc7&pv;B4s+%+5fH{MNP;xar3je7yK!OA`2 zzHJxXr?%^(L!ZNf?)+IjPxbXZ&DZyow;CLL!6}iCrb+LFys_cdLVQ%dIljS2Pe*p# z$FpQJ#_Ku$-d^qUiJlF-HsjwBwV8I7dCw7c#`1lN*IuW8n(Y&w3ut(E)3^KJY#*;| ztRkLgmDTMDV!Gp&WIO$1ZTDvtul{kFYCzZjVqe1UcRh_~j?tOYtFn zm*79Xzc~F_H;m=?8)ZlKmm~OfyK^sx_(R6>0?!}uqjJB#d7cl9WN(HwMMGw`9dl?e zUyu#uqu3+F?#sa#e7$MroqDeNJ(J&jr5T+OE?j!B-=&-81N3zE@$6s5(YFU$TJ3X8@*@pAd;hI@ zP8MgnnmOz5@_f@6WoeV|*CF1y`Tf2U@gJO*Y(K{vC%?nenYN$xKE``zBBR*5uIIgs z`c2zE#;fJ%)_z}0YlmP@t&w?L`&y@l6VBv4T;dPCbff)Apm{!hhS3YAmA;kWfqQ5BP_gC#9{dCU}`>929hiPue=64*qnnO%Ob9RXO zxXyODZ)@im`f;S4XZSt;sPNY>fe~N0^d)YX2#(oBTXmHG-F@MWA8O)xNl)%w2!f(-_&bNbRBEwZ{~a6 z0rL^_Eqfd}<62wGKjO@~_ItGFp|#hUYx`!MY9jd``SWZ#^Xz%L9vZ*IzE3f{;yZI* zV@~L3({shz)&n_@@NOktw*`Oxo=`c@fCZn%+JzvW*}!|M`Yz|_?EUNbY0{akpWMgH zqduUo9dn&IDim|}Eq@kWwRa!x-V!R~`p=$?%5Bh@Z@Xt09niuV^u8I_wSHya--tim z|2t$hnaKV4%>7GEvh2a{H2%ByPR3NT=6%ieI`5+XV!hL#Ig@_jtWFLuugR>XQAe%F z&aBNDmRY+Zhdzzt9eXW0+p6b<5-YKQ z?Yoxw9QuFYoXiZ~k7LY=-n*<1Q?c^SB8^Xy*RAYF^| zzN;g}wzjy2&UTII$22pCdjtB@{eioQWsEzuC1W1fyU?^JZ`Y^V9}BK!R<~Q9&#`-A zwv2rr8Z_E=$eBMYADvmlyk+Hqu=h>ri=>2{`?Hp1+O-sY6Y=2&(X*ni@!_T;^kd|( zer)jjk>AQ>HvB)(kB9i3+H#n_O}@TO=-c%Fg1+|pf!`|WjEUAw_UwrrCw84|I?)|E zOF%=ihx0oN^{eKg5HW~%QLOjR6MQe4&YQ@`*|E2KHHYk-g}Q9?*9ETDv%&hCG0B|g z)n(V=al};oPO=r}a=32fy%(*5iGORz>wL~Rw(iIgo}p~_>bb6BuRaG^*6%q-dtKN+ zPv<{RTK;a&ae+VFchPUFZx?&E3A?!H+4g&i!>%D*!HVD5ieHs`w!%hksD$5ldv-}JH`TMyR-I)Kv-v8(wVJ7Z4Y&B@3^bVQE?3{R)i-I)9ecyihYy5d4j(Nhq`zyE} z8}S`ys&dTp!~xdinj?7E=r_${IY$xaw)LOdo^vL#65HfVN3M%Knwb~qL;COioJS(2 z>%*@@OpRdPvV*O546X^-a)W$aGl~He>ru~)UWnEnDlEsXAq&$um9Wm zqWyY$l*TOMl=nw+-52FP1ZRxP*UbB4#Q^($HG8hOe9e3Fb$=4186INaU1!!|Ki1mD?pHIHmUAYU_g?V($iBO@mf4(_ z(>1>SM!3uF1Gi*piA#SSB?92>)aW> zoiTMxvB%m@XG^e~Jr{I!PUg9G9)1R0_)S*hIP@Op>pdR5Id3idVWWRLVeTvV*P_^I zzjvnJ`?j_V_0T?^eLg7jcQC#Vx8EqW$J(zbb2>Uf*R%7ANSWE3DUs7wY<4$G$6z+VY-8%~L&OPlt+C4~o zSl1SIeWiO8RDWt(7W%e0p|rlc=r?n@o5Mcj-UH;1)MwPyaNMci?+^OTNh&_%x;4K; z>wfQDA8IM<$p7K`)#(uHduEaMz7O?Wv-FG3GdA_%xUWZ zy!D@*YqFT*cQ4OLW`rBkm2QFL{Ul2z&R@Y-8vryHBSzJ2v#zB58eEsmp2g*CHd* z7&FWfrnn(XV`)^VnYqFo2!-q%M+{}(jo+u(IhYu%Icz!cf_6kQoooMvu z!F3Atq7%>KJ9rjU?L#kb9iZn~HLR5HNBs3s+dC8Oyi0pJ@mqX5_lFw#^Sgk^a2CFX z-vAq)tq_W7PcXM{bA|P(SjKs+KM2;^H$^`UrRmA-s<7^ zBSURxXY+i0+r?MsI{Ca}*?qB{tLeLV*+_i+b<~I7UtHsPF)t;-xo&yEPWxM{Bh58l zKjG&G*NYA6ANjqt1LuZKtU=dDExM+bZ)X+kHuzN}tNc?}ZM<)`z8Mqt@26veaxQz= zZdO^ZysQ^`E#W$+|Dfm0C0rX9FkUVl6>Xp-?~iZcJrxo1tJ%|o)p7S6c#x#uhFz@SKJ`cV%@Lf8xuek%>J(l-7Y!S(dhOAH7GB)qz+ByvJ zyVI{XjF@_F@B1;N+M~AUx_=ZGg3B3P8B+~=*Rn6~4Q0*D{&$0D`t$sdaWei;KNLN% z1-9q9)6Vk;{qm4=+MV|MW5(5WQc6PH_N-+)Y7Z~pj@C?n<7NG9leX}i`Hehp zVm)idwX7MN*|S)QKR2Z$M6RPx8`0q|=^&rUD%{cn{<8<}FvXtC$Y)v0wroC_f0&AH?z9zj$nP zfqj1m&%;S0BxJ4G!nHcG*1WX$8u%rH@@|0Ib*a6SZ8^yEik`jh{k%ezp#cK$7IMReyr;j)(fZJuxEnHqa5LOb^@uh_X)G*iNuWS<2d z>_T7MY0g~{*}0d~U?JdG-m+9;tz@eGe`{)&UnyO774B>xR;9A<&9#^ba~~^4-#>@` zneBMjVk@h2aV+wj@Jjehp5$6TIatT~cIEb38=bXjZ>QanPx;+^zJ(|5Y+62NSH13G zQO~HXk2dW$j5n$l?5Gex>|YyF&DN9d2*xs!%YnQ0$felw@E!$e;!>ko7JhL5di7yv9w%+O3 zXcO~mNPS#hko`1$$%R$M62Ut2omJMdHNW7yq@1a;7tdw6ljmgI!E*}#WU5vWrygkL zxw?0lTx^z*7IW-73V=td9=@{}r?ouWwO;*o%^ldVVAs|`W6j94lE^ChCf9^6Y0%nW zc{$HhyVH!!TD)rq-#tTH8~fhxV#_f1NH692jH~f?et~m;KJPbnO#&&6lu2?)xum@N z`VpyP9_Jn@H-(x^L;Ot;ig;VkzgI%XT)gKF(qsxS_3SeZeIZ$IP%5 z*V=n|Z%a?%Ol*Im@;~-7xht+jcg}zlrzG?bXzpgkzUg4P;3FK8YyC=P_193rYN^@8qhpcJlPn(5;rmc@ zlW+>>4Pt#+YbS)Zvfs(QVCKbv2P#Y+>#LE{f2wPoEY91H>K!wjnUBnU+pYs_o0wnD zRb6~f(cS{zF~Qg>IfVYEg!aZZ3g3=Tzm50!(2eJu?9%IPZs)R}9Y2z} z&+Vjfs4>Vqs=eh=jM;3SyS>D$K1^o0WVsu;ZXKKv;~F5`eF?FZJpgoLubFj1+j^R- z-A?9=%z@KOpvM}s&6gtwX6!c6(8*~7+s-Y zpKzgHGz|?M*VzBZy^OQ>7;j&hOmucd*Zzof56#wdl`M`pW_&+$7%kS%+WFC1&yQkz zv*m9q-e+9nZ^V%_`5WK9UpD%D@;hgR)_#HCrIoZk9y%_s|2ucuvGWqoc^Zl?uIf@u z(L41NOEjj5MP@mTURucbVI0zDc3cwkHru@+p2yb!Eu*3J`(yf<9<+fm#eLjPJO_on zxqz?haIVXN+sTahGKme+^@#HH6!kz}+*&9oHwuzgHq$e!+|S%|{1zN@!QR`ziJusoP0&smfrVEXDb(!dFHr zX6%}6GK*uOEsEXA=(yTVg z?za6?jU$_2%LB44_Eh`hv88Q$8y~d5o`>pX`)efay$*XgcJ5lXWv%oM@f`RSAs$&6 z<2S$jk*BXsPfpJ{JVo;ne_q>+K1s;T#aDVlxA%2D8~#}K065lFwbw8Q!Bb`TO+!DP zku#O~#-ERnwdVnR5SYKc+01DSfB~$xng0rQ?a^9*XA@!Tje3S?nE5_8ZLaAssN95h z6&bT@JpH|ABX0*aaXkNaFyqk-Vh-Thw0*O<{(k_NIWERp;VYG$x|=@jX8UKl*FReu z56;oArj3f8^oP)H_B7&Yi{BslYhE<%cygyUvk!~kw8Q23YbskNSW{_#v>lIK8`wIR zy;_nL3hxQs!Tyk6k0EBN@AVAUc4d~d%WF+$Oz-uhc$WCdHdbm)F8}PxOSx`v2g`5l zH}?sBx%Qlv?^7Gw$!gJBCe(6>ew%MYzTU-`Q@3lSiRqp|G)8>LZC7q<4)Y!Tl@==D z-DJdr12Jpg1m5lWgWz_&{Km%XL->38YZAL2q+fS{%kt>ka>?5@3%^f`BOkj)zA=8+ zR;~-5mtTcSytQIhI&nsO8~B0VkKJ^cG1HrQpR?956(+a%Zk|6uzS95`VO$v}T0HQK z7}tW}Omi$-bB6Nr3$EYxvQ~)cy-c0@g$k)BKXhIi?92;?*H7suNC?I=RVNZm$qN+HILty`Bn$7 zFQ@7r3hnBt>uRqrxi8DJJM?_iCVX#t%yiTGZTV9A@-&m7{H{i0&frN%Hh%?c!$j6I z6TN5X@a?^Cxybb4$*ucX8+Ph+4DT$7_vU|ZzfUy}SkS|fe>N7?h> z7~6)Agp2L_GwEBdv1&y##0ADK*UsCD)klmadkwRE+^#CGtfkFER*7Kv8kUd$9UH;NRz-N8_6AdI z{W)t(i>HIGPq%IM=~?Y?JA7QbxU+4Kj~mpEZ?BKzx|!=t+5t?PUuzeuUrX?LMjkBp zTBeWF_26eYhv@TKF1j*?h>gs<%PRSZ2b{;pw;`( zvOD0J30~K?b`L>3l-&dVT$SVc6?@U%L(A=j-gaF~o;GUDvQ=x%Jipv>^7DLr9(nb( zaOjCBwu$C6BBbfgk>|tZoa@H@%(K^Ci5o&Fb zOmL9^r@B@}TAynVza44ad>Fj-Fc=^5WcuZI9|phQ4(^S-J%_=vRiG31Uw8{|zcjn2 z(oSEkdu&@n1!)7P$68-h+Yb87_9D?eTkYD~4qqbNJFDBztJ`aZJ9IwY-e7UN+t1P4 zu{nIrJeQEYJanY5+WXMSXmc@%`&z49IQ8Q)=#k$v_C&Bx*0_IET}so4~dHFk`u z&NTDG0``nH3a@oD*K}I%uy*#=I$A@cLzdT$gX`ps*bgHW4}3QCMbGtCk7chW$i|F! zF57;X;LBnxZQTQ`chd48sP9jPeqbicE9wcU$_?je&_?l1mw}K zAHLT09r@{6n`<3z+gSM1H8cHc*S=aSYwn^S?Dd3alUK3X50ka-_H@vG_tvedlRo%? z`dsX6uQM9mzqB;E&owu?o7XqG&o?!?&+ct>Uo?$w$33Jaho+PU4^F8+h);nxC7G0Y z5O;XF{|%gyg^WDXC{od(vQnmlD?w+IH*EN^`Nxeoe&h+Iv&+gW*m3gy8r|(Jq|Kxq zBy``tSClS;R+eOBUGoSqdwN{eq)vQ)$iwp=ZFFBx5WKYEtg>rI-kAuTDR@q=rlBX^ z(uxkR=L#Ow^T)L3YOlNic%xgd3VYQyIOw$ULv@@$JO2Tf4=J>;6}l}{m1!{VdrcZ54=+FD;g%w`|K*dHBUR%e~GtJBXL-8Tf9N5lDDXEe=v514HjV~)>wsdo1IeER$iSFm&Mc|YHFWzS=Q z^8}B+^|q_G%*wh2xJdB7u6^yq#f7gY0G9}^ZLYpAa?PfH0@n(jG3xhAw%nY(3AjP< z;O5kS_IPCE_rOhpU%tb6`trMvdKb7^@UTz1|L~t{E1wSz4z0dx5U(Ishy@LOA>(56Vefnzn-xO&2 zFMS~H`>z@=1Wpq?s`dH2(3_iH0d@sXyRQ3RrxYcx1I`nC%ZRefhnMw3-kUMO$L~7& zxEq>oycW1v@PpHTxo7c=zYPa25&Wx+8$ZflH5I>ovsUnL9{k(G8*YDrJ8|7h8r`ng z^?#GP#~R(t-TysrCOrR@tZsfi-L@X&=~g0r{_FAbCO-9?Sfctsf3=~x(H*j+(H+(V z$|0qZ;z^=YzvzSAMtA6u=*1p=5cGjtDROxE5|5r6YjpGEn}^eDJ$irIkhS*+^tI@_ z9=xy5>oehB-{?+hXmn4A5lbPl&Ob<>=h08!+~^*Uyr0l%r^*+i&+(aFQx7TM=*v%Q zbSJkM6G8MLbnN%$#zuE6d}H@RiycN^3%#Y$J$WxlV@Cqo>qlA9HTLplkHhG1qWAO7 z;F-T=|2G=o-`eP2u!B?!O7=@2`Sy6D$)hiU?^h*9pf`K;^Y=Hp7quLLz8*TV&!?`7 zw;qAs;+0=KkMgl2(9uo$E!y4aUf6O3I(SQ`{37Xnxc+-RdY$Tz{gnPDc$HsR+~}T1 ze;!``%{1uLab7~BTibL5y6cs%$!m10Hy=*N(>t(x%KO2%Pjjpp%}MiQ?vQon?FJ{B zQRaB^9ewHv{D@~Ha5{Gfdck{^DK^{}0LlhmXcm~cW-@a6Q!2*A^gMGO_Zfzfp91e( zbAegH4rdX#S**}%*}b02Q#9@I(AFPYP3w={?I&9=7RMKVjqYVFB>LwvY;YOvylf+> ziKMnb-%-R|u730VVf$X%yS~xAW>2Gg75*I)n`|D?@dMGXTuWN=Q@Ucj=vOy4y1%84 z!^^*(>Cso9*X3)CH@@1*e?8ZuUp^9g+7amBt^7RdKrK7I+Lte)O~sAwQofhbE<_xn z|BjigSz5rc=dJd?z8U_KM)zj)xupS=OOntaUhMGcwCQHqNwoI#JwCm;(fvaLH0jcg z{swg@{;fjJs-h#%(>(gEYa88L^Nv7wJ^GF4eZ$5h(6NK$|Gufwy?)OT=-5H@->qzP zuiJ41dWolh^ZrKn_U0qdYoX6;bnh71=r)hEtvjTBJ9e<{lac*(;f~$#?*%rE?rQu) z=CC&P?YV<`-VzS_-6ed7u&&igL*@iz~|NYZkio3riQ&)>KtiSGe^xHEvZ+^;}Cf)iv&- zy3*NI6|R*q^_0nO>Fk=tu+6J*D;Jc`WuH%fQ}grk)X1sB_$D(o|5zU$?!m)6hN&Zc zzWfm$J^y(Bd&F_J{Bb^gGI+_Ed>jHKkMhW$Jh6D%;K~KHgY)Pr{;F%Ltp@Jo0t>sR z`QP4`^cvPC*Uhb|9$YrBvUY%3R6W1C=KN}NK}DS}$9|b9r@Oghi}T#EHH)h22XG>2 zK}BiZqJnj&j zR?n@I`Q3#TRi*Wniz|Y1uJ>0xwA8EL1C6Pf@4@3wnK((g!5-)Z0rI2&4}VhGoj7&k zG?>+yCv(`)DCiUlTUqT+9yfXH_z8g`;?bX_Q>O(4bGB4eJbKPlYPn%e12>@ej zs^?VB1(GaL5B}v!(^6~k`gxUg?(9XCRpsu2n(~UOx|6Kh^D7os1GuHt<${wAmbJOL zHC5#m)h^a8n_stRf#+j%;NZ&Y@`~CDK4hc18Y`==tcJE9CZVprvZ~6(%d6|$^XCyd zD(l_4(hKT*2Zg(^VsT|zMV-pkSK#;M7REOQd0vitS74U93Q&Q$49sV>g&faB)@1IG zAwam2g|Nru0$UUOZNtv`$>gF7U3mTvZaz8*ZRF;I3&u?uTQsjkCN7aSbQUOcmU@L@fyrkei8y>N_!ai?NQ*@@Ekd?pVs zmU(5o2DyG$x~^@47yT`RPoK;fsjFYOsI0!Srn-HD5C@8h-`qDjv7Qh%B{Pg&XeSq zBMy%y4*Tq;zZX)ghE;lw>XMtKm>T$P)v$79!IvM3SN*~dZ;!U{>PxF;Etu`2z4!;yA@*(Sa;*{STufUo8|TicIA7mhr)v-d-beicF`yq$+iD14YYO!5 zAJ_#;G9%BLccG_=B9@((1AXoCBq$}hUSs+%TvXkDjQ20AA(~c~60$U8iFOqW3497w z1i(sLiB}a1rLs2fa{WMCS6^DcNZ#Jx8it6i8MMu6)N4a=g|)ryr-Wgsagk3n|#Xju|*za%$Y>Eabr&#SLh?Aoi?#p z*~;qrp`}%eg6~truRbyz+SqFQ}{c z7GQH~7Y(f|D`j}ivM${Ywj~53yzab(HcT49EIO;}%9y06Lh$+XD(c(I!<}nO%e;k< z=$H}}J$-P`p@bQ%f5EJ(Va6N+4?h$xXL0M5^6+IfwHLI9=P(1dhY4#Hr3+?RHp%s2 zHP2t|dUAYx{ldyp+h)lS|DtME5_7BV3R^JEEU)o3s9Z32=)$?PhhI=OyKEu+f{77} zYpNDk9J}OL3oAA;cOSRpIFDXlRzGsdNKZpszCx> z3oXWj7q%J3gR3sI%x(Ph7R{N%ECuW@+f7pODN`nnJ<&ac55Qlx%O5N$aJZ_P8tb@o zs=RMqda$TAwSHioUFG>no!=FOMN`k7HeCa!cAuA3de8jn4R}n2S+02Bawc%wp0QW-Y7tqFT=-7FNi+2#M7)S=+^xJBJCf z)b+2v>Z)qIQNjB4g1VvgRkLR;sI03yk;TU$rORtrhH1&TptRObYzr&q(sQbia<<-} zJZoJGFJX}Q^ftKxOn|GXRo8G9@QH=mq7q{E(^H-10ni`Lc`i+j9Ou%O5Fcfj(D|2_J=8*jQ~@F(e;{CZl?Zzdwxma7e+A-<|NqnBAe|iI4x{xY%X8ZAC5q z_gH`2jyc9@?!o_<8BbSreY&dKm`l>_-X7khfe z&i1~?92fce}IBE$ChcZ)b8#hDD+bX#|50-5loQl2D&yl~ulP z$PMz5aiP9JKKx~5ys-A;oNs7+DEnrPfe{UlWgTh4H>*7b5fQ$*IYapb<)6EmGf$=L+db{NrNj~z?8k^@ga3(VOE}M^$4|l$syz@<=_@j4sQGU937Oj;b z2A8w>j4B9@%NHCr$IUz6gztsXd@C!B3IEwzNhbVzcvD+`T)4&NkrFz^>)Ekc`_jR@-BJvs&lmRxhlKa$yGj2_)nYx%8yk3W9Jy< zW9*)r(A47^4QDcqfzd;?Wz}WU3{tqY6Yx{?{0xA z0r$lBv%oxn(D($eOt~n}Bzi!JC~-+CV%xc2KzavD+1VtZSF#5z7ciis2Q&hhS5iEn zLBQYB(yV}e0zOUmD0>Ncp`SaQY}Z&dmPpDsnLRD0XCRRy{DVZ|RFR)qk&)j! zpL7K(J^}Rb(CD}m`F^g26gZmk`E&m!7$#~@;k`V zp6CFNaStE&kDQ`fyr;vhW@?5b8j9z7?rsi{!-GcgJt%qPZIWt zV>e~a3d&eaP-Z@5Dml`G2oz6bNWg#FyGb(AZ3oppNXPNbZNs9LE+Nu<5vfjqH2)3h zS1|F1{Zk8(%CO+?9%oKA{&<&)-Xm_>=UhSht$-3Tzaw2M0F9hmNjGEI)PhI*53O@f6ul6?uv;NjG8Jq?;@%1XrFj65mdI z&;@%pOz)E3=?JhvA5W-JdFYw+Joqn2p9^@EOe^Vo0XtEPU>NE&768FjrGL!(G1 z9u(@U89EL0L|@I&IRG;QHGc^(yG`zDD|a<=iygvs((FL)a@el?soa}E-xSEb2ViX= z_i=y+4Z$XQfJSdX)^CU=k=SWjeTXvQBn+)4V$7tyhZFQb%g9a{O42^;%axl3uPNwT zx3EliXTWJEW3+N<>o;WY`)umO}A%uRJppX*~1eIJ_S+&X+R3Hyq?8*-W_ZvcL1<(&cf^fW8a!v&1K9E+5k zpOel;0L|<0xAA<2PMi=?Gp}@9WEVlQ%}n0lHIrwy+Gd6qQn1=K8Qg5g0aj&1}QH0PZ=dH;+QTdX^~lwvk@(n%M~1Z)X3XnIAy@r`JpmCplvD zv)ydvL4x9%OV7TNXTnhkD+kUfe{U)-n7gXqh%cn zyq5L!TNarJ!#J;HZ8(NWWTrGqJE)N*qETcHX|~rgHb{);#P)V;+^3&k({-I*P28gEDI2|fJS8aGYSL~|BJ-P zV}LJmSOtWaAUoN)r5|7Ce}%}|T~PYKK3nCWxcPbJTc-J>PvhnfGy84M7UWfI-!JId3Bw6olYfNmS^6ulH$sSnKwi(h$}BMZ-`nzMJDzp#x;{9 zGW7XFhXR33492c{g)A>y1OfAf;G#4lu5Jn=* z-jPo-%jV2A7g{l_*$^M?nE*W*24-hc0jTk$m@v;Z`xrbRdM=LxLc|E?%w(`j9OVDn zYIT;URjHY&%#66%W~Ml1*oN4}tNk{RPXX{mo06sM)_I6PJL5wFc%&M4f|%s_oj zW+ScoZ)H~1n3Dr#SE9oWne!^m$<|;%&7@TlZL)^(F$0Yb#8Ko}Pa#;3^;j7iS_Gi7 zFFLIV-#c_+<>2|S$p5vV)GQrcOS((Imt@wH?l=13trk{JEsPmbGwpnTePmI=nku>x z_D9u+Ea{2;MX*m2y9r(8Z9_!1F?zji2bK-1a1U8=ec9_Mv=5D z7R8?<7&v8i$aB1q=W}tv{?~|*@0S!95f|)$O$vEV8}i&S3*B(hN(!JhI_j-3o(7S_WzQIF+Zp}&e`ZL|@&*Q!RO!xW|mucJ1 zR6f|wB;U53Nxp46lYFq9St-1~ovD1Voq2|M{O!y))mix<+O{*9YVCF=w)hVQ3T20H z7Jq^1*nT@xlyAcRex`uu!v2OPK={V6zoRM2u&}?SDazl%w_0hL0Fm3n{-&lVMPYwe zQDBHHdDWSmz zr}DuDC-!7OYN;6dBZA|*hczU*m;s2+4Qot>9#o|4d}mmLGW2pn$F@NkdN)~Ymxk@A zEYLWJP7d2)S&$)Mb=Z!}0@*WK8ny$opr^sXLL(*YS>+z|9Bi!ePZG|}86s^yQ8_a2 zGB{|VaNE$9(HUlVhI=3qZ8bhUZR6^~8ld4m(l*jNY)2?NM-r<+ANu1-e7qR;Hg78t z6}Kxb0yR9-ii&jTZ$)+Fw3CT+7_91T>98)c!ZS$=tc{PB$nehzlJQBAoJ&B?Vz+>YAMbp}`<%c|-eW7%|h7WbVq&;oe?Y(NhZP&v}-do7I!{RwS``;I~* zBK_2ePEJ0PRAjhqM2qb5ai&8IiCkn=8lh#}8Kz?;^^VXCgsNrS(BK8~00lbrZ6Xtq z9{D}#s?ip1z~JE?qBNmHbfr_BE28qFof>N)8TDF_&scqqH>eU<=A73R)PquG*A8Jy zorUzU8Ei3IDP#9szwyfqMg1~ov&wvEIaZ=<+}`0-lPR}3!yL_LRv%@RyNeiVbCHZJ zas?-&!^W`FNei)pJb-tdbSZjKcr1eA#|}bJsRUIT!50k!KF{+GY_nY;@P98Pj#Q7D>GZb)8#jLiS=GXVvYi#Cy8V7Rp)&@BrznuEJm;yOjWA z(g1uiu5W0n^%UUAq*K7fSB9plXU+iLKw2lh2SU?ibd~!L>2q+=Peap=@^BrFXR$iC zD+{yGH1ri)hdHF#1%$0>Tvj~Vb*>5hjDZzt0=g5wxjkL@nkSb^bE(D9!hZ8Z1Y4v$f~K))hBzGyJ)*y{;xi zGoFO^USyr?uwgZ9ZI0v2V8hTk%Q?xCH&b980OL-!;-^}b)j=Z7lv)Lu_iV&tGGB>Z`ak3Pyr zzseE~v}(AC{{M56=ss=h9xmSSf{Pp7KQ1@?IZuavfeDlEz@B|B$7r~B$75y6r!&D^ zgFXAK1T}?pD)GF}osd2?CULJQC{Lg_9ZLqF@@b9k{mGIUG7rhYKU5c;1ND5;B1G~u zJ}?zPmys?ZFuB!Wb!XMB0dpJaMgg>_Exp0gp6{6hz+fii|lZ0y} z;oU&O3?xh!KOSWz5C}X84`48yYk7ray`A$p`;MP2^JOc3C3}=WqCC`dg?eb zdvT!9C6qCkvCo5;u?P$Hc}O2mVYxnkL4_+x%LHsAgKwm0c=y>w<`L2czdak4D?o(K z+Dv9vm#?BfmZCm#97ZG{rw@4_&ljlnl%#27V3$Cl^SH%a#Q|YCNsj zKm)CTHx&5>-cW1}?03-yZbZXXE(tG4!m>aDev#Tu66OUGFq`e<4d_D`iG~A3PgMpL zqGOe5ATt7KjJ{2~$p>S9>sCAZdY{gu=G~n6XGklt^s~|k|7U|`(;2^xI1OM^wrUtg zFkx?0OC3l$dOW~o@HJaL*Kvvznbx69@3-mIb+=I4d3gTO{&;`cRcEhz-$$Pupo=Ve zFX(k}pBbQQ)u46zkyb;odlLL&ivoIs==VrjOLu+x?*sZuiM?l#FZR9wz22e=ee|;d zy2YZ4ee~ab^eYJ#UFD;{4A7YtEz{CKo_^*B<(U@UA_8ReIfYPwCXN4^etuiI0{EHExh4hs~-yqWx1P0hn>$&3Axql1!b)tUD;3#|&* z`t+X%bnCG9KJ26G1N2@MyDtV>dfqq5r(f^WxAG80&@(}oiayV$KOE3qOJ5R{Z}jQg z1A3mNuMFs`efqwD9#d=XLoF?Z($fNT5iNI4wY2Aa+Ms|oQ?wD5h7ZWB>0<+0iTv=s z-9iT_R7zu=jT=6)Qf?UfWk{hYqazWq*DCsC6X^-f<-eqJd))&>`P6L!&?~JcDH933 zyIBdKLl-;Sk&ynZ3SA7McRzSorKaT>z0s#v3wWySv{^Fn-n2rJoV*vlnpRKxC9}3x zmXF?|nLwW`T6ZT>W*lm1J)8zEZKZ%4$UH>afB>zB(_SJyMYlY9AMMn7_|fgk&|teB z#-NY-{m_YZp!M}_uI%;b0VLp9oqlteV-1sIogkJ!C3q(7qyQMXcF(VQW)xT&W((5q zXNK2wv+GJ*=r|etPovrye4B5Ce_({)Q`bM$?)NYkwom(ync!LSPs?a>hU0vJAv^t; zX>BOevBn9ZVdr;MYA>a7Nc{w$d-`}%tO#aRx(5mKU@e5I!&~XINhQJMz{Vz7R-ah* z2Q1rJ>l|4&orPKYeBZL^mjh^mmu1s&hx8^ayRq4~?8eQ9S$3mb9hd8@01&x$yJw%r zBtUAv+_r)~L56uOt({?BYm{5KkF@v__WOPF>K2;=4HVQtz9!;QT+=7DsT+v|XipMuXQJi@EIiF>dSeDx9#An$8rPPjsOQ--> z?b=%vEJgUqq!Ve&6MNuMlzL(>nKO7e2Li0`iby|O;*%7!#9jx9PbPzt0#9c6#1q>Q zdj`ZTpLkX~;-G+-?-T3W5#_&DY@tuQwjFU&K%DLq?{7yuHz1b##FyF;tH32l+j^h4 zrycR)fVj*fc8#_pDqya|fu59CXk>F8CFgleO~d`V9tTd7*tU_@J7}F6MnHFc z`mBI%r@F^;1NulZSgPA^eEMtFz>mv-Xxr_E0BtWlAD)N09$OiN6+jS&|{XqGNAj`${c*CSPdY=W)=p;*4ttS z{9-!zW9j7q-5U9c%z*A2DRbE&VxJh~7rQwqW^Mn3Or~YP6TXo$gZ1BQ*7i>n2V#9A zW$rvwY`HJ?t3a&nk|$~du^JMzrh6x!zSXWSo@fljUhmV{qOeBXYF8IeGzavxK7Hz; z)yi)rw$ev$w>|x2i;r#y&_%MHj-Y$7V24j{4(KKF+b4I6jx^F$FJ|=G^+5Q z`RsoS*lR6%pU?h2=CY4^Q!H(hxhw(73vb#bje8y?^&DEyWMi; zpk(d1N#mn}`1k4(Nq!{FM+fom)wLFl*UJ(|sfWAK!)&`{@zw8G|9d#!S{4jzGsz?8 z`jUeFf3?L*()ghNdwTuv`N|_29VpW?=>JzYTi!;WSK+Un8$NQq&mLU=y}DKGPfhpH z3w*0=x4J)7;!9Z$y5pMOISDSx>Vp@sP^!u=b*Ennf5M_qEdhO<=ogb=X7AYoJ;aq6 zxJ|eCW#075cy6?5t&jefkM`VX6N6C3>y-)G+ps*68{x#8wn#uwI2U`fuYF7?OxYDZ z?N)d=n_+bZdA7jA$NG7;z9Yww9}J!?@5m_S6WC0TOHr;r#V--qyOThZk+9H&j_)m@ z+R5RrkE&^M@U`CEY&8x+iE1e0Y_lS=5WL^nmL7=vhY%k+Z?azq+iduDc(PU37YY0p z+c=;2wRx=e7PiOwwI%4bMs%n1Z^=A^gbvXsofpwTsykb3y%zhU!?Lya>BAQq=k`Hu zrf_bz&DUw7_#uvSQAbCuxXq#Qy=x@lvzHB%rwg&a0t^yqoiu;q}^vj?> zA9PLwn(sL2wtXz&d%vA97YUuM1lzpMR_6o+bnc3P!5Im2jnkFZMw&7bxM#+T3%H*L zWfG!Voe!;Co@*E50^Xjb>7%2`apRm%siV(X8ne3^Cc}V>o06fBUUEV-pwIatwX99x z@gBwr%f1C&$wjz1!_7m}X+Eip8zoLuv`Qd6tJ50xDPVXK0duXUoh|&V1X|eM`|2|n zFNr^?8SKZsW}D^#(m83NxdUzce4NCu{LP(-l1jV!JeV}stf!Cq{L5q{aY*c`ZZvjc z65Ewep=V^G9Pw9wwhoZQ{lGtW20jC1{HzwMwRNrVwfI+f=0Mta)@5XM|JOZ@x-UbM za39#CjlWfu1xb~r(bjD%yQ;0St88VeVY#h~B5@}#_nij0@zWMd+kYYVQXC?_`{I*i zDa0+Yy_by|h~#B>w|=lfybFRb4kts~5FEd}a(YVVictJD+> zvYFYYpimhbf-X{*d^{5Pm|pc}BwgC?7)zLlj`5RR6_>V~HO9`Es@~=)kxc*>t5f>j z$aSEud%L_tdP~3!WLio8i2&Tr(tH4^7kZ{JO6-n8%KOUrEjH5>H)sVqm?q~IMD(WL zy8ePRm6ll=@twanok6%_zpn$9J78HQ1FT2hO~;XWj!Y27U)q7^fGZJ^voGuiOi#7v zJloAf*gL&mg|IZnu#R%|ZvLtk7cl0{4V~=or#^W< z6XpJqDYvml7CzY{>g9_PWSLKPcL&%*+SP-P!D)6hZt*l2tC9IG#p7ODp^>snC7#`Z zN12D5ZE=n6QN?8KsZHm?o(t;cnr$#U>g$(Ce@TI39;L^F4EEjMY&1gG`e-e^Lk)yn zaCDDlt@KU3rA1@msAZskEjD`9sG)n*e-xuf9}i!<_t#MboudamIt0uUx6mSIe+XPT*I7rC`qQFr5wIVrwo70F#%yu? z{KC8QBMAdQqP+=^Ye7P&8u4+*rU_0YHwrm1(uranFV--mGk*sdnxe|EShsUYGb0ey zv~z4VI859*R73cN0OZO5SEVKRhIm z(ukDHCFMS_-o8tfnv?l(ASI!S8@n{21>{~f*!TdU87@2>DbW{^a6RcddqxJXqw--0 z*S?^7{;jTiF(|%YO_2?X?j~G>{}~W%JNK(r5l5~gScbO-8m>b2Eoj)CehhC?JH7DR zJ)=#-!BAkJ-spB1B5OM5Fs4BrRp_I_pvY1p8AjXr4NudAh)(&%byep#sPie-?S9f) zzsI`U&TskyVB|Duq|R51U!bG9w{<>mF7i76s5Kcmmq$3Wo|8RhZB7pzUT^OQes6T& ziy^wpjw4@`5>Ezg-Hqh8X{!cuL$DKBA*Xef~K8B#uxlo(RJ zY$v6ty%gUVJ=P;2-26NXvgoC4#%Kj1isb8zVE5y@(8q(c?Lp zKt}IDC)$zlKNy2T-Lk~0%pPQJCzH9=$8;|aGTZ&kQLcDQPK03i#}0gabGHyli_P6A z!OO?MMYrKD(F5RC^#hlat~Et-l5pwO=5|^|)K$EE7oj?HVg`3pxwySI@-cvNH)2Y2 zW_qQ$$zxVU7d1f~53xza(;!}Vkht&)i035LnJb93gf0~r`)hF3NoD5ObT2jX8UNu0 zjc(^OKz`pv_=~%AL8IGgFK|zVzqs3pUx{xyHtwJ1OxN$#??h(wZRBD4?&lCuqaTH?Ek={*S0HXCU7Y}+ zLQTv9eDFx=RfO@xo0zgGNsmT9(#Kc$rTVl>6LXHo=iJd5rnvGvy0Z(qCVJ~`yP4PJ zJ}5S}CFCKK)^w?t0R3P?lTMf2VCOlvNIxQP_b*8wU^vPK ztD!?zoze&o)r^wVkE|T4M;WOUO}f$#N$GMTZ${~g_P!>xzc&npRHGRs&Mr~u5|CYR z7K@Bpq%`1$led;kDP~9wT1k~#=%xs**~)j{28XI6f50b=*{^)UVDd+8xkVv6;Y9L1 z_Y~q)+K;en(v-!>8GLo4+bfUpmoWJdK?%PYY6&XP^?Sd-s^fsPh)rY|YZ2S4f#v=n zq6#F@Bbp; zUwVRq_lp20l5o5ZABXHO0y6ao1^*WT6VjCTei1MsLwWBP0Tc8H1^*WT6MEW+eX*+j z&VQzZ)c<-UANC5d^#tXJzL|=66AYYA;xP#Dei1MkS^h5qtb+b80;WpAi1&*C&Ko1r z>7_b8?x$a@NRr)t_+olL{ip@g`fzbSo> zR6voqI_F}hv(Qw@evSu08(lsvLnCLhP|xngwIchMNfp@sK!xo-n$-iL+fMSH#_}^* z!$V_WT9j-T*$-Y#;iAjwj^f}M{9F@4%;wS(s*@pm{DyQvN49yE$n3)0L0ZLQ zc6pH|PcXs=g?7Z8#PKKytzXr^=JupJC|Od6os=JmXZ$zBo*ejid)V))=Qo z7)gGI^d?Qus74`#m`I7U5Mo#XC<9H7iokI|z!dMMQv%vw<+qQ3?kcWn``-Ei?M*oE6?o zB+hC^$(TdTj7$sUKLOh#fqW*fqh1Z<{{vuWAo?SK4+7EO0(@bxY5(26P5bYonpF`F z4qc%GbAe??W7##1Gc1{UCr~#t(XGaoyO=Ze2$;me=MZexT0bVm9h5pfbY%@XFwa$scRx;B@yQT1Z z<#G5nMcH>8C#Fih_A7d6|r7t#d@1kYayD{WHsFQd8f$q6edz}HC{F65&P%uU2<`(=q@e~!quX%!pwtC|}#$oN#JQ7cX z!o%nYubCmmbkN8j*@Bxtq}>@`2MCjpD!3z$B{lA`& zjvbvum0oyCuSIPGe?SHUpAVK`4xb5=RPZENE+;hzkhoPO8?^{5z1EWMqCJW!y&fc~ zixg9OJwtk2CeFotU0#*-8#LYnL#Jbrvz?v6a8JzUxmd>8IT**RiYj-UEF0C3cXnc7 zbAA}F`G>W47|Z{|mJBln|G@-iCZvYxpnt04CIBbi`M$E1WD~CbAiySk_5s$i3@QKl z;234`lEkj!`f?8%nF(P#JU{mG3rd$5=VNcaKt2`x==-sLj-U9vv%O9IQ=)7sZ66@* z_56UuleX|tBAs4g06KpUL;jo1X-S*#&7)L?hYe&3@qXo+_{Xej^Qe}dG`1JRr`LON zmFCet9-?;#R*CWW?!evZb!GOm%=}!!60>m-2^-|{eL(J-^m*Jovre`;gL2uVqlUpa z@ty!R9n@50%5w&(pQFF}BebM}>SY{9$7*|@E8q$;%Sl(_IMH9JcYgnuwm`yD08dE5)q#Y+fO+1Nz=SqvD+51z>w}W;R~5qTdcQ%^hK<~= z_rFM=3Bc`oA0T}vakokwzSoC>ec}b+dwuBSK6DgT8Z^o`*PudK1$Wdsn!|R^3~#)v zGiq*=p>X|qknBn!d2Jj|{PeEuCm?JRX?$4s8=TuyV4gt3JC38o5`f5ecK~*H1?4)H z!ibbUvjxl`b2aI=0_KxhN4gWWBj2H^&i?m#m9$B~C9Ey_kvgGw$E#>rcVRKSdm*V- zSQL*I2zwKlyGYG=OUE_zZpZbF?lI@0iYxL4zMmmI2|~XyGRbQp(_1Dw_0A6Hl1%v0 zuWsE>1U$wK3&i(L74RGxJf*MvAo&$ClSva~>8XfL{t&2+R3-okeOpNDDVhQ=np}<7 zkRzI0ou*uBm=3;1(L(Ehj(rtfuB94<1)g*1T#Et>&s5>B_wc#^ajv)cFpuZ@z2b}N zg_`gy=e#!MHd*D)Yg6vLcFLV+m6JkR&v^>f`3hP2i=-0TStTyE6%(=|H(zW+lT~La zouJ6U2*50R@HTn0-YewTXN9C(Pk5Pc`7z)Hq#DA@eELJB`r!oq9*8S5^K*Q}M0LDcBe`BEGT>z`4p`OxXHDtExf4=}}_yXw}0n%_6=?!1Q=tg0s;lD{= z3qZrHFu)JgN&O0;Lq@6oXMagUQ>&>`AvEh)l8vWWDC=jWvsM4Ih)@X4!jZFZWZF15 z&$f|%P5i|9Mc^{2+cl;9VgP0Ag!l`%@C}xxC5bl2(zL{Si9DyIn{cUNJ)hQ@ZT&@F zI@=cVoTB0!Sg_x)ChTV~Thg}U6z62BB+4uyRpS)r^z{)NLEMj1oRj4v_JG)ZG|KW6 zs$r`geL$Ia4%Scm6~U;}0DT;*kI|UIvLVDwA5WuoE;Jj(vK6K~VVrCvWoHW*O@?5Q zjk4g6zMNw)J{OzQ%IVx(!U{tQM1pM0HQ=TE}yLk)Cph7UP`)H zwjI_CrpOM8ug*pQ?H@uzolH%W=^yxw8PZ5~LmHu%pN*R7r>Y4*ZOjnLnBum^+(RX+ zRihe1})kry}{Lpdp{a@^R3R{Q%#@i^*?IP6ClT;mWmXms<587 zns}wtgX@-F4NR;L%i^6|ZhI1R)E++GF5Gjrz5 zX;Yq-yqK$*#Y%;_e{F%}KEkvA8BW@zp$lSFFH4@~g)Y#XG6Hi7T|f*7-Oge@bU}A; zgTqt2&}g_%o`>axF7y>B8t;VkZWzD2J4EAbx`W0tQpk@2>YuCnv%P+ZVHumwBWT