diff --git a/drivers/gpu/drm/asahi/file.rs b/drivers/gpu/drm/asahi/file.rs index 24e8db30e580e..a09fa33ed02a4 100644 --- a/drivers/gpu/drm/asahi/file.rs +++ b/drivers/gpu/drm/asahi/file.rs @@ -1350,9 +1350,15 @@ impl File { cls_pr_debug!(Errors, "queue_create: Invalid priority\n"); return Err(EINVAL); } - // HIGH has only been run on G17; G13 and G14 keep refusing it. + // Keep support tied to a backend and admitted profile. G13/G14 and + // unqualified M3 profiles retain their previous rejection. + let high_supported = reg_data.gpu.as_any().is::(); + #[cfg(CONFIG_DRM_ASAHI_M3_T8122)] + let high_supported = high_supported || reg_data.gpu.as_any() + .downcast_ref::() + .is_some_and(|gpu| gpu.supports_high_priority()); if data.priority == uapi::drm_asahi_priority_DRM_ASAHI_PRIORITY_HIGH - && !reg_data.gpu.as_any().is::() + && !high_supported { cls_pr_debug!( Errors, diff --git a/drivers/gpu/drm/asahi/m3/m3_board.rs b/drivers/gpu/drm/asahi/m3/m3_board.rs index c44b2a42d99d5..5d4ea4277b618 100644 --- a/drivers/gpu/drm/asahi/m3/m3_board.rs +++ b/drivers/gpu/drm/asahi/m3/m3_board.rs @@ -13,7 +13,7 @@ use crate::m3::m3_resources::{ use crate::m3::m3_soc::Soc; /// Whether the root node's compatible list contains `compatible`. -fn board_is(compatible: &[u8]) -> bool { +pub(crate) fn board_is(compatible: &[u8]) -> bool { let Some(root) = kernel::of::root() else { return false; }; diff --git a/drivers/gpu/drm/asahi/m3/m3_completion.rs b/drivers/gpu/drm/asahi/m3/m3_completion.rs index e06fcded9047e..387aaeca1b6c6 100644 --- a/drivers/gpu/drm/asahi/m3/m3_completion.rs +++ b/drivers/gpu/drm/asahi/m3/m3_completion.rs @@ -24,10 +24,9 @@ impl WorkItem for Completion { type Pointer = Arc; fn run(owner: Arc) { use crate::m3::m3_runtime::Service; - // Snapshots after a notification before sleeping for the next one: the completion - // stamps can trail the firmware's event message. - const TRAILING_POLLS: u32 = 4; - let (mut trailing, mut waited_at) = (0, u64::MAX); + // No deliberate spin: fsleep(5) selects udelay, not a scheduler sleep. + // Check immediately on IRQ, with backoff for trailing coherent witnesses. + let mut policy = super::m3_completion_policy::WaitPolicy::new(); loop { let (outcome, events) = { let mut guard = owner.shared.lock(); @@ -37,23 +36,20 @@ impl WorkItem for Completion { // Preserve the root-only CPU-overlap diagnostic: disabled waits retain // the runtime lock so VM/timestamp preparation cannot overlap the GPU. if let Service::Waiting(messages)=outcome { - if !events.cpu_overlap() {events.wait_past(messages);continue;} + if !events.cpu_overlap() { + events.wait_past(messages, policy.next_wait_us(events.wait_ceiling_us())); + continue; + } } (outcome,events) }; match outcome { Service::Idle => break, - Service::Progress => { trailing = 0; waited_at = u64::MAX; } + Service::Progress => policy.progress(), Service::Waiting(messages) => { - if messages != waited_at { waited_at = messages; trailing = 0; } - if trailing < TRAILING_POLLS { - trailing += 1; - kernel::time::delay::fsleep(kernel::time::Delta::from_micros(5)); - } else { - // Until the next firmware event, or a bounded 100 us resnapshot. - events.wait_past(messages); - trailing = 0; - } + // IRQs and timer expiry alone are not retirement progress. + // Keep backing off until service publishes or retires work. + events.wait_past(messages, policy.next_wait_us(events.wait_ceiling_us())); } } } diff --git a/drivers/gpu/drm/asahi/m3/m3_completion_policy.rs b/drivers/gpu/drm/asahi/m3/m3_completion_policy.rs new file mode 100644 index 0000000000000..9c0da8c3246e3 --- /dev/null +++ b/drivers/gpu/drm/asahi/m3/m3_completion_policy.rs @@ -0,0 +1,77 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Bounded fallback for witnesses which can become visible after the last IRQ. +//! +//! Notifications request a fresh snapshot, but do not prove witness progress. +//! Only publication or retirement resets this policy. In particular, neither a +//! timer wake nor a stream of early firmware notifications renews a spin budget. + +pub(crate) struct WaitPolicy { + next_us: u64, +} + +impl WaitPolicy { + pub(crate) const fn new() -> Self { + Self { next_us: 25 } + } + + pub(crate) fn progress(&mut self) { + self.next_us = 25; + } + + /// No deliberate busy delay: IRQs still wake each sleep immediately. + /// The caller supplies the root-selected ceiling (25..=1000 us). + pub(crate) fn next_wait_us(&mut self, ceiling_us: u64) -> u64 { + let wait = self.next_us.min(ceiling_us.clamp(25, 1000)); + self.next_us = (wait * 2).min(1000); + wait + } +} + +#[cfg_attr(not(test), kernel::macros::kunit_tests(asahi_m3_completion_policy))] +mod tests { + use super::WaitPolicy; + + #[test] + fn no_progress_backs_off_and_stays_bounded() { + let mut policy = WaitPolicy::new(); + for expected in [25, 50, 100, 200, 400, 800, 1000, 1000] { + assert_eq!(policy.next_wait_us(1000), expected); + } + for _ in 0..100_000 { + assert_eq!(policy.next_wait_us(1000), 1000); + } + } + + #[test] + fn publication_or_retirement_restarts_fast_detection() { + let mut policy = WaitPolicy::new(); + for _ in 0..8 { policy.next_wait_us(1000); } + policy.progress(); + assert_eq!(policy.next_wait_us(1000), 25); + assert_eq!(policy.next_wait_us(1000), 50); + } + + #[test] + fn live_ceiling_changes_apply_to_the_next_wait() { + let mut policy = WaitPolicy::new(); + for _ in 0..8 { policy.next_wait_us(1000); } + assert_eq!(policy.next_wait_us(25), 25); + assert_eq!(policy.next_wait_us(1000), 50); + assert_eq!(policy.next_wait_us(100), 100); + assert_eq!(policy.next_wait_us(100), 100); + } + + #[test] + fn fallback_keeps_timeout_checks_frequent_without_any_irq() { + let mut policy = WaitPolicy::new(); + let mut elapsed_us = 0; + while elapsed_us < 2_000_000 { + let wait = policy.next_wait_us(1000); + assert!(wait <= 1000); + elapsed_us += wait; + } + // This bounds the requested delay, not scheduler dispatch latency. + assert!(elapsed_us < 2_001_000); + } +} diff --git a/drivers/gpu/drm/asahi/m3/m3_compute.rs b/drivers/gpu/drm/asahi/m3/m3_compute.rs index 34aa0c0d2cd3f..b6e9444ed5449 100644 --- a/drivers/gpu/drm/asahi/m3/m3_compute.rs +++ b/drivers/gpu/drm/asahi/m3/m3_compute.rs @@ -40,25 +40,28 @@ pub(crate) struct Compute { barriers: Option, /// The register lists of the SoC's GPU (`m3_soc::Soc::registers`). registers: command::RegisterSet, + profile: super::m3_profile::Profile, /// Last: drops after the command aliases. _lower_tlbi: crate::m3::m3_memory::LowerTlbi, } impl Compute { pub(crate) fn new(dev: &driver::AsahiDrmDevice, uat: &mmu::Uat, vm: &mmu::Vm, stats: Region, control: crate::m3::agx_compute::Control, - registers: command::RegisterSet) -> Result { + registers: command::RegisterSet, profile: super::m3_profile::Profile) -> Result { let lower_tlbi=crate::m3::m3_memory::LowerTlbi; let binding=uat.bind(vm)?; let mut objects=KVec::new(); let mut client_command=None; let mut command_bytes = KVec::new(); - command_bytes.resize(command::SIZE, 0u8, GFP_KERNEL)?; + command_bytes.resize(if profile == super::m3_profile::Profile::Extended { + command::EXTENDED_SIZE + } else { command::SIZE }, 0u8, GFP_KERNEL)?; let mut sequence_bytes = KVec::new(); sequence_bytes.resize(microsequence::STORAGE, 0u8, GFP_KERNEL)?; // A private compute aperture, disjoint from the retained render graph. // Address-bearing graph fields are constructed from their live owners. for index in 0..storage::COUNT { - let a=storage::batch_allocation(index).map_err(|_|EINVAL)?; + let a=storage::batch_allocation_for(profile,index).map_err(|_|EINVAL)?; let mut b=match a.mapping { Mapping::CommandAlias(va)=>Buffer::at(dev,uat.kernel_vm(),Some((uat.kernel_lower_vm(),va)),None,a.size,true)?, Mapping::Client(va)=>Buffer::at(dev,uat.kernel_vm(),Some((vm,va)),None,a.size,true)?, @@ -68,10 +71,13 @@ impl Compute { if matches!(a.mapping,Mapping::CommandAlias(_)) {client_command=Some(b.map_gpu_view(vm)?);} objects.push(b,GFP_KERNEL)?; } - Self::encode_command(&mut objects, &mut command_bytes, binding.slot(), control, 1, 0, registers)?; - Self::encode_sequence(&mut objects, &mut sequence_bytes, stats, binding.slot(), timeline::stamp(0xc1000000,1), 0)?; - UMA_POOL.initialize(&mut objects)?; - Self::initialize_state(&mut objects, binding.slot())?; + Self::encode_command(&mut objects, &mut command_bytes, binding.slot(), control, 1, 0, + registers, profile, binding.generation())?; + Self::encode_sequence(&mut objects, &mut sequence_bytes, stats, binding.slot(), timeline::stamp(0xc1000000,1), 0, profile, 0)?; + if profile == super::m3_profile::Profile::Extended { + UMA_POOL.initialize_extended(&mut objects, 1, true)?; + } else { UMA_POOL.initialize(&mut objects)?; } + Self::initialize_state(&mut objects, binding.slot(), profile)?; // Queue references are firmware VAs from live allocations. In contrast // to render's unpublished seed entries, compute dispatches entry zero // immediately, so its initial producer cursor remains one. @@ -80,8 +86,13 @@ impl Compute { job_list: address(JOB_LIST)?, gpu_scratch: address(QUEUE_SCRATCH)?, context: address(GPU_CONTEXT)?, uuid: 0xc10000 }; let command = address(storage::COMMAND)?; - let mut bytes = [0; queue::INFO_SIZE]; - info.encode(&mut bytes).map_err(|_| EINVAL)?; + let mut bytes = KVec::new(); + bytes.resize(if profile == super::m3_profile::Profile::Extended { + queue::QueueInfo::EXTENDED_SIZE + } else { queue::INFO_SIZE }, 0u8, GFP_KERNEL)?; + if profile == super::m3_profile::Profile::Extended { + info.encode_extended(address(storage::QUEUE)?, &mut bytes).map_err(|_| EINVAL)?; + } else { info.encode(&mut bytes).map_err(|_| EINVAL)?; } objects[storage::QUEUE].write(0, &bytes)?; let mut state = [0; queue::STATE_SIZE]; queue::RingState { capacity: queue::QUALIFIED_CAPACITY, done: 0, read: 0, write: 1 } @@ -107,54 +118,74 @@ impl Compute { // the invalidation. crate::mem::sync();crate::mem::tlbi_all();crate::mem::sync(); unsafe {core::arch::asm!("isb",options(nostack,preserves_flags))}; - Ok(Self {_client_command:client_command.ok_or(EINVAL)?,objects,command_bytes,sequence_bytes,_binding:binding,stats,last_progress_ns:0,sequence:1,head:1,cached_views:KVec::new(),slot:0,batch_count:1,checkpoint:(0,0,0),barriers:None,registers,_lower_tlbi:lower_tlbi}) + Ok(Self {_client_command:client_command.ok_or(EINVAL)?,objects,command_bytes,sequence_bytes,_binding:binding,stats,last_progress_ns:0,sequence:1,head:1,cached_views:KVec::new(),slot:0,batch_count:1,checkpoint:(0,0,0),barriers:None,registers,profile,_lower_tlbi:lower_tlbi}) } fn encode_command(objects: &mut [Buffer], bytes: &mut [u8], context: u32, control: crate::m3::agx_compute::Control, sequence: u64, slot: usize, - register_set: command::RegisterSet) -> Result { - let offset = |owner:usize| storage::slot_offset(owner,slot).map_err(|_|EINVAL); + register_set: command::RegisterSet, profile: super::m3_profile::Profile, generation: u8) -> Result { + let offset = |owner:usize| storage::slot_offset_for(profile,owner,slot).map_err(|_|EINVAL); let fw = |owner: usize| queue::FirmwareVa::new(objects[owner].va()+offset(owner)? as u64).map_err(|_| EINVAL); let preemption = &objects[PREEMPTION]; let value = command::Command { context, counter: timeline::previous(sequence), notifier: fw(NOTIFIER)?, - preemption: GpuRegion::new(preemption.gpu_va()?+offset(PREEMPTION)? as u64, storage::allocation(PREEMPTION).map_err(|_|EINVAL)?.size).map_err(|_| EINVAL)?, + preemption: GpuRegion::new(preemption.gpu_va()?+offset(PREEMPTION)? as u64, storage::allocation_for(profile,PREEMPTION).map_err(|_|EINVAL)?.size).map_err(|_| EINVAL)?, cdm: command::GpuVa::new(control.base).map_err(|_| EINVAL)?, cdm_last: command::GpuVa::new(control.end.checked_sub(4).ok_or(EINVAL)?).map_err(|_| EINVAL)?, gpu_alias: command::GpuVa::new(objects[storage::COMMAND].gpu_va()?+offset(storage::COMMAND)? as u64).map_err(|_| EINVAL)?, - microsequence: fw(storage::MICROSEQUENCE)?, microsequence_size: microsequence::LENGTH as u32, + microsequence: fw(storage::MICROSEQUENCE)?, microsequence_size: + if profile == super::m3_profile::Profile::Extended { microsequence::Sequence::EXTENDED_LENGTH as u32 } + else { microsequence::LENGTH as u32 }, stamp: fw(storage::STAMP)?, fw_stamp: fw(storage::FW_STAMP)?, stamp_value: timeline::stamp(0xc1000000,sequence), timestamps: [fw(TIMESTAMPS[0])?, fw(TIMESTAMPS[1])?], pool: fw(storage::POOL)?, tail_scratch: fw(TAIL_SCRATCH)?, flush_stamps: false, register_set, }; - value.encode(bytes).map_err(|_| EINVAL)?; + value.encode_with_usc(profile, generation, 0, + if profile == super::m3_profile::Profile::Extended { control.usc_base } else { 0 }, bytes).map_err(|_| EINVAL)?; objects[storage::COMMAND].write(offset(storage::COMMAND)?, bytes) } fn encode_sequence(objects: &mut [Buffer], bytes: &mut [u8], stats: Region, - context: u32, stamp_value: u32, slot: usize) -> Result { - let offset = |owner:usize| storage::slot_offset(owner,slot).map_err(|_|EINVAL); + context: u32, stamp_value: u32, slot: usize, profile: super::m3_profile::Profile, + queue_sequence: u64) -> Result { + let offset = |owner:usize| storage::slot_offset_for(profile,owner,slot).map_err(|_|EINVAL); let fw = |owner: usize| queue::FirmwareVa::new(objects[owner].va()+offset(owner)? as u64).map_err(|_| EINVAL); - let region = |owner: usize| Region::new(objects[owner].va()+offset(owner)? as u64, storage::allocation(owner).map_err(|_|EINVAL)?.size).map_err(|_| EINVAL); + let region = |owner: usize| Region::new(objects[owner].va()+offset(owner)? as u64, storage::allocation_for(profile,owner).map_err(|_|EINVAL)?.size).map_err(|_| EINVAL); let value = microsequence::Sequence { command: region(storage::COMMAND)?, stats, notifier: region(NOTIFIER)?, - queue: fw(storage::QUEUE)?, context, generation: 1, pool: fw(storage::POOL)?, + queue: fw(storage::QUEUE)?, context, + generation: if profile == super::m3_profile::Profile::Extended { 0 } else { 1 }, pool: fw(storage::POOL)?, scratch: [fw(MICROSEQUENCE_SCRATCH[0])?, fw(MICROSEQUENCE_SCRATCH[1])?, fw(MICROSEQUENCE_SCRATCH[2])?, fw(MICROSEQUENCE_SCRATCH[3])?, fw(MICROSEQUENCE_SCRATCH[4])?], fw_stamp: fw(storage::FW_STAMP)?, stamp_value, }; - value.encode(bytes).map_err(|_| EINVAL)?; + if profile == super::m3_profile::Profile::Extended { + let auxiliary = command::GpuVa::new(objects[TAIL_SCRATCH].gpu_va()? + offset(TAIL_SCRATCH)? as u64).map_err(|_| EINVAL)?; + let context_store = GpuRegion::new(objects[MICROSEQUENCE_SCRATCH[1]].gpu_va()? + + offset(MICROSEQUENCE_SCRATCH[1])? as u64, 0x40000).map_err(|_| EINVAL)?; + value.encode_extended(auxiliary, context_store, queue_sequence, bytes).map_err(|_| EINVAL)?; + } else { value.encode(bytes).map_err(|_| EINVAL)?; } objects[storage::MICROSEQUENCE].write(offset(storage::MICROSEQUENCE)?, bytes) } - fn initialize_state(objects: &mut [Buffer], context_slot: u32) -> Result { + fn initialize_state(objects: &mut [Buffer], context_slot: u32, profile: super::m3_profile::Profile) -> Result { let address = |owner: usize| queue::FirmwareVa::new(objects[owner].va()).map_err(|_| EINVAL); let notification = state::Notifier { threshold: address(storage::EVENT)?, context: context_slot }; let list_owner = address(JOB_LIST)?; let mut context = [0; state::CONTEXT_SIZE]; - state::context(&mut context).map_err(|_| EINVAL)?; - objects[GPU_CONTEXT].write(0, &context)?; - let mut notifier = [0; state::NOTIFIER_SIZE]; - notification.encode(&mut notifier).map_err(|_| EINVAL)?; + if profile == super::m3_profile::Profile::Extended { + state::context_extended(&mut context).map_err(|_| EINVAL)?; + } else { state::context(&mut context).map_err(|_| EINVAL)?; } + let context_size = if profile == super::m3_profile::Profile::Extended { + state::EXTENDED_CONTEXT_SIZE + } else { state::CONTEXT_SIZE }; + objects[GPU_CONTEXT].write(0, &context[..context_size])?; + let mut notifier = KVec::new(); + notifier.resize(if profile == super::m3_profile::Profile::Extended { + state::EXTENDED_NOTIFIER_SIZE + } else { state::NOTIFIER_SIZE }, 0u8, GFP_KERNEL)?; + if profile == super::m3_profile::Profile::Extended { + notification.encode_extended(&mut notifier).map_err(|_| EINVAL)?; + } else { notification.encode(&mut notifier).map_err(|_| EINVAL)?; } objects[NOTIFIER].write(0, ¬ifier)?; let mut list = [0; state::JOB_LIST_SIZE]; state::job_list(&mut list, list_owner).map_err(|_| EINVAL)?; @@ -182,7 +213,7 @@ impl Compute { let binding = uat.bind(vm)?; let mut views = KVec::new(); for i in 0..storage::COUNT { - if matches!(storage::allocation(i).map_err(|_|EINVAL)?.mapping,Mapping::Client(_)) { + if matches!(storage::allocation_for(self.profile,i).map_err(|_|EINVAL)?.mapping,Mapping::Client(_)) { views.push((i,self.objects[i].map_gpu_view(vm)?),GFP_KERNEL)?; } } @@ -221,18 +252,17 @@ impl Compute { if self.batch_count>=storage::SLOTS {return Err(E2BIG);} self.slot=self.batch_count; for i in storage::RESET { - let offset=storage::slot_offset(i,self.slot).map_err(|_|EINVAL)?; - let size=storage::allocation(i).map_err(|_|EINVAL)?.size; + let offset=storage::slot_offset_for(self.profile,i,self.slot).map_err(|_|EINVAL)?; + let size=storage::allocation_for(self.profile,i).map_err(|_|EINVAL)?.size; self.objects[i].fill_range(offset,size,if i==PREEMPTION {0xcc} else {0})?; } self.objects[NOTIFIER].u32(state::NOTIFIER_CONTEXT,self._binding.slot())?; self.sequence=self.sequence.checked_add(1).ok_or(EOVERFLOW)?; - Self::encode_command(&mut self.objects,&mut self.command_bytes,self._binding.slot(),control,self.sequence,self.slot,self.registers)?; + Self::encode_command(&mut self.objects,&mut self.command_bytes,self._binding.slot(),control,self.sequence,self.slot,self.registers,self.profile,self._binding.generation())?; let stamp=self.stamp(); - Self::encode_sequence(&mut self.objects,&mut self.sequence_bytes,self.stats,self._binding.slot(),stamp,self.slot)?; - self.objects[storage::EVENT].u32(0,self.sequence as u32)?; - self.objects[storage::COUNTER].u32(0,self.sequence as u32)?; - let address=self.objects[storage::COMMAND].va()+storage::slot_offset(storage::COMMAND,self.slot).map_err(|_|EINVAL)? as u64; + Self::encode_sequence(&mut self.objects,&mut self.sequence_bytes,self.stats,self._binding.slot(),stamp,self.slot,self.profile,self.sequence - 1)?; + self.write_counters()?; + let address=self.objects[storage::COMMAND].va()+storage::slot_offset_for(self.profile,storage::COMMAND,self.slot).map_err(|_|EINVAL)? as u64; self.objects[storage::RING].u64(usize::from(self.head)*8,address)?; self.head=(self.head+1)%queue::QUALIFIED_CAPACITY as u16; self.batch_count+=1; @@ -245,39 +275,55 @@ impl Compute { pub(crate) fn abort_batch(&mut self)->Result { (self.sequence,self.head,self.batch_count)=self.checkpoint; self.slot=self.batch_count.saturating_sub(1); - self.objects[storage::EVENT].u32(0,self.sequence as u32)?; - self.objects[storage::COUNTER].u32(0,self.sequence as u32)?; + self.write_counters()?; self.objects[storage::QUEUE_STATE].u32(queue::WRITE,u32::from(self.head))?; agx_memory::publish();Ok(()) } + fn write_counters(&mut self) -> Result { + for index in [storage::EVENT, storage::COUNTER] { + if self.profile == super::m3_profile::Profile::Extended { + self.objects[index].u64(0, self.sequence)?; + } else { self.objects[index].u32(0, self.sequence as u32)?; } + } + Ok(()) + } + fn offset(&self,owner:usize)->Result { - storage::slot_offset(owner,self.slot).map_err(|_|EINVAL) + storage::slot_offset_for(self.profile,owner,self.slot).map_err(|_|EINVAL) } /// Same cache-generation contract as M4: StartCompute and the persistent /// notifier must use the same tag before the firmware flushes user stamps. /// RTKit 2419: StartCompute.counter1 +0x34; JobMeta.flush_stamps +0x804. pub(crate) fn prepare_completion(&mut self,coalesce:bool)->Result { + if coalesce && self.profile == super::m3_profile::Profile::Extended { return Err(EINVAL); } let sequence=self.offset(storage::MICROSEQUENCE)?; let command=self.offset(storage::COMMAND)?; self.objects[storage::MICROSEQUENCE].u32(sequence+microsequence::GENERATION,0)?; - self.objects[storage::COMMAND].u32(command+command::FLUSH_STAMPS,u32::from(!coalesce))?; + self.objects[storage::COMMAND].u32(command+if self.profile == super::m3_profile::Profile::Extended { 0x81c } else { command::FLUSH_STAMPS },u32::from(!coalesce))?; agx_memory::publish();Ok(()) } pub(crate) fn set_user_timestamps(&mut self,addresses:[u64;2])->Result { - let offset=self.offset(storage::COMMAND)?+command::USER_TIMESTAMPS; + let offset=self.offset(storage::COMMAND)?+if self.profile == super::m3_profile::Profile::Extended { + 0x844 + } else { command::USER_TIMESTAMPS }; self.objects[storage::COMMAND].u64(offset,addresses[0])?; self.objects[storage::COMMAND].u64(offset+8,addresses[1])?; let pointer=if addresses==[0,0] {0} else {self.objects[storage::COMMAND].va()+offset as u64}; let sequence=self.offset(storage::MICROSEQUENCE)?; - for start in [microsequence::TIMESTAMP_START,microsequence::TIMESTAMP_END] { + for start in if self.profile == super::m3_profile::Profile::Extended { [0x1bc, 0x20c] } + else { [microsequence::TIMESTAMP_START,microsequence::TIMESTAMP_END] } { self.objects[storage::MICROSEQUENCE].u64(sequence+start+microsequence::USER_TIMESTAMP_POINTER,pointer)?; } agx_memory::publish();Ok(()) } pub(crate) fn set_attachments(&mut self,attachments:&crate::m3::agx_attachments::Attachments)->Result { let sequence=self.offset(storage::MICROSEQUENCE)?; - self.objects[storage::MICROSEQUENCE].write(sequence+microsequence::ATTACHMENTS,&attachments.0)?; - self.objects[storage::MICROSEQUENCE].write(sequence+microsequence::HAS_ATTACHMENTS,&[u8::from(attachments.count()!=0)])?; + self.objects[storage::MICROSEQUENCE].write(sequence+if self.profile == super::m3_profile::Profile::Extended { + microsequence::Sequence::EXTENDED_ATTACHMENTS + } else { microsequence::ATTACHMENTS },&attachments.0)?; + self.objects[storage::MICROSEQUENCE].write(sequence+if self.profile == super::m3_profile::Profile::Extended { + microsequence::Sequence::EXTENDED_HAS_ATTACHMENTS + } else { microsequence::HAS_ATTACHMENTS },&[u8::from(attachments.count()!=0)])?; agx_memory::publish();Ok(()) } fn stamp(&self) -> u32 { timeline::stamp(0xc1000000,self.sequence) } @@ -304,10 +350,12 @@ impl Compute { let barriers=self.barriers.as_mut().ok_or(EINVAL)?; let offset=self.slot*64; let mut bytes=[0;crate::m3::m3_sync_layout::BARRIER_SIZE]; - crate::m3::m3_sync_layout::Barrier { + let barrier=crate::m3::m3_sync_layout::Barrier { stamp:dependency.stamp,wait_value:dependency.value,event:dependency.event, self_value:value,uuid:0xc10000,dependency:crate::m3::m3_sync_layout::Dependency::PreviousPass, - }.encode(&mut bytes).map_err(|_|EINVAL)?; + }; + if self.profile==super::m3_profile::Profile::Extended {barrier.encode_extended(&mut bytes)} + else {barrier.encode(&mut bytes)}.map_err(|_|EINVAL)?; barriers.write(offset,&bytes)?; let address=barriers.va()+offset as u64; let (slots,next)=queue::insert_before_last(self.head,queue::QUALIFIED_CAPACITY).map_err(|_|EINVAL)?; @@ -319,26 +367,31 @@ impl Compute { agx_memory::publish();crate::mem::sync();Ok(()) } pub(crate) fn gpu_ns(&mut self)->Result { - let mut total=0; - for slot in 0..self.batch_count {total+=self.slot_gpu_ns(slot)?;} + let mut total: u64 = 0; + for slot in 0..self.batch_count { + total = total.checked_add(self.slot_gpu_ns(slot)?).ok_or(EOVERFLOW)?; + } Ok(total) } pub(crate) fn slot_gpu_ns(&mut self,slot:usize)->Result { if slot>=self.batch_count {return Err(EINVAL);} - let start=self.objects[TIMESTAMPS[0]].read_u64(storage::slot_offset(TIMESTAMPS[0],slot).map_err(|_|EINVAL)?)?; - let end=self.objects[TIMESTAMPS[1]].read_u64(storage::slot_offset(TIMESTAMPS[1],slot).map_err(|_|EINVAL)?)?; - Ok(if start!=0 && end>=start {(end-start)*1000/24} else {0}) + let start=self.objects[TIMESTAMPS[0]].read_u64(storage::slot_offset_for(self.profile,TIMESTAMPS[0],slot).map_err(|_|EINVAL)?)?; + let end=self.objects[TIMESTAMPS[1]].read_u64(storage::slot_offset_for(self.profile,TIMESTAMPS[1],slot).map_err(|_|EINVAL)?)?; + if start == 0 || end < start { return Ok(0); } + self.profile.gpu_duration_ns(end - start).ok_or(EOVERFLOW) } /// A batch span includes gaps between its compute commands, unlike gpu_ns. /// Read only after stamps, queues, events and idle state prove retirement. pub(crate) fn batch_gpu_span(&mut self)->Result<[u64;2]> { if self.batch_count == 0 {return Err(EINVAL);} - Ok([self.objects[TIMESTAMPS[0]].read_u64(storage::slot_offset(TIMESTAMPS[0],0).map_err(|_|EINVAL)?)?, - self.objects[TIMESTAMPS[1]].read_u64(storage::slot_offset(TIMESTAMPS[1],self.batch_count-1).map_err(|_|EINVAL)?)?]) + Ok([self.profile.profiler_ticks(self.objects[TIMESTAMPS[0]].read_u64(storage::slot_offset_for(self.profile,TIMESTAMPS[0],0).map_err(|_|EINVAL)?)?), + self.profile.profiler_ticks(self.objects[TIMESTAMPS[1]].read_u64(storage::slot_offset_for(self.profile,TIMESTAMPS[1],self.batch_count-1).map_err(|_|EINVAL)?)?)]) } pub(crate) fn head(&self) -> u16 { self.head } pub(crate) fn ordinal(&self)->u64 {self.sequence} - pub(crate) fn first(&self) -> bool { self.sequence == self.batch_count as u64 } + pub(crate) fn first(&self) -> bool { + self.profile == super::m3_profile::Profile::Legacy && self.sequence == self.batch_count as u64 + } pub(crate) fn progress(&mut self, dev: &driver::AsahiDrmDevice) -> Result { // Report real completions by elapsed time as well as sequence count; // batches need not land on a multiple of 128. @@ -361,10 +414,15 @@ impl Compute { pub(crate) fn queue(&self)->u64 {self.objects[storage::QUEUE].va()} pub(crate) fn complete(&mut self)->Result { - Ok(self.objects[storage::STAMP].read_u32(0)?==self.stamp() - && self.objects[storage::FW_STAMP].read_u32(0)?==self.stamp() - && self.objects[storage::QUEUE_STATE].read_u32(queue::DONE)?==u32::from(self.head) - && self.objects[storage::QUEUE_STATE].read_u32(queue::READ)?==u32::from(self.head)) + let stamped = self.objects[storage::STAMP].read_u32(0)? == self.stamp() + && self.objects[storage::FW_STAMP].read_u32(0)? == self.stamp() + && self.objects[storage::QUEUE_STATE].read_u32(queue::DONE)? == u32::from(self.head) + && self.objects[storage::QUEUE_STATE].read_u32(queue::READ)? == u32::from(self.head); + if !stamped { return Ok(false); } + if self.profile == super::m3_profile::Profile::Extended { + return Ok(self.objects[storage::POOL].read_u64(0x54)? == self.sequence); + } + Ok(true) } pub(crate) fn log(&mut self,dev:&driver::AsahiDrmDevice)->Result { for off in (0..512).step_by(128) { diff --git a/drivers/gpu/drm/asahi/m3/m3_config.rs b/drivers/gpu/drm/asahi/m3/m3_config.rs index bfb408b07a530..02b847e210dc9 100644 --- a/drivers/gpu/drm/asahi/m3/m3_config.rs +++ b/drivers/gpu/drm/asahi/m3/m3_config.rs @@ -177,6 +177,16 @@ impl Config { Ok(()) } + pub(crate) fn global_table_views(&mut self, vm: &mmu::Vm) -> Result> { + let mut mappings = KVec::new(); + if self.profile == super::m3_profile::Profile::Extended { + for index in [super::m3_init_26::GLOBAL_POOL, super::m3_init_26::GLOBAL_PARAMETERS] { + mappings.push(self.objects[index].map_gpu_view(vm)?, GFP_KERNEL)?; + } + } + Ok(mappings) + } + pub(crate) fn root(&self) -> u64 { self.objects[storage::ROOT].va() } pub(crate) fn log_globals_debug(&mut self,dev:&driver::AsahiDrmDevice)->Result { let value=self.objects[GLOBALS].read_u32(crate::m3::m3_globals_debug::OFFSET)?; @@ -236,6 +246,18 @@ impl Config { self.thermal.busy(now); Ok(()) } + pub(crate) fn render_stats(&self) -> Result<[init::Region;2]> { + if self.profile==super::m3_profile::Profile::Legacy { + let value=self.stats_region()?;return Ok([value;2]); + } + let region=|index:usize,offset:usize| { + let object=&self.objects[index]; + init::Region::new(object.va().checked_add(offset as u64).ok_or(EOVERFLOW)?, + object.size().checked_sub(offset).ok_or(EINVAL)?).map_err(|_|EINVAL) + }; + Ok([region(UNKNOWN_C0,4)?,region(UNKNOWN_C1,8)?]) + } + pub(crate) fn stats_region(&self) -> Result { let owner = &self.objects[if self.profile == super::m3_profile::Profile::Extended { UNKNOWN_C3 diff --git a/drivers/gpu/drm/asahi/m3/m3_device.rs b/drivers/gpu/drm/asahi/m3/m3_device.rs index 36a4d2c575f31..9897690d923dc 100644 --- a/drivers/gpu/drm/asahi/m3/m3_device.rs +++ b/drivers/gpu/drm/asahi/m3/m3_device.rs @@ -128,6 +128,7 @@ impl Device { } e.sgx_setup() } + None if device.firmware.profile() == super::m3_profile::Profile::Extended => None, None => Some(soc.sgx_setup.ok_or(ENODEV)?), }; if let Some((setup_offset, setup_value)) = setup { @@ -178,7 +179,6 @@ impl Device { if self.firmware.profile() == super::m3_profile::Profile::Extended { // Adaptive clock gating can leave individual engine banks inaccessible // even while the GPU performance-state register says powered up. - dev_info!(self.dev.as_ref(), "M3: adaptive firmware fault evidence uses retained metadata; engine banks may be gated\n"); return Ok(()); } let sgx_guard = self.sgx.try_access().ok_or(ENODEV)?; diff --git a/drivers/gpu/drm/asahi/m3/m3_drm.rs b/drivers/gpu/drm/asahi/m3/m3_drm.rs index c86813d1f7423..b11b612184ed9 100644 --- a/drivers/gpu/drm/asahi/m3/m3_drm.rs +++ b/drivers/gpu/drm/asahi/m3/m3_drm.rs @@ -38,7 +38,7 @@ struct CompletionView(Arc); impl kernel::debugfs::Writer for CompletionView { fn write(&self, f: &mut kernel::fmt::Formatter<'_>) -> kernel::fmt::Result { let (us, irqs, sleeps, timers, preparations) = self.0.snapshot(); - writeln!(f, "version=1 wait_us={} notifications={} sleeps={} timer_wakes={} cpu_preparations={}", + writeln!(f, "version=2 policy=adaptive wait_us={} notifications={} sleeps={} timer_wakes={} cpu_preparations={}", us, irqs, sleeps, timers, preparations) } } @@ -96,6 +96,11 @@ pub(crate) struct Backend { } impl Backend { + /// HIGH promises host publication ordering, not firmware preemption. + pub(crate) fn supports_high_priority(&self) -> bool { + self.soc.chip_id == 0x8122 && self.profile == super::m3_profile::Profile::Extended + } + /// Admits and starts the M3 runtime on `soc` (the T8122), with the DRM device `drm` that the /// caller registers afterwards. pub(crate) fn start( @@ -113,10 +118,12 @@ impl Backend { crate::m3::t8122_start::not_admitted(pdev.as_ref(), soc); })?; dev_info!(pdev.as_ref(), "M3: resource admission complete\n"); - // An armed T8122 start supplies the values its table leaves out. An unarmed T8122 - // needs a complete table, which it does not have: it is refused here. - let experiment = crate::m3::t8122_start::arm(pdev.as_ref(), soc)?; - if experiment.is_none() { + // Extended T8122 uses its dedicated initialization graph. Keep the + // legacy guarded-start and completeness checks for older interfaces. + let extended=crate::m3::t8122_start::is_t8122(soc) + && crate::m3::t8122_admission::loaded_profile(pdev)?==super::m3_profile::Profile::Extended; + let experiment = if extended {None} else {crate::m3::t8122_start::arm(pdev.as_ref(), soc)?}; + if experiment.is_none() && !extended { soc.require_complete(pdev.as_ref())?; } let experiment = experiment.as_ref(); diff --git a/drivers/gpu/drm/asahi/m3/m3_firmware.rs b/drivers/gpu/drm/asahi/m3/m3_firmware.rs index 6bc87a33aaa05..94a2d3384d08a 100644 --- a/drivers/gpu/drm/asahi/m3/m3_firmware.rs +++ b/drivers/gpu/drm/asahi/m3/m3_firmware.rs @@ -62,7 +62,7 @@ pub(crate) static KNOWN_IMAGES_T8122: [KnownImage; 1] = [KnownImage { initdata_magic: Some(G15_V14_8_3_INITDATA), }]; -/// Image record for the extended format. The SoC's runtime selection does not use it yet. +/// Exact normalized image identity for the extended firmware interface. pub(crate) static T8122_IMAGES_26: [KnownImage; 1] = [KnownImage { name: "T8122 C0 firmware-compat 26.6.2", uuid: None, @@ -164,7 +164,7 @@ pub(crate) static T8122_LAYOUT: Layout = Layout { pub(crate) const TEXT_SHA256: [u8; 32] = [ 0x11, 0xe4, 0x9f, 0x75, 0xb, 0x67, 0x1a, 0x2b, 0x13, 0xd0, 0x92, 0xdb, 0x1c, 0xbc, 0xa3, 0xa8, 0x77, 0x55, 0x86, 0xd6, 0xb1, 0xe3, 0xaa, 0xe8, 0xa6, 0xa4, 0x64, 0xa7, 0x80, 0x2, 0x93, 0xd5]; -/// T8122 C0, firmware-compat 26.6.2. Selection remains separate from runtime admission. +/// T8122 C0, firmware-compat 26.6.2 segment and loader-mutable bounds. pub(crate) static T8122_LAYOUT_26: Layout = Layout { text_size: 393216, data_size: 1228800, @@ -313,7 +313,10 @@ pub(crate) fn identify_loaded( prelude::*, }; - let layout = soc.firmware.ok_or(ENODEV)?; + let extended=crate::m3::t8122_start::is_t8122(soc) + && crate::m3::t8122_admission::loaded_profile(pdev)?==super::m3_profile::Profile::Extended; + if extended && experiment.is_some() {return Err(EINVAL);} + let layout = if extended {&T8122_LAYOUT_26} else {soc.firmware.ok_or(ENODEV)?}; if !Firmware::matching_layout(layout, &resources) { dev_err!(pdev.as_ref(), "M3 {}: unsupported firmware segment layout\n", soc.gpu_name); return Err(ENODEV); @@ -337,6 +340,7 @@ pub(crate) fn identify_loaded( // The T8122 start experiment identifies the same image, with its own InitData version. let images = match experiment { Some(_) => &KNOWN_IMAGES_T8122_EXPERIMENT[..], + None if extended => &T8122_IMAGES_26[..], None => soc.images, }; let image = crate::m3::m3_board::identify(pdev.as_ref(), bytes, &digest, images).ok_or(ENODEV)?; diff --git a/drivers/gpu/drm/asahi/m3/m3_fragment_command.rs b/drivers/gpu/drm/asahi/m3/m3_fragment_command.rs index 44671ee3cd0ab..e0aaf2584284a 100644 --- a/drivers/gpu/drm/asahi/m3/m3_fragment_command.rs +++ b/drivers/gpu/drm/asahi/m3/m3_fragment_command.rs @@ -3,6 +3,8 @@ use super::{agx_render::Geometry, agx_render_state::{Program,DepthStencil}, m3_compute_layout::{GpuVa,RegisterSet}, m3_init_layout::{Error,Region}, m3_queue_layout::FirmwareVa, m3_render_sequence::fragment}; pub(crate) const SIZE:usize=0xc73; +pub(crate) const EXTENDED_SIZE:usize=0xca0; +pub(crate) const EXTENDED_FLUSH:usize=0xbdc; pub(crate) const REGISTERS:usize=fragment::REGISTERS; /// The registers G15G does not take; the others keep the G15S order. pub(crate) const G15G_OMIT:[u32;4]=[0x160a8,0x160b8,0xa5a9,0xd429]; @@ -74,16 +76,30 @@ impl Command { /// Check references before touching output. The caller owns validation of /// client resource mappings; this layer serializes their semantic state. pub(crate) fn encode(&self,out:&mut[u8])->Result<(),Error> { - if out.len()Result<(),Error> { + self.encode_format(out,Some(dispatch)) + } + fn encode_format(&self,out:&mut[u8],dispatch:Option)->Result<(),Error> { + let extended=dispatch.is_some(); + let size=if extended {EXTENDED_SIZE} else {SIZE}; + if out.len()=1<<42 || self.register_set!=RegisterSet::G15G {return Err(Error::Address);} + } if self.context>=64 {return Err(Error::Address);} - let seq=self.sequence.at(0,fragment::LENGTH)?; + let sequence_size=if extended {0x370} else {fragment::LENGTH}; + let seq=self.sequence.at(0,sequence_size)?; let alias=self.gpu_alias.get(); - GpuVa::new(alias.checked_add((SIZE-3) as u64).ok_or(Error::Address)?).map_err(|_|Error::Address)?; + GpuVa::new(alias.checked_add(((size-1)&!3) as u64).ok_or(Error::Address)?).map_err(|_|Error::Address)?; let registers=alias+REGISTERS as u64; let s=&self.state;let g=&s.geometry; let layer_metadata=self.heap.get().checked_add(if g.layers>1 {0x200}else{0}).ok_or(Error::Address)?; GpuVa::new(layer_metadata).map_err(|_|Error::Address)?; - let user=super::agx_render_state::compact(self.scene_user.get().checked_add(0x40).ok_or(Error::Address)?) + let user=super::agx_render_state::compact(self.scene_user.get().checked_add(if extended {0} else {0x40}).ok_or(Error::Address)?) .map_err(|_|Error::Address)? & !15; let width=(g.pixels&0xffff)+1;let height=(g.pixels>>16)+1; let tx=width.div_ceil(32);let ty=height.div_ceil(32); @@ -116,43 +132,85 @@ impl Command { (0x101c1,0),(0xd469,0),(0x1a0f9,8),(0x1c838,0),(0x1ca28,user), (0xa5a9,0x6000400020),(0xd429,0x200000001), ]; - let bytes=&mut out[..SIZE];bytes.fill(0); + let bytes=&mut out[..size];bytes.fill(0); word(bytes,0,1); // fragment opcode; work counter remains zero word(bytes,0xc,self.context); for (offset,value) in [(0x14,seq),(0x20,self.notifier.get()),(0x28,self.manager.get()), (0x30,self.scene.get()),(0x38,self.empty.get()),(0x40,self.tilemap.get()),(0x780,registers)] {pointer(bytes,offset,value);} - word(bytes,SEQUENCE_LENGTH,fragment::LENGTH as u32); - // The qualified patch path updates register MSAA but retains these - // producer header values. Representation cleanup must not change that. - pointer(bytes,0x48,0x88);word(bytes,0x50,1); - bytes[0x54..0x56].copy_from_slice(&((g.y_blocks>>18) as u16).to_le_bytes()); - bytes[0x56..0x58].copy_from_slice(&((g.x_blocks>>18) as u16).to_le_bytes()); - word(bytes,0x68,s.merge_upper[0]);word(bytes,0x6c,s.merge_upper[1]); - pointer(bytes,0x78,u64::from(g.tiles)); + word(bytes,SEQUENCE_LENGTH,sequence_size as u32); + if extended { + pointer(bytes,4,dispatch.as_ref().map_or(0,|native|native.serial)); + word(bytes,0x50,1); + word(bytes,0x54,g.isp_blocks); + word(bytes,0x60,g.x_blocks); + word(bytes,0x64,g.y_blocks); + word(bytes,0x68,g.blocks); + word(bytes,0x78,g.tiles); + } else { + pointer(bytes,0x48,0x88);word(bytes,0x50,1); + bytes[0x54..0x56].copy_from_slice(&((g.y_blocks>>18) as u16).to_le_bytes()); + bytes[0x56..0x58].copy_from_slice(&((g.x_blocks>>18) as u16).to_le_bytes()); + word(bytes,0x68,s.merge_upper[0]);word(bytes,0x6c,s.merge_upper[1]); + pointer(bytes,0x78,u64::from(g.tiles)); + } let g15g=self.register_set==RegisterSet::G15G; let mut count=0; - for (register,value) in values.into_iter().filter(|(r,_)| !g15g || !G15G_OMIT.contains(r)) { - let n=count;count+=1; - word(bytes,REGISTERS+n*REGISTER_STRIDE,register);pointer(bytes,REGISTERS+n*REGISTER_STRIDE+4,value); + let mut emit=|register,value| { + word(bytes,REGISTERS+count*REGISTER_STRIDE,register); + pointer(bytes,REGISTERS+count*REGISTER_STRIDE+4,value); + count+=1; + }; + for (register,mut value) in values { + if extended { + if matches!(register,0x160b8|0x120a1|0x1c838|0x1ca28) {continue;} + if register==0x10791 {emit(0xa301,0);} + if register==0x16068 {value=(value&!0x3ffff)|0x3717f;} + if matches!(register,0x15109|0x15101) {value&=!3;} + } else if g15g && G15G_OMIT.contains(®ister) {continue;} + emit(register,value); + } + if let Some(native)=&dispatch { + for register in [0x160e0,0x1499,0xa341] {emit(register,fragment::UUID.into());} + emit(0x1c838,0); + emit(0x1ca28,user); + emit(0x10069,native.usc_window); } bytes[0x788..0x78a].copy_from_slice(&(count as u16).to_le_bytes()); bytes[0x78a..0x78c].copy_from_slice(&((count*REGISTER_STRIDE) as u16).to_le_bytes()); // Start3DJobParameters3: mirror records use their actual packed widths. for (offset,value) in [(0,s.depth_bias),(0x10,s.scissor),(0x20,s.query), - (0x168,s.partial_background.resources),(0x170,s.partial_background.address), + (0x168,if extended {s.background.resources}else{s.partial_background.resources}), + (0x170,if extended {s.background.address}else{s.partial_background.address}), (0x198,s.partial_background.resources),(0x1a0,s.partial_background.address), (0x1a8,s.zls_control),(0x1b8,s.depth.base),(0x1c0,s.depth.stride), (0x1c8,s.depth_compression.stride),(0x1e0,s.depth_compression.base), (0x1d0,s.depth.base),(0x1d8,s.depth.base), (0x1e8,s.stencil.base),(0x1f0,s.stencil.stride),(0x200,s.stencil.base),(0x208,s.stencil.base), (0x1f8,s.stencil_compression.stride),(0x210,s.stencil_compression.base), - (0x26c,s.partial_eot.address),(0x28c,s.partial_eot.address),(0x2c0,u64::from(s.depth_dimensions))] { - pointer(bytes,PARAMETERS+offset,value); + (0x26c,if extended {s.eot.address}else{s.partial_eot.address}),(0x28c,s.partial_eot.address),(0x2c0,u64::from(s.depth_dimensions))] { + pointer(bytes,PARAMETERS+offset+if extended && offset<=0x20 {0x10}else{0},value); } for (offset,value) in [(0x1b0,0x04040404),(0x228,s.mirror_tilebuffer_control), (0x230,s.isp_control),(0x238,width),(0x23c,height),(0x240,0x100000), - (0x248,s.mirror_tile_mode),(0x264,s.partial_eot.resources as u32),(0x284,s.partial_eot.resources as u32), + (0x248,s.mirror_tile_mode),(0x264,if extended {s.eot.resources as u32}else{s.partial_eot.resources as u32}),(0x284,s.partial_eot.resources as u32), (0x298,s.depth_clear),(0x29c,s.stencil_clear),(0x2a0,s.sample_size)] {word(bytes,PARAMETERS+offset,value);} + if let Some(native)=dispatch { + pointer(bytes,0xbc0,self.stamp.get()); + pointer(bytes,0xbc8,self.fw_stamp.get()); + word(bytes,0xbd0,self.stamp_value); + word(bytes,0xbd4,1); + word(bytes,EXTENDED_FLUSH,1); + word(bytes,0xbe0,fragment::UUID); + pointer(bytes,0xc10,self.timestamps[0].get()); + pointer(bytes,0xc18,self.timestamps[1].get()); + pointer(bytes,0xc20,self.user_timestamps[0].get()); + pointer(bytes,0xc28,self.user_timestamps[1].get()); + pointer(bytes,0xc46,self.pool.get()); + bytes[0xc4e]=1; + pointer(bytes,0xc57,native.firmware_aux.get()); + bytes[0xc5f]=native.incarnation; + bytes[0xc84]=1; + } else { word(bytes,0xb88,0x67); // EncoderParams.encoder_id word(bytes,0xb90,u32::MAX); // EncoderParams.unk_mask word(bytes,0xb94,u32::from(s.process_empty_tiles)); @@ -167,6 +225,46 @@ impl Command { pointer(bytes,fragment::COMMAND_TAIL+offset,value); } bytes[fragment::COMMAND_TAIL+0x36]=1;bytes[fragment::COMMAND_TAIL+0x4f]=1; + } Ok(()) } } + +#[cfg(test)] +mod extended_tests { + use super::*; + #[test] + fn native_fragment_uses_distinct_program_mirrors_and_completion_tail() { + let fw=|n:u64|FirmwareVa::new(0xffff_fc25_3000_0000+n*0x4000).unwrap(); + let gpu=|n:u64|GpuVa::new(0x10_0300_0000+n*0x4000).unwrap(); + let program=|n:u64|Program{address:0x1_0000_0000+n*0x1000,resources:n}; + let surface=DepthStencil{base:0,stride:0}; + let command=Command {context:3,register_set:RegisterSet::G15G, + sequence:Region::new(fw(0).get(),0x1000).unwrap(),notifier:fw(1),manager:fw(2),scene:fw(3),empty:fw(4), + gpu_alias:GpuVa::new(0x72_1000_4000).unwrap(),tilemap:gpu(0),heap:gpu(1),auxiliary:gpu(2),scene_user:gpu(3), + scratch:gpu(4),pool:fw(5),stamp:fw(6),fw_stamp:fw(7),stamp_value:0x3d000100, + timestamps:[fw(8),fw(9)],user_timestamps:[fw(10),fw(11)], + state:State {geometry:Geometry::new(64,64,super::super::agx_render::Utile::Pixels32, + super::super::agx_render::Utile::Pixels32,1).unwrap(),multisample:0,merge_upper:[0;2], + eot:program(1),background:program(2),partial_eot:program(3),partial_background:program(4), + scissor:0,depth_bias:0,query:0,isp_control:0xc000,tilebuffer_control:1,mirror_tilebuffer_control:1, + tile_mode:0x280,mirror_tile_mode:0x280,depth:surface,stencil:surface,depth_compression:surface, + stencil_compression:surface,zls_control:0,depth_dimensions:0,depth_clear:0,stencil_clear:0, + sample_size:4,process_empty_tiles:false}}; + let dispatch=||super::super::m3_tiler_command::Dispatch26 {incarnation:9,serial:1,peer_stamp:0, + usc_window:1<<32,firmware_aux:fw(12),scene_scratch:gpu(4)}; + let mut bytes=[0xa5;EXTENDED_SIZE+8]; + command.encode_extended(dispatch(),&mut bytes).unwrap(); + let word=|at|u32::from_le_bytes(bytes[at..at+4].try_into().unwrap()); + let pointer=|at|u64::from_le_bytes(bytes[at..at+8].try_into().unwrap()); + assert_eq!(word(0xbd0),command.stamp_value); + assert_eq!(word(EXTENDED_FLUSH),1); + assert_eq!(pointer(0x900),command.state.background.address); + assert_eq!(pointer(0x930),command.state.partial_background.address); + assert_eq!(bytes[0xc5f],9); + assert_eq!(&bytes[EXTENDED_SIZE..],&[0xa5;8]); + let mut short=[0xa5;EXTENDED_SIZE-1]; + assert!(command.encode_extended(dispatch(),&mut short).is_err()); + assert!(short.iter().all(|b|*b==0xa5)); + } +} diff --git a/drivers/gpu/drm/asahi/m3/m3_parameter_layout.rs b/drivers/gpu/drm/asahi/m3/m3_parameter_layout.rs index 684505615adf3..97c1367b29455 100644 --- a/drivers/gpu/drm/asahi/m3/m3_parameter_layout.rs +++ b/drivers/gpu/drm/asahi/m3/m3_parameter_layout.rs @@ -61,3 +61,77 @@ impl Manager { Ok(()) } } + +impl Manager { + pub(crate) const EXTENDED_SIZE: usize = 0xc0; + + /// J613's extended manager has both firmware and GPU page-list aliases. + /// The block ring is read through its canonical firmware mapping. + pub(crate) fn encode_extended(&self, pages_fw: FirmwareVa, blocks_fw: FirmwareVa, + hardware_id: u32, output: &mut [u8]) -> Result<(), Error> { + let capacity = self.block_count.checked_add(1).and_then(u32::checked_next_power_of_two) + .ok_or(Error::Size)?; + if hardware_id >= 32 || capacity as usize > self.blocks.size() / 8 { + return Err(Error::Size); + } + let bytes = output.get_mut(..Self::EXTENDED_SIZE).ok_or(Error::Size)?; + bytes.fill(0); + let word = |bytes: &mut [u8], offset: usize, value: u32| { + bytes[offset..offset + 4].copy_from_slice(&value.to_le_bytes()); + }; + let address = |bytes: &mut [u8], offset: usize, value: u64| { + bytes[offset..offset + 8].copy_from_slice(&value.to_le_bytes()); + }; + word(bytes, 8, hardware_id); + word(bytes, 12, hardware_id); + address(bytes, 0x1c, pages_fw.get()); + address(bytes, 0x24, self.pages.base()); + word(bytes, 0x2c, self.pages.size() as u32); + word(bytes, 0x30, self.page_count); + word(bytes, 0x34, capacity); + word(bytes, 0x38, self.block_count); + address(bytes, 0x40, blocks_fw.get()); + address(bytes, 0x48, self.control.get()); + word(bytes, 0x50, self.page_count - 1); + word(bytes, 0x54, BLOCK_SIZE as u32); + address(bytes, 0x60, self.counter.get()); + word(bytes, 0x78, self.page_count); + word(bytes, 0x7c, self.page_count); + Ok(()) + } +} + +#[cfg(test)] +mod extended_tests { + use super::*; + fn fixture() -> Manager { + Manager::new(GpuRegion::new(0x5000_0000, 0x4000).unwrap(), + GpuRegion::new(0x6000_0000, 0x4000).unwrap(), + FirmwareVa::new(0xffff_fc24_0010_0000).unwrap(), + FirmwareVa::new(0xffff_fc24_0020_0000).unwrap(), 32).unwrap() + } + #[test] fn native_manager_keeps_distinct_aliases_and_hardware_identity() { + let mut bytes = [0xa5; Manager::EXTENDED_SIZE + 8]; + let pages = FirmwareVa::new(0xffff_fc24_0030_0000).unwrap(); + let blocks = FirmwareVa::new(0xffff_fc24_0040_0000).unwrap(); + fixture().encode_extended(pages, blocks, 7, &mut bytes).unwrap(); + let word = |at| u32::from_le_bytes(bytes[at..at + 4].try_into().unwrap()); + let pointer = |at| u64::from_le_bytes(bytes[at..at + 8].try_into().unwrap()); + assert_eq!(word(8), 7); assert_eq!(word(12), 7); + assert_eq!(pointer(0x1c), pages.get()); + assert_eq!(pointer(0x24), 0x5000_0000); + assert_eq!(pointer(0x40), blocks.get()); + assert_eq!(word(0x34), 64); assert_eq!(word(0x38), 32); + assert_eq!(word(0x50), 127); + assert_eq!(pointer(0x80), 0); + assert_eq!(&bytes[Manager::EXTENDED_SIZE..], &[0xa5; 8]); + } + #[test] fn native_manager_rejects_short_storage_and_invalid_identity_before_mutation() { + let fw = FirmwareVa::new(0xffff_fc24_0010_0000).unwrap(); + let mut bytes = [0xa5; Manager::EXTENDED_SIZE]; + assert!(fixture().encode_extended(fw, fw, 32, &mut bytes).is_err()); + assert_eq!(bytes, [0xa5; Manager::EXTENDED_SIZE]); + assert!(fixture().encode_extended(fw, fw, 0, &mut bytes[..MANAGER_SIZE]).is_err()); + assert_eq!(bytes, [0xa5; Manager::EXTENDED_SIZE]); + } +} diff --git a/drivers/gpu/drm/asahi/m3/m3_pass.rs b/drivers/gpu/drm/asahi/m3/m3_pass.rs index 94c2172e33279..9c0eb070039cd 100644 --- a/drivers/gpu/drm/asahi/m3/m3_pass.rs +++ b/drivers/gpu/drm/asahi/m3/m3_pass.rs @@ -8,10 +8,10 @@ use layout::{Field,Space}; pub(crate) struct Pass {buffers:KVec} impl Pass { pub(crate) fn new(dev:&driver::AsahiDrmDevice,uat:&mmu::Uat,vm:&mmu::Vm, - slot:usize,clusters:u32,aliases:&mut KVec)->Result { + slot:usize,clusters:u32,profile:super::m3_profile::Profile,aliases:&mut KVec)->Result { let mut buffers=KVec::with_capacity(layout::COUNT,GFP_KERNEL)?; for field in layout::FIELDS { - let a=layout::board_allocation(slot,field,clusters).map_err(|_|EINVAL)?; + let a=layout::allocation_for(profile,slot,field,clusters).map_err(|_|EINVAL)?; let mut b=match a.space { Space::ClientGpu=>Buffer::at_prot(dev,uat.kernel_vm(),Some((vm,a.address)),None, a.size,prot::PROT_GPU_FW_SHARED_RW,prot::Prot::from_pte(a.access.pte()))?, diff --git a/drivers/gpu/drm/asahi/m3/m3_pass_layout.rs b/drivers/gpu/drm/asahi/m3/m3_pass_layout.rs index 5ace477a44fff..f88a2a2c9b4a5 100644 --- a/drivers/gpu/drm/asahi/m3/m3_pass_layout.rs +++ b/drivers/gpu/drm/asahi/m3/m3_pass_layout.rs @@ -219,3 +219,39 @@ mod tests { } } } + +pub(crate) fn allocation_for(profile: super::m3_profile::Profile, slot:usize, field:Field, clusters:u32) + -> Result { + let mut owner = board_allocation(slot,field,clusters)?; + if profile == super::m3_profile::Profile::Legacy { return Ok(owner); } + if owner.space == Space::Firmware { + owner.address = 0xffff_fc25_0000_0000 + (slot * COUNT + field as usize) as u64 * 0x10_0000; + } + match field { + Field::Scene => owner.size = 0x400, + Field::TilerCommand | Field::FragmentCommand => { + let fragment = field == Field::FragmentCommand; + owner.size = if fragment {0xca0} else {0x940}; + owner.gpu_alias = Some(0x72_1000_0000 + slot as u64 * 0x4_0000 + + u64::from(fragment) * 0x4000); + } + Field::TilerToFragment | Field::TilerDependency | Field::FragmentDependency => owner.size = 0x40, + _ => {} + } + Ok(owner) +} + +#[cfg(test)] +mod extended_tests { + use super::*; + #[test] fn extended_command_aliases_keep_whole_disjoint_pages() { + for slot in 0..SLOTS { + for field in [Field::TilerCommand,Field::FragmentCommand] { + let owner = allocation_for(super::super::m3_profile::Profile::Extended,slot,field,1).unwrap(); + assert_eq!(owner.address & 0x3fff,0); + assert_eq!(owner.gpu_alias.unwrap() & 0x3fff,0); + assert!(owner.size <= 0x4000); + } + } + } +} diff --git a/drivers/gpu/drm/asahi/m3/m3_profile.rs b/drivers/gpu/drm/asahi/m3/m3_profile.rs index 004ac2c7d1eef..04b872ed9ea80 100644 --- a/drivers/gpu/drm/asahi/m3/m3_profile.rs +++ b/drivers/gpu/drm/asahi/m3/m3_profile.rs @@ -32,6 +32,15 @@ impl Profile { } } + /// Keep the diagnostic AP/GPU timeline in its established 24 MHz units. + /// User timestamps and duration calculations retain native firmware units. + pub(crate) const fn profiler_ticks(self, timestamp: u64) -> u64 { + match self { + Self::Legacy => timestamp, + Self::Extended => (timestamp / 1000) * 24 + (timestamp % 1000) * 24 / 1000, + } + } + pub(crate) fn gpu_duration_ns(self, elapsed: u64) -> Option { match self { Self::Legacy => elapsed.checked_mul(1000).map(|value| value / 24), @@ -49,6 +58,18 @@ impl Profile { mod tests { use super::*; + #[test] + fn profiling_uses_one_clock_domain_without_changing_native_durations() { + for value in [0, 1, 999, 1000, 24_000_000, u64::MAX] { + assert_eq!(Profile::Legacy.profiler_ticks(value), value); + assert_eq!( + Profile::Extended.profiler_ticks(value), + ((value as u128) * 24 / 1000) as u64 + ); + } + assert_eq!(Profile::Extended.profiler_ticks(1_000_000_000), 24_000_000); + } + #[test] fn profile_acknowledgement_is_exact_and_file_query_covers_hal() { for profile in [Profile::Legacy, Profile::Extended] { diff --git a/drivers/gpu/drm/asahi/m3/m3_render.rs b/drivers/gpu/drm/asahi/m3/m3_render.rs index e7a02982e96f1..9a9f63d15b784 100644 --- a/drivers/gpu/drm/asahi/m3/m3_render.rs +++ b/drivers/gpu/drm/asahi/m3/m3_render.rs @@ -26,7 +26,7 @@ pub(crate) struct Render { passes:KVec, init_bm:Buffer, // All active mappings above must drop before their ASID binding. _binding:mmu::VmBind, - stats: Region, sequence_bytes: KVec, tiler_bytes: KVec, fragment_bytes: KVec, + stats: [Region;2], profile:super::m3_profile::Profile, sequence_bytes: KVec, tiler_bytes: KVec, fragment_bytes: KVec, last_progress_ns:i64, draw:u64, heads:[u16;2], cached_views:KVec, slot:usize, batch_count:usize, early_count:u32, @@ -67,37 +67,45 @@ const RENDER_QUEUES: [QueueObjects; 2] = [ scratch: s::FRAGMENT_SCRATCH, tiler: false, uuid: 0x3d0000 }, ]; impl Render { - pub(crate) fn new(dev:&driver::AsahiDrmDevice,uat:&mmu::Uat,vm:&mmu::Vm,stats:Region,clusters:u32, - registers:crate::m3::m3_compute_layout::RegisterSet)->Result { + pub(crate) fn new(dev:&driver::AsahiDrmDevice,uat:&mmu::Uat,vm:&mmu::Vm,stats:[Region;2],clusters:u32, + registers:crate::m3::m3_compute_layout::RegisterSet,profile:super::m3_profile::Profile)->Result { let lower_tlbi=crate::m3::m3_memory::LowerTlbi; let binding=uat.bind(vm)?; let mut objects=KVec::new();let mut aliases=KVec::new(); for index in 0..s::COUNT { - let a=s::allocation(index).map_err(|_|EINVAL)?; + let a=s::allocation_for(profile,index).map_err(|_|EINVAL)?; let protection=prot::Prot::from_pte(a.access.pte()); let b=match a.space { Space::ClientGpu=>Buffer::at_prot(dev,uat.kernel_vm(),Some((vm,a.address)),None, a.size,prot::PROT_GPU_FW_SHARED_RW,protection)?, - Space::Firmware=>Buffer::at_prot(dev,uat.kernel_vm(),None,Some(a.address), + Space::Firmware=>Buffer::at_prot(dev,uat.kernel_vm(), + a.gpu_alias.map(|address|(uat.kernel_lower_vm(),address)),Some(a.address), a.size,protection,prot::PROT_GPU_SHARED_RW)?, }; + let mut b=b; + if a.gpu_alias.is_some() { aliases.push(b.map_gpu_view(vm)?,GFP_KERNEL)?; } objects.push(b,GFP_KERNEL)?; } // InitBM is shared, not part of a pass. Preserve its qualified mapping. - let init_bm=Buffer::at_prot(dev,uat.kernel_vm(),None,Some(0xfffffc200062bfe0), + let init_bm=Buffer::at_prot(dev,uat.kernel_vm(),None,Some(if profile==super::m3_profile::Profile::Extended {0xffff_fc24_4000_0000} + else {0xfffffc200062bfe0}), sync::INIT_BM_SIZE,prot::PROT_FW_SHARED_RW,prot::PROT_GPU_SHARED_RW)?; let mut passes=KVec::with_capacity(pass_layout::SLOTS,GFP_KERNEL)?; - for slot in 0..pass_layout::SLOTS {passes.push(Pass::new(dev,uat,vm,slot,clusters,&mut aliases)?,GFP_KERNEL)?;} + for slot in 0..pass_layout::SLOTS {passes.push(Pass::new(dev,uat,vm,slot,clusters,profile,&mut aliases)?,GFP_KERNEL)?;} let mut sequence_bytes=KVec::new(); sequence_bytes.resize(sequence::STORAGE,0u8,GFP_KERNEL)?; let mut tiler_bytes=KVec::new(); - tiler_bytes.resize(tiler::SIZE,0u8,GFP_KERNEL)?; + tiler_bytes.resize(if profile==super::m3_profile::Profile::Extended {tiler::EXTENDED_SIZE} else {tiler::SIZE},0u8,GFP_KERNEL)?; let mut fragment_bytes=KVec::new(); - fragment_bytes.resize(fragment::SIZE,0u8,GFP_KERNEL)?; - let mut job=Self{passes,init_bm,stats,sequence_bytes,tiler_bytes,fragment_bytes,objects,_aliases:aliases,_binding:binding,last_progress_ns:0,draw:0,heads:[0;2],cached_views:KVec::new(),slot:0,batch_count:0,early_count:0,stage_timestamps:[true;2],base:0,checkpoint:(0,[0;2],0,0,0,0),clusters,registers,_lower_tlbi:lower_tlbi}; + fragment_bytes.resize(if profile==super::m3_profile::Profile::Extended {fragment::EXTENDED_SIZE} else {fragment::SIZE},0u8,GFP_KERNEL)?; + let mut job=Self{passes,init_bm,stats,profile,sequence_bytes,tiler_bytes,fragment_bytes,objects,_aliases:aliases,_binding:binding,last_progress_ns:0,draw:0,heads:[0;2],cached_views:KVec::new(),slot:0,batch_count:0,early_count:0,stage_timestamps:[true;2],base:0,checkpoint:(0,[0;2],0,0,0,0),clusters,registers,_lower_tlbi:lower_tlbi}; // Lists and the matching manager are built before InitBM publication. job.initialize_parameter_buffer()?; - for pool in &UMA_POOLS { pool.initialize(&mut job.objects)?; } + for (index,pool) in UMA_POOLS.iter().enumerate() { + if profile==super::m3_profile::Profile::Extended { + pool.initialize_extended(&mut job.objects,33+index as u8,false)?; + } else {pool.initialize(&mut job.objects)?;} + } job.initialize_state()?; job.initialize_queues()?; dev_info!(dev.as_ref(),"M3 native render storage initialized, PB={} MiB\n",PB_BLOCKS as usize*PB_BLOCK_SIZE/(1024*1024)); @@ -113,8 +121,14 @@ impl Render { let control = queue::FirmwareVa::new(self.objects[BLOCK_CONTROL].va()).map_err(|_| EINVAL)?; let counter = queue::FirmwareVa::new(self.objects[s::BM_COUNTER].va()).map_err(|_| EINVAL)?; let manager = parameter::Manager::new(pages, blocks, control, counter, PB_BLOCKS).map_err(|_| EINVAL)?; - let mut bytes = [0; parameter::MANAGER_SIZE]; - manager.encode(&mut bytes).map_err(|_| EINVAL)?; + let extended=self.profile==super::m3_profile::Profile::Extended; + let mut bytes = [0; parameter::Manager::EXTENDED_SIZE]; + let length=if extended { + manager.encode_extended(queue::FirmwareVa::new(self.objects[PARAMETER_PAGE_LIST].va()).map_err(|_|EINVAL)?, + queue::FirmwareVa::new(self.objects[PARAMETER_BLOCK_LIST].va()).map_err(|_|EINVAL)?,0,&mut bytes) + .map_err(|_|EINVAL)?; + parameter::Manager::EXTENDED_SIZE + } else {manager.encode(&mut bytes).map_err(|_|EINVAL)?;parameter::MANAGER_SIZE}; for group in 0..PB_GROUPS { let buffer = &self.objects[PB_FIRST + group]; let backing = parameter::Backing::new(GpuRegion::new(buffer.gpu_va()?, buffer.size()).map_err(|_| EINVAL)?) @@ -128,18 +142,30 @@ impl Render { backing.page(page).map_err(|_| EINVAL)?)?; } } - self.objects[BUFFER_MANAGER].write(0, &bytes) + self.objects[BUFFER_MANAGER].write(0, &bytes[..length]) } fn initialize_state(&mut self) -> Result { let address = |owner: usize| queue::FirmwareVa::new(self.objects[owner].va()).map_err(|_| EINVAL); let notification = state::Notifier { threshold: address(s::EVENT_COUNT)?, context: self._binding.slot() }; let list_owner = address(JOB_LIST)?; let mut context = [0; state::CONTEXT_SIZE]; - state::context(&mut context).map_err(|_| EINVAL)?; - self.objects[GPU_CONTEXT].write(0, &context)?; - let mut notifier = [0; state::NOTIFIER_SIZE]; - notification.encode(&mut notifier).map_err(|_| EINVAL)?; - self.objects[NOTIFIER].write(0, ¬ifier)?; + if self.profile==super::m3_profile::Profile::Extended { + state::context_extended(&mut context).map_err(|_|EINVAL)?; + self.objects[GPU_CONTEXT].write(0,&context[..state::EXTENDED_CONTEXT_SIZE])?; + for engine in 0..2 { + let value=state::Notifier {threshold:queue::FirmwareVa::new(self.objects[s::EVENT_COUNT].va()+engine as u64*8).map_err(|_|EINVAL)?, + context:self._binding.slot()}; + let mut notifier=[0;state::EXTENDED_NOTIFIER_SIZE]; + value.encode_extended(&mut notifier).map_err(|_|EINVAL)?; + self.objects[NOTIFIER].write(engine*state::EXTENDED_NOTIFIER_SIZE,¬ifier)?; + } + } else { + state::context(&mut context).map_err(|_| EINVAL)?; + self.objects[GPU_CONTEXT].write(0, &context)?; + let mut notifier = [0; state::NOTIFIER_SIZE]; + notification.encode(&mut notifier).map_err(|_| EINVAL)?; + self.objects[NOTIFIER].write(0, ¬ifier)?; + } let mut list = [0; state::JOB_LIST_SIZE]; state::job_list(&mut list, list_owner).map_err(|_| EINVAL)?; self.objects[JOB_LIST].write(0, &list)?; @@ -157,6 +183,10 @@ impl Render { state::counter(bytes, value).map_err(|_| EINVAL)?; self.objects[owner].write(0, bytes)?; } + if self.profile==super::m3_profile::Profile::Extended { + self.objects[s::EVENT_COUNT].fill(0)?; + self.objects[s::BM_COUNTER].fill(0)?; + } Ok(()) } fn initialize_queues(&mut self) -> Result { @@ -172,8 +202,13 @@ impl Render { }else{[self.passes[0].get(P::TilerToFragment).va(),self.passes[0].get(P::FragmentCommand).va()]}; let seeds=[queue::FirmwareVa::new(seed_addresses[0]).map_err(|_|EINVAL)?, queue::FirmwareVa::new(seed_addresses[1]).map_err(|_|EINVAL)?]; - let mut bytes = [0; queue::INFO_SIZE]; - info.encode(&mut bytes).map_err(|_| EINVAL)?; + let mut bytes=KVec::new(); + bytes.resize(if self.profile==super::m3_profile::Profile::Extended { + queue::QueueInfo::EXTENDED_SIZE + } else {queue::INFO_SIZE},0u8,GFP_KERNEL)?; + if self.profile==super::m3_profile::Profile::Extended { + info.encode_extended(address(owner.info)?,&mut bytes).map_err(|_|EINVAL)?; + } else {info.encode(&mut bytes).map_err(|_|EINVAL)?;} self.objects[owner.info].write(0, &bytes)?; let mut state = [0; queue::STATE_SIZE]; queue::RingState { capacity: queue::QUALIFIED_CAPACITY, done: 0, read: 0, write: 0 } @@ -187,9 +222,12 @@ impl Render { Ok(()) } fn write_barrier(&mut self, owner: P, command: sync::Barrier) -> Result { - let mut bytes = [0; sync::BARRIER_SIZE]; - command.encode(&mut bytes).map_err(|_| EINVAL)?; - self.passes[self.slot].get_mut(owner).write(0, &bytes) + let mut bytes=[0;sync::Barrier::EXTENDED_SIZE]; + let length=if self.profile==super::m3_profile::Profile::Extended { + command.encode_extended(&mut bytes).map_err(|_|EINVAL)?; + sync::Barrier::EXTENDED_SIZE + } else {command.encode(&mut bytes).map_err(|_|EINVAL)?;sync::BARRIER_SIZE}; + self.passes[self.slot].get_mut(owner).write(0,&bytes[..length]) } // Construct synchronization from the actual owners after resetting a pass, // before publishing its ring entries. InitBM is shared and dispatched only @@ -201,9 +239,11 @@ impl Render { let ta_stamp = address(s::TA_FW_STAMP)?; let fs_stamp = address(s::FRAGMENT_FW_STAMP)?; let mut init = [0; sync::INIT_BM_SIZE]; - sync::InitBufferManager { context: self._binding.slot(), slot: 0, - block_count: PB_BLOCKS, manager, stamp: timeline::stamp(s::STAMP_TA - 256, 1) } - .encode(&mut init).map_err(|_| EINVAL)?; + let value=sync::InitBufferManager { context: self._binding.slot(), slot: 0, + block_count: PB_BLOCKS, manager, stamp: timeline::stamp(s::STAMP_TA - 256, 1) }; + if self.profile==super::m3_profile::Profile::Extended { + value.encode_extended(&mut init).map_err(|_|EINVAL)?; + } else {value.encode(&mut init).map_err(|_|EINVAL)?;} self.init_bm.write(0, &init)?; let ta_value = timeline::stamp(s::STAMP_TA - 256, draw); let fs_value = timeline::stamp(s::STAMP_FRAGMENT - 256, draw); @@ -225,7 +265,7 @@ impl Render { } // Prepare each private command after its retired slot is reset. The // reusable host staging allocation is copied before the next slot is built. - fn prepare_tiler(&mut self,r:agx_uapi::UapiRenderCommand,draw:u64)->Result { + fn prepare_tiler(&mut self,r:agx_uapi::UapiRenderCommand,usc:u64,draw:u64)->Result { use crate::m3::m3_compute_layout::GpuVa; let pass=&self.passes[self.slot]; let shared=|index:usize| queue::FirmwareVa::new(self.objects[index].va()).map_err(|_|EINVAL); @@ -243,7 +283,7 @@ impl Render { let value=tiler::Command { context:self._binding.slot(),register_set:self.registers, notifier:shared(NOTIFIER)?,manager:shared(BUFFER_MANAGER)?, - scene:fw(P::Scene)?,empty:shared(547)?,gpu_alias:GpuVa::new(command.gpu_va()?).map_err(|_|EINVAL)?, + scene:fw(P::Scene)?,empty:if self.profile==super::m3_profile::Profile::Extended {queue::FirmwareVa::new(pass.get(P::Scene).va()+0x100).map_err(|_|EINVAL)?} else {shared(547)?},gpu_alias:GpuVa::new(command.gpu_va()?).map_err(|_|EINVAL)?, sequence:Region::new(seq.va(),seq.size()).map_err(|_|EINVAL)?,pool:shared(UMA_POOLS[0].manager)?, scratch:fw(P::TilerScratch)?,stamp:shared(s::TA_STAMP)?,fw_stamp:shared(s::TA_FW_STAMP)?, stamp_value:timeline::stamp(s::STAMP_TA - 256, draw),timestamps:[fw(P::TilerStart)?,fw(P::TilerEnd)?], @@ -253,7 +293,12 @@ impl Render { vdm:crate::m3::agx_render_state::compact(r.vdm_base).map_err(|_|EINVAL)?, multisample:r.multisample_control,ppp:r.ppp_control,process_empty_tiles:r.flags&2!=0, }; - value.encode(&mut self.tiler_bytes).map_err(|_|EINVAL)?; + if self.profile==super::m3_profile::Profile::Extended { + value.encode_extended(tiler::Dispatch26 {incarnation:self._binding.generation(),serial:draw.checked_mul(2).ok_or(EOVERFLOW)?, + peer_stamp:timeline::stamp(s::STAMP_FRAGMENT-256,draw),usc_window:usc, + firmware_aux:queue::FirmwareVa::new(self.objects[UMA_POOLS[0].manager].va()+0x100).map_err(|_|EINVAL)?, + scene_scratch:gpu(P::FragmentScratch)?},&mut self.tiler_bytes).map_err(|_|EINVAL)?; + } else {value.encode(&mut self.tiler_bytes).map_err(|_|EINVAL)?;} self.passes[self.slot].get_mut(P::TilerCommand).write(0,&self.tiler_bytes) } fn prepare_sequences(&mut self,draw:u64)->Result { @@ -266,14 +311,21 @@ impl Render { let notifier=&self.objects[NOTIFIER]; let owners=sequence::Owners { command:region(if fs{P::FragmentCommand}else{P::TilerCommand})?,sequence:region(target)?, - stats:self.stats,notifier:Region::new(notifier.va(),notifier.size()).map_err(|_|EINVAL)?, + stats:self.stats[usize::from(fs)],notifier:Region::new(notifier.va()+ + if fs && self.profile==super::m3_profile::Profile::Extended {0x200} else {0}, + if self.profile==super::m3_profile::Profile::Extended {0x200} else {notifier.size()}).map_err(|_|EINVAL)?, queue:shared(if fs{s::FRAGMENT_QUEUE}else{s::TA_QUEUE})?,scene:fw(P::Scene)?, manager:shared(BUFFER_MANAGER)?,pool:shared(UMA_POOLS[usize::from(fs)].manager)?, fw_stamp:shared(if fs{s::FRAGMENT_FW_STAMP}else{s::TA_FW_STAMP})?, context:self._binding.slot(),stamp:timeline::stamp((if fs{s::STAMP_FRAGMENT}else{s::STAMP_TA}) - 256, draw), }; let timed=self.stage_timestamps[usize::from(fs)]; - let length=if fs { + let length=if self.profile==super::m3_profile::Profile::Extended { + let pool=&self.objects[UMA_POOLS[usize::from(fs)].manager]; + let auxiliary=super::m3_compute_layout::GpuVa::new(pool.gpu_va()?+if fs{0x200}else{0x100}).map_err(|_|EINVAL)?; + let serial=draw.checked_mul(2).and_then(|v|v.checked_sub(u64::from(fs))).ok_or(EOVERFLOW)?; + owners.encode_extended(fs,auxiliary,serial,&mut self.sequence_bytes).map_err(|_|EINVAL)? + } else if fs { let b=pass.get(P::FragmentScratch); let scratch=GpuRegion::new(b.gpu_va()?,b.size()).map_err(|_|EINVAL)?; sequence::Fragment{owners,scratch,queue_command_count:timeline::previous(draw)}.encode_timestamps(&mut self.sequence_bytes,timed).map_err(|_|EINVAL)? @@ -290,6 +342,9 @@ impl Render { fn patch_context(&mut self)->Result { let context=self._binding.slot(); self.objects[NOTIFIER].u32(state::NOTIFIER_CONTEXT, context)?; + if self.profile==super::m3_profile::Profile::Extended { + self.objects[NOTIFIER].u32(0x200+state::NOTIFIER_CONTEXT,context)?; + } Ok(()) } /// Pass geometry; TPC storage is per tiler accelerator, one per GPU cluster. @@ -320,7 +375,8 @@ impl Render { let value=fragment::Command { context:self._binding.slot(),register_set:self.registers, sequence:Region::new(seq.va(),seq.size()).map_err(|_|EINVAL)?, - notifier:shared(NOTIFIER)?,manager:shared(BUFFER_MANAGER)?,scene:fw(P::Scene)?,empty:shared(547)?, + notifier:queue::FirmwareVa::new(self.objects[NOTIFIER].va()+ + if self.profile==super::m3_profile::Profile::Extended {0x200} else {0}).map_err(|_|EINVAL)?,manager:shared(BUFFER_MANAGER)?,scene:fw(P::Scene)?,empty:if self.profile==super::m3_profile::Profile::Extended {queue::FirmwareVa::new(pass.get(P::Scene).va()+0x100).map_err(|_|EINVAL)?} else {shared(547)?}, gpu_alias:GpuVa::new(command.gpu_va()?).map_err(|_|EINVAL)?,tilemap:gpu(P::Tilemap)?, heap:gpu(P::HeapMetadata)?,auxiliary:gpu(P::Auxiliary)?,scene_user:gpu(P::SceneUser)?, scratch:gpu(P::FragmentScratch)?,pool:shared(UMA_POOLS[1].manager)?,stamp:shared(s::FRAGMENT_STAMP)?, @@ -341,15 +397,34 @@ impl Render { stencil_clear:r.stencil_clear,sample_size:u32::from(r.sample_size),process_empty_tiles:r.flags&2!=0, }, }; - value.encode(&mut self.fragment_bytes).map_err(|_|EINVAL)?; + if self.profile==super::m3_profile::Profile::Extended { + value.encode_extended(tiler::Dispatch26 {incarnation:self._binding.generation(),serial:draw.checked_mul(2).and_then(|v|v.checked_sub(1)).ok_or(EOVERFLOW)?, + peer_stamp:0,usc_window:usc, + firmware_aux:queue::FirmwareVa::new(self.objects[UMA_POOLS[1].manager].va()+0x200).map_err(|_|EINVAL)?, + scene_scratch:gpu(P::FragmentScratch)?},&mut self.fragment_bytes).map_err(|_|EINVAL)?; + } else {value.encode(&mut self.fragment_bytes).map_err(|_|EINVAL)?;} self.passes[self.slot].get_mut(P::FragmentCommand).write(0,&self.fragment_bytes) } + pub(crate) fn set_attachments(&mut self,attachments:&[crate::m3::agx_attachments::Attachments;2])->Result { + // Preserve the legacy renderer's existing packet format. The native + // interface consumes separate attachment apertures for both stages. + if self.profile!=super::m3_profile::Profile::Extended {return Ok(());} + for (fragment,list) in attachments.iter().enumerate() { + let field=if fragment==0 {P::TilerSequence}else{P::FragmentSequence}; + let buffer=self.passes[self.slot].get_mut(field); + sequence::patch_extended_attachments(fragment!=0,list,|offset,data|buffer.write(offset,data))?; + } + crate::m3::agx_memory::publish();Ok(()) + } pub(crate) fn set_user_timestamps(&mut self,addresses:[[u64;2];2])->Result { if addresses.iter().enumerate().any(|(stage,pair)|*pair!=[0;2] && !self.stage_timestamps[stage]) {return Err(EINVAL);} for (stage,(command,sequence,offset,start,end)) in [ (P::TilerCommand,P::TilerSequence,sequence::ta::USER_PAIR,sequence::ta::TIMESTAMP_START,sequence::ta::TIMESTAMP_END), (P::FragmentCommand,P::FragmentSequence,sequence::fragment::USER_PAIR,sequence::fragment::TIMESTAMP_START,sequence::fragment::TIMESTAMP_END) ].into_iter().enumerate() { + let (offset,start,end)=if self.profile==super::m3_profile::Profile::Extended { + if stage==0 {(0x8d8,0x1cc,0x21c)} else {(0xc20,0x1e4,0x234)} + } else {(offset,start,end)}; let pair=addresses[stage]; if !self.stage_timestamps[stage] {continue;} self.passes[self.slot].get_mut(command).u64(offset,pair[0])?; @@ -365,19 +440,19 @@ impl Render { pub(crate) fn gpu_ns(&mut self,base:usize,count:usize)->Result { let start=self.passes[base].get_mut(P::TilerStart).read_u64(0)?; let end=self.passes[Self::at(base,count-1)].get_mut(P::FragmentEnd).read_u64(0)?; - Ok(if start!=0 && end>=start {(end-start)*1000/24} else {0}) + self.profile.gpu_duration_ns(if start!=0 && end>=start {end-start} else {0}).ok_or(EOVERFLOW) } /// The batch's first tiler start and last fragment end, in 24 MHz GPU timestamp ticks. pub(crate) fn gpu_ticks(&mut self,base:usize,count:usize)->Result<(u64,u64)> { - Ok((self.passes[base].get_mut(P::TilerStart).read_u64(0)?, - self.passes[Self::at(base,count-1)].get_mut(P::FragmentEnd).read_u64(0)?)) + Ok((self.profile.profiler_ticks(self.passes[base].get_mut(P::TilerStart).read_u64(0)?), + self.profile.profiler_ticks(self.passes[Self::at(base,count-1)].get_mut(P::FragmentEnd).read_u64(0)?))) } /// Read retained timestamps only after exact batch retirement. The first /// TA and final fragment delimit the batch in the shared 24 MHz counter. pub(crate) fn batch_gpu_span(&mut self)->Result<[u64;2]> { if self.batch_count == 0 {return Err(EINVAL);} - Ok([self.passes[self.base].get_mut(P::TilerStart).read_u64(0)?, - self.passes[Self::at(self.base,self.batch_count-1)].get_mut(P::FragmentEnd).read_u64(0)?]) + let (start,end)=self.gpu_ticks(self.base,self.batch_count)?; + Ok([start,end]) } pub(crate) fn stage_ns(&mut self,base:usize,count:usize)->Result<[u64;3]> { let mut total=[0;3]; @@ -390,7 +465,7 @@ impl Render { pub(crate) fn slot_stage_ns(&mut self,base:usize,count:usize,index:usize)->Result<[u64;3]> { if index>=count {return Err(EINVAL);} let slot=Self::at(base,index); - let duration=|a:u64,b:u64| if a!=0 && b>=a {(b-a)*1000/24} else {0}; + let duration=|a:u64,b:u64| self.profile.gpu_duration_ns(if a!=0 && b>=a {b-a} else {0}).unwrap_or(0); let fs_start=self.passes[slot].get_mut(P::FragmentStart).read_u64(0)?; let fs_end=self.passes[slot].get_mut(P::FragmentEnd).read_u64(0)?; let ta_start=self.passes[slot].get_mut(P::TilerStart).read_u64(0)?; @@ -415,7 +490,9 @@ impl Render { } /// Whether any draw has been published (the first one initializes the buffer manager). pub(crate) fn started(&self)->bool {self.draw>0} - pub(crate) fn first(&self)->bool {self.draw==self.batch_count as u64} + pub(crate) fn first(&self)->bool { + self.profile==super::m3_profile::Profile::Legacy && self.draw==self.batch_count as u64 + } /// Start a batch in pass slots `base..`. With `overlap`, earlier batches may still run: then /// the VM must already be bound, no storage may grow and the slots must not be in flight /// (the caller keeps the in-flight passes plus this batch within SLOTS). @@ -447,13 +524,15 @@ impl Render { let binding=uat.bind(vm)?; let mut views=KVec::new();let mut aliases=KVec::new(); for i in 0..s::COUNT { - if s::allocation(i).map_err(|_|EINVAL)?.space==Space::ClientGpu { + let a=s::allocation_for(self.profile,i).map_err(|_|EINVAL)?; + if a.gpu_alias.is_some() {aliases.push(self.objects[i].map_gpu_view(vm)?,GFP_KERNEL)?;} + if a.space==Space::ClientGpu { views.push((ViewOwner::Shared(i),self.objects[i].map_gpu_view(vm)?),GFP_KERNEL)?; } } for (slot,pass) in self.passes.iter_mut().enumerate() { for field in pass_layout::FIELDS { - let a=pass_layout::board_allocation(slot,field,self.clusters).map_err(|_|EINVAL)?; + let a=pass_layout::allocation_for(self.profile,slot,field,self.clusters).map_err(|_|EINVAL)?; if a.space==Space::ClientGpu {views.push((ViewOwner::Pass{slot,field},pass.get_mut(field).map_gpu_view(vm)?),GFP_KERNEL)?;} if a.gpu_alias.is_some() {aliases.push(pass.get_mut(field).map_gpu_view(vm)?,GFP_KERNEL)?;} } @@ -498,12 +577,22 @@ impl Render { self.base=base; Ok(()) } + fn publish_counts(&mut self, draw:u64) -> Result { + if self.profile==super::m3_profile::Profile::Extended { + for offset in [0,8] {self.objects[s::EVENT_COUNT].u64(offset,draw)?;} + for counter in [TA_POOL_COUNTER,FRAGMENT_POOL_COUNTER] {self.objects[counter].u64(0,draw)?;} + } else { + self.objects[s::BM_COUNTER].u32(0,draw as u32)?; + self.objects[s::EVENT_COUNT].u32(0,timeline::events(draw,2))?; + } + Ok(()) + } + /// Restore producer state only when no batch is in flight. Running firmware can /// consume append's WRITE updates without a new doorbell, so overlap cannot abort. pub(crate) fn abort_batch(&mut self)->Result { (self.draw,self.heads,self.batch_count,self.slot,self.early_count,self.base)=self.checkpoint; - self.objects[s::BM_COUNTER].u32(0,self.draw as u32)?; - self.objects[s::EVENT_COUNT].u32(0,timeline::events(self.draw,2))?; + self.publish_counts(self.draw)?; for (stage,pointers) in [s::TA_POINTERS,s::FRAGMENT_POINTERS].into_iter().enumerate() { self.objects[pointers].u32(queue::WRITE,u32::from(self.heads[stage]))?; } @@ -547,8 +636,20 @@ impl Render { self.draw=self.draw.checked_add(1).ok_or(EOVERFLOW)?; self.batch_count+=1; let draw=self.draw; - self.objects[s::BM_COUNTER].u32(0,draw as u32)?; - self.objects[s::EVENT_COUNT].u32(0,timeline::events(draw,2))?; + self.publish_counts(draw)?; + if self.profile==super::m3_profile::Profile::Extended { + let pass=&self.passes[self.slot]; + let scratch=pass.get(P::FragmentScratch); + let scratch_gpu=GpuRegion::new(scratch.gpu_va()?,scratch.size()).map_err(|_|EINVAL)?; + let scratch_fw=queue::FirmwareVa::new(scratch.va()).map_err(|_|EINVAL)?; + let list=pass.get(P::SceneUser); + let list=GpuRegion::new(list.gpu_va()?,list.size()).map_err(|_|EINVAL)?; + let metadata=pass.get(P::Scene); + let owner=Region::new(metadata.va(),metadata.size()).map_err(|_|EINVAL)?; + let mut bytes=[0;scene::EXTENDED_SIZE]; + scene::encode_extended(&mut bytes,owner,scratch_gpu,scratch_fw,list).map_err(|_|EINVAL)?; + self.passes[self.slot].get_mut(P::Scene).write(0,&bytes)?; + } else { let entry_offset = (self.draw % u64::from(MANAGER_SCENE_COUNT)) as usize * 4; let scene_list = &mut self.objects[s::BM_SCENES]; if entry_offset + 4 > scene_list.size() { return Err(ERANGE); } @@ -560,7 +661,8 @@ impl Render { let mut bytes = [0; scene::SIZE]; scene::Scene { manager_entry, manager_misc, user_buffer }.encode(&mut bytes).map_err(|_| EINVAL)?; self.passes[self.slot].get_mut(P::Scene).write(0, &bytes)?; - self.prepare_tiler(r,draw)?; + } + self.prepare_tiler(r,usc,draw)?; self.patch_context()?; self.prepare_fragment(r,usc,draw)?; self.prepare_sequences(draw)?; @@ -655,7 +757,13 @@ impl Render { if self.objects[stamp].read_u32(0)?!=value || self.objects[fw].read_u32(0)?!=value || self.objects[pointers].read_u32(queue::DONE)?!=u32::from(head) || self.objects[pointers].read_u32(queue::READ)?!=u32::from(head) {return Ok(false);} - } Ok(true) + } + if self.profile==super::m3_profile::Profile::Extended && self.draw!=0 { + for pool in UMA_POOLS { + if self.objects[pool.manager].read_u64(0x54)?!=self.draw {return Ok(false);} + } + } + Ok(true) } pub(crate) fn log(&mut self,dev:&driver::AsahiDrmDevice)->Result { // Retain allocator state as well as payloads: an exhausted parameter diff --git a/drivers/gpu/drm/asahi/m3/m3_render_sequence.rs b/drivers/gpu/drm/asahi/m3/m3_render_sequence.rs index f9a0483e77b21..3d421e9906692 100644 --- a/drivers/gpu/drm/asahi/m3/m3_render_sequence.rs +++ b/drivers/gpu/drm/asahi/m3/m3_render_sequence.rs @@ -184,3 +184,171 @@ impl Fragment { Ok(if timed {LENGTH} else {without_timestamps(bytes,TIMESTAMP_START,FINALIZE,LENGTH,0xa4)}) } } + +impl Owners { + pub(crate) fn encode_extended(&self, fragment:bool, + auxiliary:super::m3_compute_layout::GpuVa, serial:u64, output:&mut[u8]) -> Result { + if self.context>=64 || serial==0 || self.stamp==0 {return Err(Error::Address);} + let command=self.command.at(0,if fragment {0xca0}else{0x940})?; + let sequence=self.sequence.at(0,STORAGE)?; + let activity=self.stats.at(0,8)?; + let scene_tail=self.scene.get().checked_add(0x54).ok_or(Error::Address)?; + let (clock,pair,aux_time,elapsed)=if fragment {(0xc08,0xc10,0xc70,0xb84)}else{(0x8c0,0x8c8,0x928,0x83c)}; + let first=if fragment {0x1e4}else{0x1cc}; + let finalize=if fragment {0x280}else{0x268}; + let pool=if fragment {0x1b0}else{0x190}; + let length=if fragment {0x370}else{0x300}; + let uuid=if fragment {fragment::UUID}else{ta::UUID}; + let bytes=output.get_mut(..STORAGE).ok_or(Error::Size)?; + bytes.fill(0); + word(bytes,0,if fragment {7}else{5}); + pointer(bytes,0x14,command+if fragment {0x80}else{0x40}); + pointer(bytes,pool,self.pool.get()); + pointer(bytes,pool+0x18,auxiliary.get()); + pointer(bytes,pool+0x28,serial); + word(bytes,pool+0x30,u32::from(fragment)); + if fragment { + pointer(bytes,0x1c,self.scene.get()); + pointer(bytes,0x24,activity); + pointer(bytes,0x2c,command+0xb70); + pointer(bytes,0x34,scene_tail); + pointer(bytes,0x3c,command+0xbb0); + pointer(bytes,0x44,command+0xbb4); + pointer(bytes,0x4c,self.queue.get()); + pointer(bytes,0x54,command); + word(bytes,0x5c,self.context); + word(bytes,0x60,1); + pointer(bytes,0x7c,command+0xa58); + pointer(bytes,0x84,command+0xbf0); + word(bytes,0xa0,uuid); + } else { + pointer(bytes,0x1c,self.manager.get()); + pointer(bytes,0x24,self.scene.get()); + pointer(bytes,0x2c,activity); + pointer(bytes,0x34,self.queue.get()); + pointer(bytes,0x3c,command+0x85c); + word(bytes,0x44,self.context); + word(bytes,0x48,1); + pointer(bytes,0x64,command+0x784); + pointer(bytes,0x6c,command+0x8a8); + word(bytes,0x7c,uuid); + } + for (at,opcode,selected) in [(first,0x8000_0003,pair),(first+0x50,3,pair+8)] { + word(bytes,at,opcode); + pointer(bytes,at+4,command+clock); + pointer(bytes,at+0xc,command+pair); + pointer(bytes,at+0x14,command+selected); + pointer(bytes,at+0x1c,self.queue.get()); + pointer(bytes,at+0x2c,command+aux_time); + pointer(bytes,at+0x34,command+elapsed); + pointer(bytes,at+0x44,uuid.into()); + } + word(bytes,first+0x4c,1); + word(bytes,finalize,if fragment {8}else{6}); + if fragment { + word(bytes,finalize+4,uuid); + pointer(bytes,finalize+0xc,self.fw_stamp.get()); + word(bytes,finalize+0x14,self.stamp); + pointer(bytes,finalize+0x1c,self.scene.get()); + pointer(bytes,finalize+0x24,self.manager.get()); + word(bytes,finalize+0x2c,1); + pointer(bytes,finalize+0x30,activity); + pointer(bytes,finalize+0x38,command+0xbb0); + pointer(bytes,finalize+0x40,command+0xbb4); + pointer(bytes,finalize+0x48,command+0xb70); + pointer(bytes,finalize+0x50,self.queue.get()); + pointer(bytes,finalize+0x58,command); + word(bytes,finalize+0x60,self.context); + pointer(bytes,finalize+0x64,command+0xa58); + pointer(bytes,finalize+0x98,sequence+pool as u64); + word(bytes,finalize+0xa0,(-(finalize as i32))as u32); + pointer(bytes,finalize+0xa5,command+0xc60); + word(bytes,finalize+0xad,1); + bytes[finalize+0xb1]=1; + // J613 terminates this record after the 16-bit event field. + bytes[finalize+0xb2..finalize+0xb4].copy_from_slice(&1u16.to_le_bytes()); + word(bytes,finalize+0xb4,0x4000_0002); + } else { + pointer(bytes,finalize+4,self.scene.get()); + pointer(bytes,finalize+0xc,self.manager.get()); + pointer(bytes,finalize+0x14,activity); + pointer(bytes,finalize+0x1c,self.queue.get()); + pointer(bytes,finalize+0x24,command+0x85c); + word(bytes,finalize+0x2c,self.context); + pointer(bytes,finalize+0x34,command+0x784); + word(bytes,finalize+0x40,uuid); + pointer(bytes,finalize+0x48,self.fw_stamp.get()); + word(bytes,finalize+0x50,self.stamp); + pointer(bytes,finalize+0x78,sequence+pool as u64); + word(bytes,finalize+0x80,(-(finalize as i32))as u32); + pointer(bytes,finalize+0x85,command+0x918); + bytes[finalize+0x8d]=1; + word(bytes,finalize+0x94,0x4000_0002); + } + Ok(length) + } +} + +#[cfg(test)] +mod native_tests { + use super::*; + #[test] fn j613_fragment_ends_before_the_m4_tail_and_keeps_relative_restart() { + let fw=|index:u64|FirmwareVa::new(0xffff_fc25_2000_0000+index*0x4000).unwrap(); + let o=Owners {command:Region::new(fw(0).get(),0xca0).unwrap(),sequence:Region::new(fw(1).get(),STORAGE).unwrap(), + stats:Region::new(fw(2).get(),0x1000).unwrap(),notifier:Region::new(fw(3).get(),0x200).unwrap(), + queue:fw(4),scene:fw(5),manager:fw(6),pool:fw(7),fw_stamp:fw(8),context:3,stamp:0x3d000100}; + let mut output=[0xa5;STORAGE+8]; + let length=o.encode_extended(true,super::super::m3_compute_layout::GpuVa::new(0x72_3000_0200).unwrap(),1,&mut output).unwrap(); + assert_eq!(length,0x370); + let word=|at|u32::from_le_bytes(output[at..at+4].try_into().unwrap()); + assert_eq!(word(0x1e4),0x8000_0003); + assert_eq!(word(0x230),1); + assert_eq!(word(0x234),3); + assert_eq!(word(0x280),8); + assert_eq!(word(0x320),(-0x280i32)as u32); + assert_eq!(word(0x334),0x4000_0002); + assert_eq!(&output[STORAGE..],&[0xa5;8]); + } +} + +/// Native start records reserve one packed attachment list per engine. The +/// matching finalizer flag enables completion visibility for that same list. +pub(crate) const fn extended_attachment_offsets(fragment:bool)->(usize,usize) { + if fragment {(0xa8,0x324)} else {(0x88,0x2ec)} +} + +/// Send both pieces through the same bounded writer used for firmware storage. +#[cfg(any(test, CONFIG_DRM_ASAHI_M3_T8122))] +pub(crate) fn patch_extended_attachments(fragment:bool, + attachments:&super::agx_attachments::Attachments, + mut write:impl FnMut(usize,&[u8])->Result<(),E>)->Result<(),E> { + let (start,flag)=extended_attachment_offsets(fragment); + write(start,&attachments.0)?; + write(flag,&[u8::from(attachments.count()!=0)]) +} + +#[cfg(test)] +mod attachment_tests { + use super::*; + use super::super::agx_attachments::{Attachments,SIZE}; + #[test] fn each_engine_keeps_its_list_and_can_clear_a_previous_list() { + for (fragment,start,flag,pool) in [(false,0x88,0x2ec,0x190),(true,0xa8,0x324,0x1b0)] { + let mut bytes=[0xa5;STORAGE]; + let list=Attachments::new(&[(0x100_0040,0x180),(0x200_0000,0x4000)]).unwrap(); + let mut apply=|list:&Attachments|patch_extended_attachments(fragment,list,|at,data|{ + bytes.get_mut(at..at+data.len()).ok_or(Error::Size)?.copy_from_slice(data);Ok::<(),Error>(()) + }); + apply(&list).unwrap(); + assert_eq!(&bytes[start..start+SIZE],&list.0); + assert_eq!(bytes[flag],1); + assert_eq!(&bytes[pool..pool+8],&[0xa5;8]); + patch_extended_attachments(fragment,&Attachments::EMPTY,|at,data|{ + bytes[at..at+data.len()].copy_from_slice(data);Ok::<(),Error>(()) + }).unwrap(); + assert!(bytes[start..start+SIZE].iter().all(|b|*b==0)); + assert_eq!(bytes[flag],0); + assert_eq!(bytes[start-1],0xa5); + assert_eq!(bytes[start+SIZE],0xa5); + } + } +} diff --git a/drivers/gpu/drm/asahi/m3/m3_rtkit.rs b/drivers/gpu/drm/asahi/m3/m3_rtkit.rs index be81c4a4e9d92..ba8ad365ce20a 100644 --- a/drivers/gpu/drm/asahi/m3/m3_rtkit.rs +++ b/drivers/gpu/drm/asahi/m3/m3_rtkit.rs @@ -38,7 +38,7 @@ impl EventWait { Arc::pin_init( pin_init!(EventWait { count: AtomicU64::new(0), - wait_us: AtomicU64::new(100), + wait_us: AtomicU64::new(1000), overlap_cpu: AtomicBool::new(crate::m3::m3_params::unlocked_wait()), cpu_preparations: AtomicU64::new(0), sleeps: AtomicU64::new(0), @@ -55,7 +55,7 @@ impl EventWait { self.arrived.complete(); } - /// Diagnostic tuning changes only the bounded resnapshot cadence, never + /// Diagnostic tuning sets the adaptive fallback ceiling in microseconds, never /// stamps, pipe retirement, event counts, fences or the two-second timeout. /// Atomic publication permits root to compare policies in the same boot. pub(crate) fn set_wait_us(&self, us: u64) -> Result { @@ -64,6 +64,10 @@ impl EventWait { Ok(()) } + pub(crate) fn wait_ceiling_us(&self) -> u64 { + self.wait_us.load(Ordering::Relaxed) + } + pub(crate) fn cpu_overlap(&self) -> bool { self.overlap_cpu.load(Ordering::Relaxed) } @@ -82,11 +86,11 @@ impl EventWait { } /// Sleep until a notification later than `seen` has been recorded or - /// the selected 25..1000 us elapse (default 100 us). A one-jiffy fallback alone takes up to 4 ms at HZ=250, + /// the selected 25..1000 us fallback elapses. A one-jiffy fallback alone takes up to 4 ms at HZ=250, /// even when the GPU stamp arrives just after its notification. The timer /// adds a bounded resnapshot without replacing IRQ wakeups or busy-waiting. /// Only the single completion worker may call this method. - pub(crate) fn wait_past(self: &Arc, seen: u64) -> bool { + pub(crate) fn wait_past(self: &Arc, seen: u64, wait_us: u64) -> bool { // Events can arrive while the worker is polling. Discard stale wakeup // tokens before checking the sequence. A concurrent event either moves // the sequence or leaves a token, including between the check and wait. @@ -98,7 +102,7 @@ impl EventWait { // on every return, before a subsequent wait can arm the timer. self.sleeps.fetch_add(1, Ordering::Relaxed); let timer = self.clone().start(Delta::from_micros( - self.wait_us.load(Ordering::Relaxed) as i64)); + wait_us.clamp(25, 1000) as i64)); // Retain one tick as a backup bound; normally either the real // interrupt or our high-resolution timer supplies the token. self.arrived.wait_for_completion_timeout(1); diff --git a/drivers/gpu/drm/asahi/m3/m3_runtime.rs b/drivers/gpu/drm/asahi/m3/m3_runtime.rs index 078540398987a..e34236fd57a3f 100644 --- a/drivers/gpu/drm/asahi/m3/m3_runtime.rs +++ b/drivers/gpu/drm/asahi/m3/m3_runtime.rs @@ -389,8 +389,8 @@ impl Inner { let index=found.unwrap_or(self.jobs.len()); if found.is_none() { let job=match control { - crate::m3::m3_submit::Command::Compute(c)=>NativeJob::Compute(crate::m3::m3_compute::Compute::new(&self.drm,&self.uat,&packet.vm,self.config.stats_region().map_err(early)?,c,self.device.soc().registers).map_err(early)?), - crate::m3::m3_submit::Command::Render{..}=>NativeJob::Render(crate::m3::m3_render::Render::new(&self.drm,&self.uat,&packet.vm,self.config.stats_region().map_err(early)?,self.device.soc().clusters,self.device.soc().registers).map_err(early)?), + crate::m3::m3_submit::Command::Compute(c)=>NativeJob::Compute(crate::m3::m3_compute::Compute::new(&self.drm,&self.uat,&packet.vm,self.config.stats_region().map_err(early)?,c,self.device.soc().registers,self.device.firmware().profile()).map_err(early)?), + crate::m3::m3_submit::Command::Render{..}=>NativeJob::Render(crate::m3::m3_render::Render::new(&self.drm,&self.uat,&packet.vm,self.config.render_stats().map_err(early)?,self.device.soc().clusters,self.device.soc().registers,self.device.firmware().profile()).map_err(early)?), }; self.jobs.push(job,GFP_KERNEL).map_err(|_|early(ENOMEM))?; } else { @@ -439,6 +439,7 @@ impl Inner { let timed=timestamps.map(|pair|policy::stage_timestamps(internal_timestamps,chip,pair)); j.append(command,usc,coalesce,timed)?; j.set_user_timestamps(timestamps)?; + j.set_attachments(&p.render_attachments[*next])?; coalesced_stamps+=u64::from(coalesce); } Ok(()) @@ -469,6 +470,9 @@ impl Inner { crate::mem::sync(); self.t8122_verdict=false; let previous_events=self.config.completed_events; + if self.device.firmware().profile() == super::m3_profile::Profile::Extended { + packet.vm.mark_binding_published().map_err(early)?; + } let publication_tick=if crate::debug::debug_enabled(crate::debug::DebugFlags::M3PassTiming) {physical_counter()} else {0}; self.state.health.set_gpu_pending(true); self.gpu_pending = true; @@ -535,7 +539,7 @@ impl Inner { let control=first_compute.ok_or(EINVAL)?; if existing.is_none() { let job=crate::m3::m3_compute::Compute::new(&self.drm,&self.uat,&packet.vm, - self.config.stats_region()?,control,self.device.soc().registers)?; + self.config.stats_region()?,control,self.device.soc().registers,self.device.firmware().profile())?; self.jobs.push(NativeJob::Compute(job),GFP_KERNEL)?; } else { let NativeJob::Compute(j)=&mut self.jobs[compute_index] else {return Err(EINVAL);}; @@ -560,6 +564,7 @@ impl Inner { let timed=timestamps.map(|pair|policy::stage_timestamps(internal_timestamps,self.device.soc().chip_id,pair)); j.append_after(command,usc,false,predecessor,timed)?; j.set_user_timestamps(timestamps)?; + j.set_attachments(&p.render_attachments[*i])?; dependencies+=usize::from(predecessor.is_some()); previous=Some((true,j.completion_dependency()?)); }, @@ -1059,7 +1064,7 @@ impl Runtime { // TTBAT of the admitted reservations, the firmware kernel root (`pagetables`) as // every context's upper half, and the host window (root entry 2) through the // shared L2. The RTKit is awake and no initdata or GPU job has been published. - let (uat,window)=adopted.insert(adopt_uat(&drm,device.soc(),&device.firmware().resources)?); + let (uat,window)=adopted.insert(adopt_uat(&drm,device.soc(),&device.firmware().resources,device.firmware().profile())?); let config=Config::new(&drm,uat,device.firmware(),&contents)?; // Link every kernel table the configuration created before the firmware sees an // address in it, then drop any translation cached before they existed. @@ -1074,6 +1079,8 @@ impl Runtime { crate::m3::t8122_start::prepare_verdict(pdev.as_ref(),device.soc(),started,e); state.health.mark_failed(); if device.stop_asc().is_err() { + // Without module unloading, the C helper is an unbound no-op. + #[cfg(CONFIG_MODULE_UNLOAD)] unsafe {kernel::bindings::__module_get(kernel::module::this_module::().as_ptr())}; core::mem::forget(transport);core::mem::forget(state); core::mem::forget(adopted); @@ -1103,7 +1110,8 @@ impl Runtime { // (7.1.12's M3 VM job lifetime, the shared `mmu::lifetime` on 7.2). let vm=self.inner.uat.new_vm(id, range)?.with_tracked_status()?; let mut reserved=KVec::new();reserved.push(reserved_range,GFP_KERNEL)?;reserved.push(0x100_8000_0000..0x101_0000_0000,GFP_KERNEL)?;reserved.push(0x10_0000_0000..0x10_0800_0000,GFP_KERNEL)?;reserved.push(0x10_7400_0000..0x10_7402_8000,GFP_KERNEL)?; - vm.install_driver_mappings(KVec::new(),reserved)?; + let mappings = self.inner.config.global_table_views(&vm)?; + vm.install_driver_mappings(mappings,reserved)?; Ok(vm) } pub(crate) fn map_timestamp(&mut self,mut bo:crate::gem::ObjectRef,range:core::ops::Range)->Result { @@ -1342,6 +1350,7 @@ impl Drop for Runtime { for batch in self.inner.active.iter() {Inner::finish_all(&batch.entries,Err(ENODEV));} if policy::retain_runtime(self.inner.device.stop_asc().is_err(),self.inner.gpu_pending) { // SAFETY: a live module reference pins callbacks with retained DMA. + #[cfg(CONFIG_MODULE_UNLOAD)] unsafe { kernel::bindings::__module_get(kernel::module::this_module::().as_ptr()) }; return; } @@ -1369,13 +1378,15 @@ pub(crate) fn physical_counter() -> u64 { /// (ttbs, pagetables, handoff, shared-l2) and the PPL exchange with its startup epoch, as the /// G15 backend does it on 7.2 (`g15::uat`). fn adopt_uat(drm: &driver::AsahiDrmDevice, soc: &'static crate::m3::m3_soc::Soc, - resources: &crate::m3::agx_resources::Resources) -> Result<(mmu::Uat, crate::g15::uat::Window)> { + resources: &crate::m3::agx_resources::Resources, profile: super::m3_profile::Profile) + -> Result<(mmu::Uat, crate::g15::uat::Window)> { let cfg = soc.hwcfg.ok_or(ENODEV)?; let [ttbs, tables, handoff, shared_l2, ..] = resources.regions; let range = |r: crate::m3::agx_resources::Region| r.base..r.base + r.size; let geometry = mmu::UatGeometry { ias: cfg.uat_ias.into(), oas: cfg.uat_oas }; let regions = mmu::AdoptedRegions { handoff: range(handoff), ttbs: range(ttbs), kernel_root: tables.base }; - let uat = mmu::Uat::new_adopted(drm, geometry, ®ions)?; + let uat = mmu::Uat::new_adopted_with_user_upper(drm, geometry, ®ions, + profile == super::m3_profile::Profile::Legacy)?; let mut window = crate::g15::uat::Window::from_regions(handoff.base, ttbs.base, tables.base, shared_l2.base)?; window.bootstrap(drm.as_ref().as_ref(), &uat)?; // 7.1.12 cleaned every reserved-table write to the point of coherency; these regions may be diff --git a/drivers/gpu/drm/asahi/m3/m3_scene_layout.rs b/drivers/gpu/drm/asahi/m3/m3_scene_layout.rs index 880cf4c2b9a75..ea3c2064ef39e 100644 --- a/drivers/gpu/drm/asahi/m3/m3_scene_layout.rs +++ b/drivers/gpu/drm/asahi/m3/m3_scene_layout.rs @@ -39,3 +39,43 @@ impl Scene { Ok(()) } } + +pub(crate) const EXTENDED_SIZE: usize = 0x400; + +/// Encode a native scene header from its retained aliases. The local +/// statistics and parameter scratch remain part of this same owned record. +pub(crate) fn encode_extended(output: &mut [u8], owner: super::m3_init_layout::Region, + scratch_gpu: super::m3_pool_layout::GpuRegion, scratch_fw: FirmwareVa, + scene_list: super::m3_pool_layout::GpuRegion) -> Result<(), Error> { + let statistics = owner.at(0x200, 0x40).map_err(|_| Error::Size)?; + owner.at(0, EXTENDED_SIZE).map_err(|_| Error::Size)?; + if scratch_gpu.size() < 0x4000 || scene_list.size() < 0x4000 { return Err(Error::Size); } + let compact = super::agx_render_state::compact(scene_list.base()).map_err(|_| Error::Address)?; + let bytes = output.get_mut(..EXTENDED_SIZE).ok_or(Error::Size)?; + bytes.fill(0); + bytes[..8].copy_from_slice(&scratch_gpu.base().to_le_bytes()); + bytes[8..16].copy_from_slice(&scratch_fw.get().to_le_bytes()); + bytes[0x28..0x30].copy_from_slice(&compact.to_le_bytes()); + bytes[0x40..0x48].copy_from_slice(&statistics.to_le_bytes()); + Ok(()) +} + +#[cfg(test)] +mod extended_tests { + use super::*; + use super::super::{m3_init_layout::Region, m3_pool_layout::GpuRegion}; + #[test] fn native_scene_keeps_scratch_aliases_and_local_statistics_distinct() { + let owner = Region::new(0xffff_fc25_0010_0000, EXTENDED_SIZE).unwrap(); + let scratch = GpuRegion::new(0x10_0200_0000, 0x4000).unwrap(); + let scratch_fw = FirmwareVa::new(0xffff_fc25_0020_0000).unwrap(); + let list = GpuRegion::new(0x10_0300_0000, 0x4000).unwrap(); + let mut bytes = [0xa5; EXTENDED_SIZE + 8]; + encode_extended(&mut bytes, owner, scratch, scratch_fw, list).unwrap(); + let read = |at| u64::from_le_bytes(bytes[at..at + 8].try_into().unwrap()); + assert_eq!(read(0), scratch.base()); + assert_eq!(read(8), scratch_fw.get()); + assert_eq!(read(0x28), 0x300_0000); + assert_eq!(read(0x40), owner.at(0x200, 0x40).unwrap()); + assert_eq!(&bytes[EXTENDED_SIZE..], &[0xa5; 8]); + } +} diff --git a/drivers/gpu/drm/asahi/m3/m3_shared_layout.rs b/drivers/gpu/drm/asahi/m3/m3_shared_layout.rs index 9b8d503836d0a..0a2262858556d 100644 --- a/drivers/gpu/drm/asahi/m3/m3_shared_layout.rs +++ b/drivers/gpu/drm/asahi/m3/m3_shared_layout.rs @@ -100,3 +100,29 @@ pub(crate) fn fault_buffers()->impl Iterator { .chain((PB_FIRST..PB_FIRST+PB_GROUPS).map(|i|(i,256))) .chain([(TILEMAP_LABEL,1280),(HEAP_METADATA_LABEL,256)]) } + +pub(crate) fn allocation_for(profile: super::m3_profile::Profile, index: usize) -> Result { + let mut owner = allocation(index)?; + if profile == super::m3_profile::Profile::Legacy { return Ok(owner); } + if owner.space == Space::Firmware { + owner.address = 0xffff_fc24_0000_0000 + index as u64 * 0x10_0000; + } + match index { + // Extended firmware reads the uncached kernel aliases supplied to + // encode_extended; these lower client mappings are GPU-only. + PARAMETER_PAGE_LIST | PARAMETER_BLOCK_LIST => owner.access = Access::GpuCachedRw, + 8 | 20 => owner.size = 0x20000, + 9 | 21 => { owner.address &= !0x3fff; owner.size = 0x4000; } + 10 | 22 => { + owner.size = 0x4000; + owner.gpu_alias = Some(0x72_3000_0000 + u64::from(index == 22) * 0x4000); + } + GPU_CONTEXT => owner.size = 0x38, + NOTIFIER => owner.size = 0x400, + EVENT_COUNT => owner.size = 16, + BUFFER_MANAGER => owner.size = 0xc0, + FRAGMENT_QUEUE | TA_QUEUE => owner.size = 0x24c0, + _ => {} + } + Ok(owner) +} diff --git a/drivers/gpu/drm/asahi/m3/m3_submit.rs b/drivers/gpu/drm/asahi/m3/m3_submit.rs index c3a25c9a2dc04..155bda714e9d9 100644 --- a/drivers/gpu/drm/asahi/m3/m3_submit.rs +++ b/drivers/gpu/drm/asahi/m3/m3_submit.rs @@ -42,6 +42,7 @@ pub(crate) struct Packet { timestamps:KVec<[Option;4]>, pub(crate) attachments:KVec, pub(crate) wide_visibility:KVec, + pub(crate) render_attachments:KVec<[crate::m3::agx_attachments::Attachments;2]>, completion:UserFence, finish_claimed: AtomicBool, /// Set before a batch with any of this packet's commands can change firmware-visible state, @@ -197,7 +198,7 @@ impl queue::Queue for Queue { fn submit(&mut self,_reg_data:&crate::driver::AsahiRegData<'_>,_id:u64,mut syncs:KVec,in_sync_count:usize, raw:&[u8],objects:Pin<&xarray::XArray>>)->Result { let vm_job=self.vm.retain_job()?; - let mut parser=agx_uapi::UapiCommandParser::new(raw);let mut commands=KVec::new();let mut timestamps=KVec::new();let mut flushes=KVec::new();let mut visibility=KVec::new(); + let mut parser=agx_uapi::UapiCommandParser::new(raw);let mut commands=KVec::new();let mut timestamps=KVec::new();let mut flushes=KVec::new();let mut visibility=KVec::new();let mut render_attachments=KVec::new(); while let Some(command)=parser.next_hardware().map_err(|_|EINVAL)? { if let agx_uapi::ParsedHardwareCommand::Render{payload,vertex_attachments,fragment_attachments,..}=command { if let Err(e)=validate_render(payload,self.usc,&AddressSpace(&self.vm),self.early_tiling) { @@ -212,6 +213,7 @@ impl queue::Queue for Queue { Destination::resolve(objects,payload.fragment_timestamps.start)?, Destination::resolve(objects,payload.fragment_timestamps.end)?],GFP_KERNEL)?; flushes.push(crate::m3::agx_attachments::Attachments::EMPTY,GFP_KERNEL)?; + render_attachments.push([encode_attachments(&self.vm,&vertex_attachments)?,encode_attachments(&self.vm,&fragment_attachments)?],GFP_KERNEL)?; visibility.push(false,GFP_KERNEL)?; continue; } @@ -227,6 +229,7 @@ impl queue::Queue for Queue { // The common translator validates the original command/attachment // addresses; this M3 scheduling hint changes none of them. payload.flags &= !wide; + render_attachments.push([crate::m3::agx_attachments::Attachments::EMPTY;2],GFP_KERNEL)?; flushes.push(encode_attachments(&self.vm,&attachments)?,GFP_KERNEL)?; let c=agx_uapi::translate_compute_command(payload,attachments, agx_uapi::QueueUscWindow{base:self.usc,user_start:0x4000,user_end:(1u64<<42)-0x8000}, @@ -240,7 +243,7 @@ impl queue::Queue for Queue { commands.push(Command::Compute(crate::m3::agx_compute::Control{base:c.control_stream_base,end:c.control_stream_end,usc_base:self.usc}),GFP_KERNEL)?; } parser.finish().map_err(|_|EINVAL)?; - let packet=Arc::new(Packet{vm:self.vm.clone(),commands,timestamps,attachments:flushes,wide_visibility:visibility, + let packet=Arc::new(Packet{vm:self.vm.clone(),commands,timestamps,attachments:flushes,wide_visibility:visibility,render_attachments, completion:self.fences.new_fence(0,Completion)?.into(), finish_claimed:AtomicBool::new(false),published:AtomicBool::new(false),urgency:self.urgency,owner:Owner::current(), vm_job:KBox::pin_init(new_mutex!(Some(vm_job)),GFP_KERNEL)?},GFP_KERNEL)?; diff --git a/drivers/gpu/drm/asahi/m3/m3_sync_layout.rs b/drivers/gpu/drm/asahi/m3/m3_sync_layout.rs index 92d533efa8234..2cf5e18914043 100644 --- a/drivers/gpu/drm/asahi/m3/m3_sync_layout.rs +++ b/drivers/gpu/drm/asahi/m3/m3_sync_layout.rs @@ -91,3 +91,58 @@ impl Barrier { Ok(()) } } + +impl InitBufferManager { + /// The extended rebuild path requires a zero staged-block count. Appending + /// already-staged blocks would leave a previous hardware descriptor live. + pub(crate) fn encode_extended(&self, output: &mut [u8]) -> Result<(), Error> { + if self.context >= 64 || self.slot >= 32 { return Err(Error::Address); } + Self { context: self.context, slot: self.slot, block_count: 0, + manager: self.manager, stamp: self.stamp }.encode(output) + } +} + +impl Barrier { + pub(crate) const EXTENDED_SIZE: usize = 0x40; + + pub(crate) fn encode_extended(&self, output: &mut [u8]) -> Result<(), Error> { + if self.event >= 128 || self.wait_value == 0 || self.self_value == 0 { + return Err(Error::Address); + } + let bytes = output.get_mut(..Self::EXTENDED_SIZE).ok_or(Error::Size)?; + bytes.fill(0); + u32_at(bytes, 0, 4); + address_at(bytes, 4, self.stamp); + address_at(bytes, 12, self.stamp); + u32_at(bytes, 0x14, self.wait_value); + u32_at(bytes, 0x20, self.event); + u32_at(bytes, 0x24, self.self_value); + if matches!(self.dependency, Dependency::PreviousPass) { u32_at(bytes, 0x3c, 1); } + Ok(()) + } +} + +#[cfg(test)] +mod extended_tests { + use super::*; + #[test] fn full_rebuild_never_uses_the_legacy_staged_count() { + let mut bytes = [0xa5; INIT_BM_SIZE]; + InitBufferManager { context: 3, slot: 7, block_count: 2048, + manager: FirmwareVa::new(0xffff_fc24_0010_0000).unwrap(), stamp: 0x100 } + .encode_extended(&mut bytes).unwrap(); + assert_eq!(&bytes[12..20], &[0; 8]); + } + #[test] fn extended_dependencies_use_their_native_tail_and_preserve_trailing_storage() { + for previous in [false, true] { + let mut bytes = [0xa5; Barrier::EXTENDED_SIZE + 8]; + Barrier { stamp: FirmwareVa::new(0xffff_fc24_0010_0000).unwrap(), wait_value: 0x100, + event: 2, self_value: 0x200, uuid: 9, + dependency: if previous { Dependency::PreviousPass } else { Dependency::TilerToFragment } } + .encode_extended(&mut bytes).unwrap(); + assert_eq!(&bytes[0x18..0x20], &[0; 8]); + assert_eq!(&bytes[0x20..0x24], &[2, 0, 0, 0]); + assert_eq!(u32::from_le_bytes(bytes[0x3c..0x40].try_into().unwrap()), previous.into()); + assert_eq!(&bytes[Barrier::EXTENDED_SIZE..], &[0xa5; 8]); + } + } +} diff --git a/drivers/gpu/drm/asahi/m3/m3_tiler_command.rs b/drivers/gpu/drm/asahi/m3/m3_tiler_command.rs index eeda857a3ed88..caa61621e0b04 100644 --- a/drivers/gpu/drm/asahi/m3/m3_tiler_command.rs +++ b/drivers/gpu/drm/asahi/m3/m3_tiler_command.rs @@ -3,6 +3,17 @@ use super::{agx_render::Geometry, m3_compute_layout::{GpuVa,RegisterSet}, m3_init_layout::{Error,Region}, m3_queue_layout::FirmwareVa, m3_render_sequence::ta}; pub(crate) const SIZE:usize=0x92b; +pub(crate) const EXTENDED_SIZE:usize=0x940; +pub(crate) const EXTENDED_FLUSH:usize=0x894; +pub(crate) struct Dispatch26 { + pub(crate) incarnation:u8, + pub(crate) serial:u64, + pub(crate) peer_stamp:u32, + pub(crate) usc_window:u64, + pub(crate) firmware_aux:FirmwareVa, + pub(crate) scene_scratch:GpuVa, +} + pub(crate) const REGISTERS:usize=ta::REGISTERS; /// The registers G15G does not take; the others keep the G15S order. pub(crate) const G15G_OMIT:[u32;14]=[0x1c041,0x1c9d0,0x1c948,0x1c888,0x1c890,0x1c918,0x1c089,0x1c9e0,0x16c41,0x1ca40,0x1c9a8,0x1c920,0xa5a1,0xd419]; @@ -57,17 +68,32 @@ impl Command { /// All address checks precede mutation. Zero all reserved fields, then /// emit semantic registers and explicit packed command/tail fields. pub(crate) fn encode(&self,out:&mut[u8])->Result<(),Error> { - if out.len() Result<(),Error> { + self.encode_format(out, Some(dispatch)) + } + fn encode_format(&self, out: &mut [u8], dispatch: Option) -> Result<(),Error> { + let extended = dispatch.is_some(); + let size = if extended { EXTENDED_SIZE } else { SIZE }; + if out.len()= 1 << 42 || self.register_set != RegisterSet::G15G { + return Err(Error::Address); + } + } if self.context>=64 || self.page_count==0 || self.tpc_bytes==0 {return Err(Error::Address);} let registers=self.gpu_alias.get().checked_add(REGISTERS as u64).ok_or(Error::Address)?; - GpuVa::new(self.gpu_alias.get().checked_add((SIZE-3) as u64).ok_or(Error::Address)?) + GpuVa::new(self.gpu_alias.get().checked_add(((size-1)&!3) as u64).ok_or(Error::Address)?) .map_err(|_|Error::Address)?; - let seq=self.sequence.at(0,ta::LENGTH)?; + let sequence_size = if extended { 0x300 } else { ta::LENGTH }; + let seq=self.sequence.at(0,sequence_size)?; let tilemap=compact(self.tilemap.get())?; let tpc=compact(self.tpc.get())?; let heap=compact(self.heap.get())?; let preemption=[compact(self.preemption[0].get())?,compact(self.preemption[1].get())?,compact(self.preemption[2].get())?]; - let user=compact(self.scene_user.get().checked_add(0x40).ok_or(Error::Address)?)?; + let user=compact(self.scene_user.get().checked_add(if extended {0} else {0x40}).ok_or(Error::Address)?)?; let g=&self.geometry; let layers=g.layers; let layer_metadata=compact(self.heap.get().checked_add(if layers>1 {0x200}else{0}).ok_or(Error::Address)?)?; @@ -99,23 +125,67 @@ impl Command { (0x1c830,0),(0x1ca30,user),(0x16c39,user), (0x1c910,0xa0_00000000|(self.initial_scene_entry.get()&0xf_ffffffff)|1), ]; - let bytes=&mut out[..SIZE];bytes.fill(0); // TA opcode is zero - pointer(bytes,4,1); // qualified work-command counter remains one + let bytes=&mut out[..size];bytes.fill(0); // TA opcode is zero + pointer(bytes,4,dispatch.as_ref().map_or(1, |native| native.serial)); word(bytes,CONTEXT,self.context); - for (offset,value) in [(0x14,self.notifier.get()),(0x24,self.manager.get()), - (0x2c,self.scene.get()),(0x34,self.empty.get()),(0x740,registers), + for (offset,value) in [(if extended {0x10} else {0x14},self.notifier.get()), + (if extended {0x20} else {0x24},self.manager.get()), + (if extended {0x28} else {0x2c},self.scene.get()), + (if extended {0x30} else {0x34},self.empty.get()),(0x740,registers), (0x760,self.tpc.get()),(0x768,self.tpc_bytes),(0x770,seq)] {pointer(bytes,offset,value);} let g15g=self.register_set==RegisterSet::G15G; let mut count=0; - for (register,value) in values.into_iter().filter(|(r,_)| !g15g || !G15G_OMIT.contains(r)) { - let n=count;count+=1; - word(bytes,REGISTERS+n*REGISTER_STRIDE,register); - pointer(bytes,REGISTERS+n*REGISTER_STRIDE+4,value); + let mut emit = |register, value| { + word(bytes, REGISTERS + count * REGISTER_STRIDE, register); + pointer(bytes, REGISTERS + count * REGISTER_STRIDE + 4, value); + count += 1; + }; + for (register, mut value) in values { + if g15g && G15G_OMIT.contains(®ister) { continue; } + if extended && matches!(register, 0x12099 | 0x1c898) { continue; } + if let Some(native) = &dispatch { + if register == 0x101e1 { + emit(0xa309, 0); + emit(0x1c8e0, u64::MAX); + emit(0x1c8e8, u64::MAX); + emit(0x1c898, 0); + } + if register == 0x1c910 { + let address = native.scene_scratch.get(); + let escaped = (address & (1 << 42)) >> 3; + let bias = if escaped == 0 { 0x70_0000_0000 } else { 0 }; + value = ((address + bias) & 0x7f_ffff_fffe) | escaped | 1; + } + } + emit(register, value); + } + if let Some(native) = &dispatch { + emit(0xa5a1, 0x60_0040_0020); + emit(0xd419, 0x2_0000_0001); + for register in [0x1ca10, 0x14a1, 0xa349] { emit(register, ta::UUID.into()); } + emit(0x10061, native.usc_window); } bytes[0x748..0x74a].copy_from_slice(&(count as u16).to_le_bytes()); bytes[0x74a..0x74c].copy_from_slice(&((count*REGISTER_STRIDE) as u16).to_le_bytes()); - word(bytes,SEQUENCE_LENGTH,ta::LENGTH as u32); + word(bytes,SEQUENCE_LENGTH,sequence_size as u32); word(bytes,0x77c,1); // fragment event slot + if let Some(native) = dispatch { + word(bytes, STAMP_VALUE, native.peer_stamp); + pointer(bytes, 0x7a8, preemption[0]); + pointer(bytes, 0x878, self.stamp.get()); + pointer(bytes, 0x880, self.fw_stamp.get()); + word(bytes, 0x888, self.stamp_value); + word(bytes, EXTENDED_FLUSH, 1); + word(bytes, 0x898, ta::UUID); + pointer(bytes, 0x8c8, self.timestamps[0].get()); + pointer(bytes, 0x8d0, self.timestamps[1].get()); + pointer(bytes, 0x8d8, self.user_timestamps[0].get()); + pointer(bytes, 0x8e0, self.user_timestamps[1].get()); + pointer(bytes, 0x8fe, self.pool.get()); + bytes[0x906] = 1; + pointer(bytes, 0x90f, native.firmware_aux.get()); + bytes[0x917] = native.incarnation; + } else { word(bytes,STAMP_VALUE,self.stamp_value); pointer(bytes,ta::STRUCT3+0x24,preemption[0]); word(bytes,ta::STRUCT3+0xbc,0x67); // retained producer encoder_id @@ -132,6 +202,41 @@ impl Command { (0x47,self.scratch.get())] {pointer(bytes,ta::COMMAND_TAIL+offset,value);} bytes[ta::COMMAND_TAIL+0x36]=1; // unk_846 bytes[ta::COMMAND_TAIL+0x4f]=1; // unk_85f + } Ok(()) } } + +#[cfg(test)] +mod extended_tests { + use super::*; + fn command() -> Command { + let fw=|index:u64|FirmwareVa::new(0xffff_fc25_1000_0000+index*0x4000).unwrap(); + let gpu=|index:u64|GpuVa::new(0x10_0100_0000+index*0x4000).unwrap(); + Command {context:3,register_set:RegisterSet::G15G,notifier:fw(0),manager:fw(1),scene:fw(2),empty:fw(3), + gpu_alias:GpuVa::new(0x72_1000_0000).unwrap(),sequence:Region::new(fw(4).get(),0x1000).unwrap(), + pool:fw(5),scratch:fw(6),stamp:fw(7),fw_stamp:fw(8),stamp_value:0x7a000100, + timestamps:[fw(9),fw(10)],user_timestamps:[fw(11),fw(12)],preemption:[gpu(0),gpu(1),gpu(2)], + tilemap:gpu(3),tpc:gpu(4),tpc_bytes:0x4000,heap:gpu(5),scene_user:gpu(6), + initial_scene_entry:fw(13),geometry:Geometry::new(64,64,super::super::agx_render::Utile::Pixels32, + super::super::agx_render::Utile::Pixels32,1).unwrap(),page_count:128, + vdm:0x100000,multisample:0,ppp:0,process_empty_tiles:false} + } + #[test] fn native_tiler_keeps_peer_completion_and_usc_window_explicit() { + let value=command();let mut bytes=[0xa5;EXTENDED_SIZE+8]; + value.encode_extended(Dispatch26 {incarnation:9,serial:2,peer_stamp:0x3d000100, + usc_window:1<<32,firmware_aux:value.scratch, + scene_scratch:GpuVa::new(0x10_0200_0000).unwrap()},&mut bytes).unwrap(); + let word=|at|u32::from_le_bytes(bytes[at..at+4].try_into().unwrap()); + assert_eq!(word(0x780),0x3d000100); + assert_eq!(word(0x888),value.stamp_value); + assert_eq!(word(0x894),1); + assert_eq!(word(0x748)&0xffff,66); + assert_eq!(bytes[0x917],9); + assert_eq!(&bytes[EXTENDED_SIZE..],&[0xa5;8]); + let registers=&bytes[REGISTERS..REGISTERS+66*REGISTER_STRIDE]; + assert!(!registers.chunks_exact(12).any(|r|u32::from_le_bytes(r[..4].try_into().unwrap())==0x12099)); + let usc=registers.chunks_exact(12).find(|r|u32::from_le_bytes(r[..4].try_into().unwrap())==0x10061).unwrap(); + assert_eq!(u64::from_le_bytes(usc[4..].try_into().unwrap()),1<<32); + } +} diff --git a/drivers/gpu/drm/asahi/m3/mod.rs b/drivers/gpu/drm/asahi/m3/mod.rs index d885da1a8647c..eafd5732bceac 100644 --- a/drivers/gpu/drm/asahi/m3/mod.rs +++ b/drivers/gpu/drm/asahi/m3/mod.rs @@ -133,6 +133,8 @@ mod t8122_knobs; #[cfg(CONFIG_DRM_ASAHI_M3_T8122)] mod t8122_start; +mod m3_completion_policy; + mod m3_compute_layout; mod m3_compute_sequence; diff --git a/drivers/gpu/drm/asahi/m3/t8122_admission.rs b/drivers/gpu/drm/asahi/m3/t8122_admission.rs index fdc41024899f1..fb280ebcde520 100644 --- a/drivers/gpu/drm/asahi/m3/t8122_admission.rs +++ b/drivers/gpu/drm/asahi/m3/t8122_admission.rs @@ -1,6 +1,6 @@ // SPDX-License-Identifier: GPL-2.0-only OR MIT -//! Read-only T8122 identity admission. Firmware startup remains disabled. +//! Read-only T8122 identity and firmware-interface admission. use super::m3_profile::Profile; @@ -53,6 +53,28 @@ pub(crate) fn validate_identity_for( }) } +/// Firmware-interface selection is exact and does not change physical GPU encodings. +pub(crate) fn profile_for(compat:&[u32],j613:bool)->Option { + match compat { + [14,8,3]=>Some(Profile::Legacy), + [26,6,2] if j613=>Some(Profile::Extended), + _=>None, + } +} + +#[cfg(not(test))] +pub(crate) fn loaded_profile(pdev:&kernel::platform::Device>) + ->kernel::error::Result { + use kernel::{c_str,prelude::*}; + let node=pdev.as_ref().of_node().ok_or(ENODEV)?; + let compat:KVec=node.get_property(c_str!("apple,firmware-compat"))?; + #[cfg(CONFIG_DRM_ASAHI_M3_T8122)] + let j613 = super::m3_board::board_is(b"apple,j613"); + #[cfg(not(CONFIG_DRM_ASAHI_M3_T8122))] + let j613 = false; + profile_for(&compat, j613).ok_or(ENODEV) +} + /// Shape of the voltage-sorted running-state tables: one entry per distinct millivolt, /// and the highest primary frequency in MHz. Voltages round up, matching `PwrConfig`. /// The first point is the off state. Both primary and secondary frequencies must increase @@ -108,7 +130,7 @@ pub(crate) fn opp_table_shape(opps: impl IntoIterator) -> Opt pub(crate) fn read_identity( pdev: &kernel::platform::Device>, ) -> kernel::error::Result { - read_identity_for(pdev, Profile::Legacy) + read_identity_for(pdev, loaded_profile(pdev)?) } #[cfg(not(test))] @@ -300,3 +322,17 @@ mod tests { assert!(validate_identity(&[14, 8, 3], 1, 1, G15G_C0, [0x7f, 0, 0]).is_none()); } } + +#[cfg(test)] +mod selection_tests { + use super::*; + #[test] fn extended_admission_requires_exact_firmware_and_j613() { + for board in [false,true] { + assert_eq!(profile_for(&[14,8,3],board),Some(Profile::Legacy)); + assert_eq!(profile_for(&[26,6,2],board),if board{Some(Profile::Extended)}else{None}); + for wrong in [&[][..],&[26,6][..],&[26,6,1][..],&[26,6,2,0][..],&[14,8,4][..]] { + assert_eq!(profile_for(wrong,board),None); + } + } + } +} diff --git a/tools/asahi/m3-fence-latency.py b/tools/asahi/m3-fence-latency.py new file mode 100644 index 0000000000000..c8428d65dbf0d --- /dev/null +++ b/tools/asahi/m3-fence-latency.py @@ -0,0 +1,63 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only OR MIT +"""Join existing M3 timing logs. Input must be one boot, captured without gaps.""" +import collections +import json +import math +import re +import sys + +milestones = {} +packets = collections.defaultdict(list) +fences = [] +invalid = collections.Counter() +for line in open(sys.argv[1]): + fields = {k: int(v, 0) for k, v in re.findall(r'(\w+)=(-?0x[0-9a-fA-F]+|-?\d+)', line)} + if 'M3_BATCH_MILESTONE ' in line: + key = (fields['vm'], fields['publication_tick']) + if key in milestones: + invalid['duplicate_milestone'] += 1 + milestones[key] = fields + elif 'M3_BATCH_PACKET ' in line: + packets[(fields['vm'], fields['packet'])].append((fields['publication_tick'], fields['command'])) + elif 'M3_PACKET_FENCE ' in line: + fences.append(fields) +values = [] +previous_signal = {} +for f in sorted(fences, key=lambda f: f['signal_before_tick']): + identity = (f['vm'], f['packet']) + previous = previous_signal.get(identity, 0) + previous_signal[identity] = f['signal_after_tick'] + if f['status'] != 0: + invalid['error_fence'] += 1 + continue + linked = packets.get((f['vm'], f['packet']), []) + # Pointer reuse is disambiguated by the completed fence's physical counter. + candidates = [(tick, command) for tick, command in linked if previous < tick <= f['signal_before_tick'] and command == f['commands'] - 1] + if not candidates: + invalid['missing_final_command'] += 1 + continue + final_tick, _ = max(candidates) + batch = milestones.get((f['vm'], final_tick)) + if not batch or batch['mixed'] or batch['render_end_tick'] <= 0: + invalid['missing_or_non_render_end'] += 1 + continue + end = batch['render_end_tick'] + before, after = f['signal_before_tick'], f['signal_after_tick'] + if not (final_tick <= end <= before <= after): + invalid['invalid_counter_order'] += 1 + continue + # GPU execution end is earlier than full coherent retirement; this includes + # trailing witnesses and dispatch/retirement, rather than claiming an IRQ is done. + values.append(((before - end) / 24, (after - end) / 24)) + +def percentiles(data): + ordered = sorted(data) + return {str(p): ordered[max(0, math.ceil(len(ordered) * p / 100) - 1)] for p in (50, 95, 99, 100)} if ordered else {} + +print(json.dumps({'render_fences': len(values), 'invalid': invalid, + 'gpu_end_to_signal_before_us': percentiles([a for a, b in values]), + 'gpu_end_to_signal_after_us': percentiles([b for a, b in values]), + 'note': '24 MHz physical counter; interval brackets signal(); includes trailing witnesses. Not presentation latency.'}, indent=2)) +if not values: + sys.exit('No valid render fences: do not report latency from this capture') diff --git a/tools/asahi/m3-runtime-tests.rs b/tools/asahi/m3-runtime-tests.rs index 458603366c954..0d6d4e5841194 100644 --- a/tools/asahi/m3-runtime-tests.rs +++ b/tools/asahi/m3-runtime-tests.rs @@ -46,3 +46,66 @@ mod m3_state_layout; #[path = "../../drivers/gpu/drm/asahi/m3/m3_compute_storage.rs"] mod m3_compute_storage; + +#[path = "../../drivers/gpu/drm/asahi/m3/m3_parameter_layout.rs"] +mod m3_parameter_layout; + +#[path = "../../drivers/gpu/drm/asahi/m3/m3_sync_layout.rs"] +mod m3_sync_layout; + +#[path = "../../drivers/gpu/drm/asahi/m3/agx_render.rs"] +mod agx_render; + +#[path = "../../drivers/gpu/drm/asahi/m3/agx_render_state.rs"] +mod agx_render_state; + +#[path = "../../drivers/gpu/drm/asahi/m3/m3_scene_layout.rs"] +mod m3_scene_layout; + +#[path = "../../drivers/gpu/drm/asahi/m3/m3_command_layout.rs"] +mod m3_command_layout; + +#[path = "../../drivers/gpu/drm/asahi/m3/m3_render_sequence.rs"] +mod m3_render_sequence; + +#[path = "../../drivers/gpu/drm/asahi/m3/m3_tiler_command.rs"] +mod m3_tiler_command; + +#[path = "../../drivers/gpu/drm/asahi/m3/m3_pass_layout.rs"] +mod m3_pass_layout; + +#[path = "../../drivers/gpu/drm/asahi/m3/m3_shared_layout.rs"] +mod m3_shared_layout; + +#[path = "../../drivers/gpu/drm/asahi/m3/m3_fragment_command.rs"] +mod m3_fragment_command; + +#[path = "../../drivers/gpu/drm/asahi/m3/agx_attachments.rs"] +mod agx_attachments; + +#[test] +fn extended_parameter_aliases_are_gpu_only_and_legacy_is_unchanged() { + use m3_pass_layout::{Access, Space}; + use m3_profile::Profile; + use m3_shared_layout as shared; + for index in 0..shared::COUNT { + let original = shared::allocation(index).unwrap(); + let legacy = shared::allocation_for(Profile::Legacy, index).unwrap(); + assert_eq!(legacy, original); + let extended = shared::allocation_for(Profile::Extended, index).unwrap(); + if [shared::PARAMETER_PAGE_LIST, shared::PARAMETER_BLOCK_LIST].contains(&index) { + assert_eq!(extended.space, Space::ClientGpu); + assert_eq!(extended.address, original.address); + assert_eq!(extended.size, original.size); + assert_eq!(extended.access, Access::GpuCachedRw); + assert_eq!(extended.access.pte(), 0xc0000000000080); + assert_eq!(original.access, Access::GpuFirmwareCachedRw); + } + if extended.space == Space::ClientGpu { + assert_ne!(extended.access, Access::GpuFirmwareCachedRw); + } + } +} + +#[path = "../../drivers/gpu/drm/asahi/m3/m3_completion_policy.rs"] +mod m3_completion_policy; diff --git a/tools/asahi/queue-priority-tests.py b/tools/asahi/queue-priority-tests.py new file mode 100644 index 0000000000000..1c5b4695c85f7 --- /dev/null +++ b/tools/asahi/queue-priority-tests.py @@ -0,0 +1,84 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only OR MIT +"""Compile the production queue admission gate with controlled credentials/backends.""" +from pathlib import Path +import subprocess +import tempfile +root = Path(__file__).resolve().parents[2] +s = (root / 'drivers/gpu/drm/asahi/file.rs').read_text() +a = s.index(' if data.flags != 0', s.index('pub(crate) fn queue_create(')) +b = s.index(' let queues_xa', a) +gate = s[a:b] +predicate = (root / 'drivers/gpu/drm/asahi/m3/m3_drm.rs').read_text().split('pub(crate) fn supports_high_priority(&self) -> bool {', 1)[1].split('\n }', 1)[0] +submit = (root / 'drivers/gpu/drm/asahi/m3/m3_submit.rs').read_text() +ordering = submit.split(' let urgency=', 1)[1].split(' Ok(Self', 1)[0] +ordering = 'let urgency=' + ordering +assert 'uapi::drm_asahi_priority_DRM_ASAHI_PRIORITY_REALTIME - data.priority' in s +unit = r''' +#![allow(dead_code, non_upper_case_globals)] +use std::any::Any; +const EINVAL:i32=22; const EACCES:i32=13; +macro_rules! cls_pr_debug { ($($arg:tt)*) => {} } +mod uapi { + pub const drm_asahi_priority_DRM_ASAHI_PRIORITY_REALTIME:u32=3; + pub const drm_asahi_priority_DRM_ASAHI_PRIORITY_HIGH:u32=2; +} +mod bindings { pub const CAP_SYS_NICE:u32=23; + pub unsafe fn capable(_:i32)->bool { CAP.with(|c|c.get()) } + thread_local! {pub static CAP:std::cell::Cell=const {std::cell::Cell::new(false)};} +} +mod g17 {pub struct Gpu;} +mod m3 {pub mod m3_drm { + #[derive(PartialEq)] pub enum Profile {Legacy, Extended} + pub struct Soc {pub chip_id:u32} + pub struct Backend {pub soc:Soc,pub profile:Profile} + impl Backend {pub fn supports_high_priority(&self)->bool { PREDICATE }} +}} +struct Gpu(Box); impl Gpu {fn as_any(&self)->&dyn Any {&*self.0}} +struct Reg {gpu:Gpu} +struct Data {flags:u32,priority:u32} +struct File(bool); impl File {fn is_current_master(&self)->bool {self.0}} +fn admit(reg_data:&Reg,data:&Data,file:&File)->Result<(),i32> { GATE Ok(()) } +mod sched { + #[derive(Debug,PartialEq)] pub enum Priority {Low,Normal,High,Kernel} +} +fn ordering(priority:u32)->Result<(u8,sched::Priority),i32> { + ORDERING + Ok((urgency,priority)) +} +fn main() { + use m3::m3_drm::*; + let mut n=0; + for backend in 0..5 { for cap in [false,true] { for master in [false,true] { + let gpu:Box=match backend { + 0=>Box::new(g17::Gpu), + 1=>Box::new(Backend{soc:Soc{chip_id:0x8122},profile:Profile::Extended}), + 2=>Box::new(Backend{soc:Soc{chip_id:0x8122},profile:Profile::Legacy}), + 3=>Box::new(Backend{soc:Soc{chip_id:0x6030},profile:Profile::Extended}), + _=>Box::new(())}; + let reg=Reg{gpu:Gpu(gpu)}; + bindings::CAP.with(|c|c.set(cap)); + for flags in [0,1,u32::MAX] { for priority in [0,1,2,3,4,u32::MAX] { + let expected=if flags!=0 || priority>=3 {Err(EINVAL)} + else if priority!=2 {Ok(())} + else if backend!=0 && !(backend==1 && cfg!(CONFIG_DRM_ASAHI_M3_T8122)) {Err(EINVAL)} + else if cap||master {Ok(())} else {Err(EACCES)}; + assert_eq!(admit(®,&Data{flags,priority},&File(master)),expected, + "backend={backend} cap={cap} master={master} flags={flags} priority={priority}"); n+=1; + }} + }}} + for (uapi,expected) in [(0,sched::Priority::Low),(1,sched::Priority::Normal),(2,sched::Priority::High)] { + assert_eq!(ordering(3-uapi),Ok((uapi as u8,expected))); + } + assert_eq!(ordering(4),Err(EINVAL)); + println!("{n} actual admission cases and LOW/MEDIUM/HIGH host mappings passed"); +} +'''.replace('PREDICATE', predicate.replace('super::m3_profile::Profile', 'Profile')).replace('GATE', gate).replace('ORDERING', ordering) +with tempfile.TemporaryDirectory(prefix='queue-priority-') as d: + p = Path(d) / 'gate.rs' + p.write_text(unit) + for enabled in [False, True]: + subprocess.run(['rustc', '--edition=2021', *(['--cfg', 'CONFIG_DRM_ASAHI_M3_T8122'] if enabled else []), str(p), '-o', str(p.with_suffix(''))], check=True) + # Without M3 compiled only the G17 backend is admitted. + subprocess.run([str(p.with_suffix(''))], check=True) +print('M3 enabled/disabled gate compilation passed') diff --git a/tools/asahi/run-host-tests.py b/tools/asahi/run-host-tests.py index ca9bae81672a7..470058e6a369b 100755 --- a/tools/asahi/run-host-tests.py +++ b/tools/asahi/run-host-tests.py @@ -53,6 +53,7 @@ def mutations(flag, names, base, expect=KILL): # test file -> [(arguments, expectation)]. '{out}' is a fresh directory. ASAHI_TESTS = { + 'queue-priority-tests.py': [([], PASS)], # The old revision reports a DART fault that clears before the second latch read as success. 'ane-tm-transient-fault-tests.py': [([], PASS), (['--old-ref', 'b50066848322c6799c1433ee3a358aab2e9faabb'], PASS)],