From c5e1b35de06e10f3587132f93ea51bac621c6510 Mon Sep 17 00:00:00 2001 From: Satya Benson Date: Fri, 30 Oct 2026 21:40:41 -0400 Subject: [PATCH 1/8] drm/asahi: dispatch G15 compute through extended T8122 firmware Dispatch extended firmware compute records through the existing M3 runtime while preserving G15 CDM streams. Retain owned pool backing and leave firmware-managed register setup to the extended interface. Signed-off-by: Satya Benson --- drivers/gpu/drm/asahi/m3/m3_compute.rs | 162 +++++++++++++++++-------- drivers/gpu/drm/asahi/m3/m3_config.rs | 10 ++ drivers/gpu/drm/asahi/m3/m3_device.rs | 2 +- drivers/gpu/drm/asahi/m3/m3_runtime.rs | 18 ++- 4 files changed, 132 insertions(+), 60 deletions(-) diff --git a/drivers/gpu/drm/asahi/m3/m3_compute.rs b/drivers/gpu/drm/asahi/m3/m3_compute.rs index 34aa0c0d2cd3f..577510b7e9ecf 100644 --- a/drivers/gpu/drm/asahi/m3/m3_compute.rs +++ b/drivers/gpu/drm/asahi/m3/m3_compute.rs @@ -40,25 +40,28 @@ pub(crate) struct Compute { barriers: Option, /// The register lists of the SoC's GPU (`m3_soc::Soc::registers`). registers: command::RegisterSet, + profile: super::m3_profile::Profile, /// Last: drops after the command aliases. _lower_tlbi: crate::m3::m3_memory::LowerTlbi, } impl Compute { pub(crate) fn new(dev: &driver::AsahiDrmDevice, uat: &mmu::Uat, vm: &mmu::Vm, stats: Region, control: crate::m3::agx_compute::Control, - registers: command::RegisterSet) -> Result { + registers: command::RegisterSet, profile: super::m3_profile::Profile) -> Result { let lower_tlbi=crate::m3::m3_memory::LowerTlbi; let binding=uat.bind(vm)?; let mut objects=KVec::new(); let mut client_command=None; let mut command_bytes = KVec::new(); - command_bytes.resize(command::SIZE, 0u8, GFP_KERNEL)?; + command_bytes.resize(if profile == super::m3_profile::Profile::Extended { + command::EXTENDED_SIZE + } else { command::SIZE }, 0u8, GFP_KERNEL)?; let mut sequence_bytes = KVec::new(); sequence_bytes.resize(microsequence::STORAGE, 0u8, GFP_KERNEL)?; // A private compute aperture, disjoint from the retained render graph. // Address-bearing graph fields are constructed from their live owners. for index in 0..storage::COUNT { - let a=storage::batch_allocation(index).map_err(|_|EINVAL)?; + let a=storage::batch_allocation_for(profile,index).map_err(|_|EINVAL)?; let mut b=match a.mapping { Mapping::CommandAlias(va)=>Buffer::at(dev,uat.kernel_vm(),Some((uat.kernel_lower_vm(),va)),None,a.size,true)?, Mapping::Client(va)=>Buffer::at(dev,uat.kernel_vm(),Some((vm,va)),None,a.size,true)?, @@ -68,10 +71,13 @@ impl Compute { if matches!(a.mapping,Mapping::CommandAlias(_)) {client_command=Some(b.map_gpu_view(vm)?);} objects.push(b,GFP_KERNEL)?; } - Self::encode_command(&mut objects, &mut command_bytes, binding.slot(), control, 1, 0, registers)?; - Self::encode_sequence(&mut objects, &mut sequence_bytes, stats, binding.slot(), timeline::stamp(0xc1000000,1), 0)?; - UMA_POOL.initialize(&mut objects)?; - Self::initialize_state(&mut objects, binding.slot())?; + Self::encode_command(&mut objects, &mut command_bytes, binding.slot(), control, 1, 0, + registers, profile, binding.generation())?; + Self::encode_sequence(&mut objects, &mut sequence_bytes, stats, binding.slot(), timeline::stamp(0xc1000000,1), 0, profile, 0)?; + if profile == super::m3_profile::Profile::Extended { + UMA_POOL.initialize_extended(&mut objects, 1, true)?; + } else { UMA_POOL.initialize(&mut objects)?; } + Self::initialize_state(&mut objects, binding.slot(), profile)?; // Queue references are firmware VAs from live allocations. In contrast // to render's unpublished seed entries, compute dispatches entry zero // immediately, so its initial producer cursor remains one. @@ -80,8 +86,13 @@ impl Compute { job_list: address(JOB_LIST)?, gpu_scratch: address(QUEUE_SCRATCH)?, context: address(GPU_CONTEXT)?, uuid: 0xc10000 }; let command = address(storage::COMMAND)?; - let mut bytes = [0; queue::INFO_SIZE]; - info.encode(&mut bytes).map_err(|_| EINVAL)?; + let mut bytes = KVec::new(); + bytes.resize(if profile == super::m3_profile::Profile::Extended { + queue::QueueInfo::EXTENDED_SIZE + } else { queue::INFO_SIZE }, 0u8, GFP_KERNEL)?; + if profile == super::m3_profile::Profile::Extended { + info.encode_extended(address(storage::QUEUE)?, &mut bytes).map_err(|_| EINVAL)?; + } else { info.encode(&mut bytes).map_err(|_| EINVAL)?; } objects[storage::QUEUE].write(0, &bytes)?; let mut state = [0; queue::STATE_SIZE]; queue::RingState { capacity: queue::QUALIFIED_CAPACITY, done: 0, read: 0, write: 1 } @@ -107,54 +118,74 @@ impl Compute { // the invalidation. crate::mem::sync();crate::mem::tlbi_all();crate::mem::sync(); unsafe {core::arch::asm!("isb",options(nostack,preserves_flags))}; - Ok(Self {_client_command:client_command.ok_or(EINVAL)?,objects,command_bytes,sequence_bytes,_binding:binding,stats,last_progress_ns:0,sequence:1,head:1,cached_views:KVec::new(),slot:0,batch_count:1,checkpoint:(0,0,0),barriers:None,registers,_lower_tlbi:lower_tlbi}) + Ok(Self {_client_command:client_command.ok_or(EINVAL)?,objects,command_bytes,sequence_bytes,_binding:binding,stats,last_progress_ns:0,sequence:1,head:1,cached_views:KVec::new(),slot:0,batch_count:1,checkpoint:(0,0,0),barriers:None,registers,profile,_lower_tlbi:lower_tlbi}) } fn encode_command(objects: &mut [Buffer], bytes: &mut [u8], context: u32, control: crate::m3::agx_compute::Control, sequence: u64, slot: usize, - register_set: command::RegisterSet) -> Result { - let offset = |owner:usize| storage::slot_offset(owner,slot).map_err(|_|EINVAL); + register_set: command::RegisterSet, profile: super::m3_profile::Profile, generation: u8) -> Result { + let offset = |owner:usize| storage::slot_offset_for(profile,owner,slot).map_err(|_|EINVAL); let fw = |owner: usize| queue::FirmwareVa::new(objects[owner].va()+offset(owner)? as u64).map_err(|_| EINVAL); let preemption = &objects[PREEMPTION]; let value = command::Command { context, counter: timeline::previous(sequence), notifier: fw(NOTIFIER)?, - preemption: GpuRegion::new(preemption.gpu_va()?+offset(PREEMPTION)? as u64, storage::allocation(PREEMPTION).map_err(|_|EINVAL)?.size).map_err(|_| EINVAL)?, + preemption: GpuRegion::new(preemption.gpu_va()?+offset(PREEMPTION)? as u64, storage::allocation_for(profile,PREEMPTION).map_err(|_|EINVAL)?.size).map_err(|_| EINVAL)?, cdm: command::GpuVa::new(control.base).map_err(|_| EINVAL)?, cdm_last: command::GpuVa::new(control.end.checked_sub(4).ok_or(EINVAL)?).map_err(|_| EINVAL)?, gpu_alias: command::GpuVa::new(objects[storage::COMMAND].gpu_va()?+offset(storage::COMMAND)? as u64).map_err(|_| EINVAL)?, - microsequence: fw(storage::MICROSEQUENCE)?, microsequence_size: microsequence::LENGTH as u32, + microsequence: fw(storage::MICROSEQUENCE)?, microsequence_size: + if profile == super::m3_profile::Profile::Extended { microsequence::Sequence::EXTENDED_LENGTH as u32 } + else { microsequence::LENGTH as u32 }, stamp: fw(storage::STAMP)?, fw_stamp: fw(storage::FW_STAMP)?, stamp_value: timeline::stamp(0xc1000000,sequence), timestamps: [fw(TIMESTAMPS[0])?, fw(TIMESTAMPS[1])?], pool: fw(storage::POOL)?, tail_scratch: fw(TAIL_SCRATCH)?, flush_stamps: false, register_set, }; - value.encode(bytes).map_err(|_| EINVAL)?; + value.encode_with_usc(profile, generation, 0, + if profile == super::m3_profile::Profile::Extended { control.usc_base } else { 0 }, bytes).map_err(|_| EINVAL)?; objects[storage::COMMAND].write(offset(storage::COMMAND)?, bytes) } fn encode_sequence(objects: &mut [Buffer], bytes: &mut [u8], stats: Region, - context: u32, stamp_value: u32, slot: usize) -> Result { - let offset = |owner:usize| storage::slot_offset(owner,slot).map_err(|_|EINVAL); + context: u32, stamp_value: u32, slot: usize, profile: super::m3_profile::Profile, + queue_sequence: u64) -> Result { + let offset = |owner:usize| storage::slot_offset_for(profile,owner,slot).map_err(|_|EINVAL); let fw = |owner: usize| queue::FirmwareVa::new(objects[owner].va()+offset(owner)? as u64).map_err(|_| EINVAL); - let region = |owner: usize| Region::new(objects[owner].va()+offset(owner)? as u64, storage::allocation(owner).map_err(|_|EINVAL)?.size).map_err(|_| EINVAL); + let region = |owner: usize| Region::new(objects[owner].va()+offset(owner)? as u64, storage::allocation_for(profile,owner).map_err(|_|EINVAL)?.size).map_err(|_| EINVAL); let value = microsequence::Sequence { command: region(storage::COMMAND)?, stats, notifier: region(NOTIFIER)?, - queue: fw(storage::QUEUE)?, context, generation: 1, pool: fw(storage::POOL)?, + queue: fw(storage::QUEUE)?, context, + generation: if profile == super::m3_profile::Profile::Extended { 0 } else { 1 }, pool: fw(storage::POOL)?, scratch: [fw(MICROSEQUENCE_SCRATCH[0])?, fw(MICROSEQUENCE_SCRATCH[1])?, fw(MICROSEQUENCE_SCRATCH[2])?, fw(MICROSEQUENCE_SCRATCH[3])?, fw(MICROSEQUENCE_SCRATCH[4])?], fw_stamp: fw(storage::FW_STAMP)?, stamp_value, }; - value.encode(bytes).map_err(|_| EINVAL)?; + if profile == super::m3_profile::Profile::Extended { + let auxiliary = command::GpuVa::new(objects[TAIL_SCRATCH].gpu_va()? + offset(TAIL_SCRATCH)? as u64).map_err(|_| EINVAL)?; + let context_store = GpuRegion::new(objects[MICROSEQUENCE_SCRATCH[1]].gpu_va()? + + offset(MICROSEQUENCE_SCRATCH[1])? as u64, 0x40000).map_err(|_| EINVAL)?; + value.encode_extended(auxiliary, context_store, queue_sequence, bytes).map_err(|_| EINVAL)?; + } else { value.encode(bytes).map_err(|_| EINVAL)?; } objects[storage::MICROSEQUENCE].write(offset(storage::MICROSEQUENCE)?, bytes) } - fn initialize_state(objects: &mut [Buffer], context_slot: u32) -> Result { + fn initialize_state(objects: &mut [Buffer], context_slot: u32, profile: super::m3_profile::Profile) -> Result { let address = |owner: usize| queue::FirmwareVa::new(objects[owner].va()).map_err(|_| EINVAL); let notification = state::Notifier { threshold: address(storage::EVENT)?, context: context_slot }; let list_owner = address(JOB_LIST)?; let mut context = [0; state::CONTEXT_SIZE]; - state::context(&mut context).map_err(|_| EINVAL)?; - objects[GPU_CONTEXT].write(0, &context)?; - let mut notifier = [0; state::NOTIFIER_SIZE]; - notification.encode(&mut notifier).map_err(|_| EINVAL)?; + if profile == super::m3_profile::Profile::Extended { + state::context_extended(&mut context).map_err(|_| EINVAL)?; + } else { state::context(&mut context).map_err(|_| EINVAL)?; } + let context_size = if profile == super::m3_profile::Profile::Extended { + state::EXTENDED_CONTEXT_SIZE + } else { state::CONTEXT_SIZE }; + objects[GPU_CONTEXT].write(0, &context[..context_size])?; + let mut notifier = KVec::new(); + notifier.resize(if profile == super::m3_profile::Profile::Extended { + state::EXTENDED_NOTIFIER_SIZE + } else { state::NOTIFIER_SIZE }, 0u8, GFP_KERNEL)?; + if profile == super::m3_profile::Profile::Extended { + notification.encode_extended(&mut notifier).map_err(|_| EINVAL)?; + } else { notification.encode(&mut notifier).map_err(|_| EINVAL)?; } objects[NOTIFIER].write(0, ¬ifier)?; let mut list = [0; state::JOB_LIST_SIZE]; state::job_list(&mut list, list_owner).map_err(|_| EINVAL)?; @@ -182,7 +213,7 @@ impl Compute { let binding = uat.bind(vm)?; let mut views = KVec::new(); for i in 0..storage::COUNT { - if matches!(storage::allocation(i).map_err(|_|EINVAL)?.mapping,Mapping::Client(_)) { + if matches!(storage::allocation_for(self.profile,i).map_err(|_|EINVAL)?.mapping,Mapping::Client(_)) { views.push((i,self.objects[i].map_gpu_view(vm)?),GFP_KERNEL)?; } } @@ -221,18 +252,17 @@ impl Compute { if self.batch_count>=storage::SLOTS {return Err(E2BIG);} self.slot=self.batch_count; for i in storage::RESET { - let offset=storage::slot_offset(i,self.slot).map_err(|_|EINVAL)?; - let size=storage::allocation(i).map_err(|_|EINVAL)?.size; + let offset=storage::slot_offset_for(self.profile,i,self.slot).map_err(|_|EINVAL)?; + let size=storage::allocation_for(self.profile,i).map_err(|_|EINVAL)?.size; self.objects[i].fill_range(offset,size,if i==PREEMPTION {0xcc} else {0})?; } self.objects[NOTIFIER].u32(state::NOTIFIER_CONTEXT,self._binding.slot())?; self.sequence=self.sequence.checked_add(1).ok_or(EOVERFLOW)?; - Self::encode_command(&mut self.objects,&mut self.command_bytes,self._binding.slot(),control,self.sequence,self.slot,self.registers)?; + Self::encode_command(&mut self.objects,&mut self.command_bytes,self._binding.slot(),control,self.sequence,self.slot,self.registers,self.profile,self._binding.generation())?; let stamp=self.stamp(); - Self::encode_sequence(&mut self.objects,&mut self.sequence_bytes,self.stats,self._binding.slot(),stamp,self.slot)?; - self.objects[storage::EVENT].u32(0,self.sequence as u32)?; - self.objects[storage::COUNTER].u32(0,self.sequence as u32)?; - let address=self.objects[storage::COMMAND].va()+storage::slot_offset(storage::COMMAND,self.slot).map_err(|_|EINVAL)? as u64; + Self::encode_sequence(&mut self.objects,&mut self.sequence_bytes,self.stats,self._binding.slot(),stamp,self.slot,self.profile,self.sequence - 1)?; + self.write_counters()?; + let address=self.objects[storage::COMMAND].va()+storage::slot_offset_for(self.profile,storage::COMMAND,self.slot).map_err(|_|EINVAL)? as u64; self.objects[storage::RING].u64(usize::from(self.head)*8,address)?; self.head=(self.head+1)%queue::QUALIFIED_CAPACITY as u16; self.batch_count+=1; @@ -245,39 +275,55 @@ impl Compute { pub(crate) fn abort_batch(&mut self)->Result { (self.sequence,self.head,self.batch_count)=self.checkpoint; self.slot=self.batch_count.saturating_sub(1); - self.objects[storage::EVENT].u32(0,self.sequence as u32)?; - self.objects[storage::COUNTER].u32(0,self.sequence as u32)?; + self.write_counters()?; self.objects[storage::QUEUE_STATE].u32(queue::WRITE,u32::from(self.head))?; agx_memory::publish();Ok(()) } + fn write_counters(&mut self) -> Result { + for index in [storage::EVENT, storage::COUNTER] { + if self.profile == super::m3_profile::Profile::Extended { + self.objects[index].u64(0, self.sequence)?; + } else { self.objects[index].u32(0, self.sequence as u32)?; } + } + Ok(()) + } + fn offset(&self,owner:usize)->Result { - storage::slot_offset(owner,self.slot).map_err(|_|EINVAL) + storage::slot_offset_for(self.profile,owner,self.slot).map_err(|_|EINVAL) } /// Same cache-generation contract as M4: StartCompute and the persistent /// notifier must use the same tag before the firmware flushes user stamps. /// RTKit 2419: StartCompute.counter1 +0x34; JobMeta.flush_stamps +0x804. pub(crate) fn prepare_completion(&mut self,coalesce:bool)->Result { + if coalesce && self.profile == super::m3_profile::Profile::Extended { return Err(EINVAL); } let sequence=self.offset(storage::MICROSEQUENCE)?; let command=self.offset(storage::COMMAND)?; self.objects[storage::MICROSEQUENCE].u32(sequence+microsequence::GENERATION,0)?; - self.objects[storage::COMMAND].u32(command+command::FLUSH_STAMPS,u32::from(!coalesce))?; + self.objects[storage::COMMAND].u32(command+if self.profile == super::m3_profile::Profile::Extended { 0x81c } else { command::FLUSH_STAMPS },u32::from(!coalesce))?; agx_memory::publish();Ok(()) } pub(crate) fn set_user_timestamps(&mut self,addresses:[u64;2])->Result { - let offset=self.offset(storage::COMMAND)?+command::USER_TIMESTAMPS; + let offset=self.offset(storage::COMMAND)?+if self.profile == super::m3_profile::Profile::Extended { + 0x844 + } else { command::USER_TIMESTAMPS }; self.objects[storage::COMMAND].u64(offset,addresses[0])?; self.objects[storage::COMMAND].u64(offset+8,addresses[1])?; let pointer=if addresses==[0,0] {0} else {self.objects[storage::COMMAND].va()+offset as u64}; let sequence=self.offset(storage::MICROSEQUENCE)?; - for start in [microsequence::TIMESTAMP_START,microsequence::TIMESTAMP_END] { + for start in if self.profile == super::m3_profile::Profile::Extended { [0x1bc, 0x20c] } + else { [microsequence::TIMESTAMP_START,microsequence::TIMESTAMP_END] } { self.objects[storage::MICROSEQUENCE].u64(sequence+start+microsequence::USER_TIMESTAMP_POINTER,pointer)?; } agx_memory::publish();Ok(()) } pub(crate) fn set_attachments(&mut self,attachments:&crate::m3::agx_attachments::Attachments)->Result { let sequence=self.offset(storage::MICROSEQUENCE)?; - self.objects[storage::MICROSEQUENCE].write(sequence+microsequence::ATTACHMENTS,&attachments.0)?; - self.objects[storage::MICROSEQUENCE].write(sequence+microsequence::HAS_ATTACHMENTS,&[u8::from(attachments.count()!=0)])?; + self.objects[storage::MICROSEQUENCE].write(sequence+if self.profile == super::m3_profile::Profile::Extended { + microsequence::Sequence::EXTENDED_ATTACHMENTS + } else { microsequence::ATTACHMENTS },&attachments.0)?; + self.objects[storage::MICROSEQUENCE].write(sequence+if self.profile == super::m3_profile::Profile::Extended { + microsequence::Sequence::EXTENDED_HAS_ATTACHMENTS + } else { microsequence::HAS_ATTACHMENTS },&[u8::from(attachments.count()!=0)])?; agx_memory::publish();Ok(()) } fn stamp(&self) -> u32 { timeline::stamp(0xc1000000,self.sequence) } @@ -319,26 +365,31 @@ impl Compute { agx_memory::publish();crate::mem::sync();Ok(()) } pub(crate) fn gpu_ns(&mut self)->Result { - let mut total=0; - for slot in 0..self.batch_count {total+=self.slot_gpu_ns(slot)?;} + let mut total: u64 = 0; + for slot in 0..self.batch_count { + total = total.checked_add(self.slot_gpu_ns(slot)?).ok_or(EOVERFLOW)?; + } Ok(total) } pub(crate) fn slot_gpu_ns(&mut self,slot:usize)->Result { if slot>=self.batch_count {return Err(EINVAL);} - let start=self.objects[TIMESTAMPS[0]].read_u64(storage::slot_offset(TIMESTAMPS[0],slot).map_err(|_|EINVAL)?)?; - let end=self.objects[TIMESTAMPS[1]].read_u64(storage::slot_offset(TIMESTAMPS[1],slot).map_err(|_|EINVAL)?)?; - Ok(if start!=0 && end>=start {(end-start)*1000/24} else {0}) + let start=self.objects[TIMESTAMPS[0]].read_u64(storage::slot_offset_for(self.profile,TIMESTAMPS[0],slot).map_err(|_|EINVAL)?)?; + let end=self.objects[TIMESTAMPS[1]].read_u64(storage::slot_offset_for(self.profile,TIMESTAMPS[1],slot).map_err(|_|EINVAL)?)?; + if start == 0 || end < start { return Ok(0); } + self.profile.gpu_duration_ns(end - start).ok_or(EOVERFLOW) } /// A batch span includes gaps between its compute commands, unlike gpu_ns. /// Read only after stamps, queues, events and idle state prove retirement. pub(crate) fn batch_gpu_span(&mut self)->Result<[u64;2]> { if self.batch_count == 0 {return Err(EINVAL);} - Ok([self.objects[TIMESTAMPS[0]].read_u64(storage::slot_offset(TIMESTAMPS[0],0).map_err(|_|EINVAL)?)?, - self.objects[TIMESTAMPS[1]].read_u64(storage::slot_offset(TIMESTAMPS[1],self.batch_count-1).map_err(|_|EINVAL)?)?]) + Ok([self.objects[TIMESTAMPS[0]].read_u64(storage::slot_offset_for(self.profile,TIMESTAMPS[0],0).map_err(|_|EINVAL)?)?, + self.objects[TIMESTAMPS[1]].read_u64(storage::slot_offset_for(self.profile,TIMESTAMPS[1],self.batch_count-1).map_err(|_|EINVAL)?)?]) } pub(crate) fn head(&self) -> u16 { self.head } pub(crate) fn ordinal(&self)->u64 {self.sequence} - pub(crate) fn first(&self) -> bool { self.sequence == self.batch_count as u64 } + pub(crate) fn first(&self) -> bool { + self.profile == super::m3_profile::Profile::Legacy && self.sequence == self.batch_count as u64 + } pub(crate) fn progress(&mut self, dev: &driver::AsahiDrmDevice) -> Result { // Report real completions by elapsed time as well as sequence count; // batches need not land on a multiple of 128. @@ -361,10 +412,15 @@ impl Compute { pub(crate) fn queue(&self)->u64 {self.objects[storage::QUEUE].va()} pub(crate) fn complete(&mut self)->Result { - Ok(self.objects[storage::STAMP].read_u32(0)?==self.stamp() - && self.objects[storage::FW_STAMP].read_u32(0)?==self.stamp() - && self.objects[storage::QUEUE_STATE].read_u32(queue::DONE)?==u32::from(self.head) - && self.objects[storage::QUEUE_STATE].read_u32(queue::READ)?==u32::from(self.head)) + let stamped = self.objects[storage::STAMP].read_u32(0)? == self.stamp() + && self.objects[storage::FW_STAMP].read_u32(0)? == self.stamp() + && self.objects[storage::QUEUE_STATE].read_u32(queue::DONE)? == u32::from(self.head) + && self.objects[storage::QUEUE_STATE].read_u32(queue::READ)? == u32::from(self.head); + if !stamped { return Ok(false); } + if self.profile == super::m3_profile::Profile::Extended { + return Ok(self.objects[storage::POOL].read_u64(0x54)? == self.sequence); + } + Ok(true) } pub(crate) fn log(&mut self,dev:&driver::AsahiDrmDevice)->Result { for off in (0..512).step_by(128) { diff --git a/drivers/gpu/drm/asahi/m3/m3_config.rs b/drivers/gpu/drm/asahi/m3/m3_config.rs index bfb408b07a530..ffd1bb963b905 100644 --- a/drivers/gpu/drm/asahi/m3/m3_config.rs +++ b/drivers/gpu/drm/asahi/m3/m3_config.rs @@ -177,6 +177,16 @@ impl Config { Ok(()) } + pub(crate) fn global_table_views(&mut self, vm: &mmu::Vm) -> Result> { + let mut mappings = KVec::new(); + if self.profile == super::m3_profile::Profile::Extended { + for index in [super::m3_init_26::GLOBAL_POOL, super::m3_init_26::GLOBAL_PARAMETERS] { + mappings.push(self.objects[index].map_gpu_view(vm)?, GFP_KERNEL)?; + } + } + Ok(mappings) + } + pub(crate) fn root(&self) -> u64 { self.objects[storage::ROOT].va() } pub(crate) fn log_globals_debug(&mut self,dev:&driver::AsahiDrmDevice)->Result { let value=self.objects[GLOBALS].read_u32(crate::m3::m3_globals_debug::OFFSET)?; diff --git a/drivers/gpu/drm/asahi/m3/m3_device.rs b/drivers/gpu/drm/asahi/m3/m3_device.rs index 05a3961369a6d..474a8b3f561ec 100644 --- a/drivers/gpu/drm/asahi/m3/m3_device.rs +++ b/drivers/gpu/drm/asahi/m3/m3_device.rs @@ -129,6 +129,7 @@ impl Device { } e.sgx_setup() } + None if device.firmware.profile() == super::m3_profile::Profile::Extended => None, None => Some(soc.sgx_setup.ok_or(ENODEV)?), }; if let Some((setup_offset, setup_value)) = setup { @@ -179,7 +180,6 @@ impl Device { if self.firmware.profile() == super::m3_profile::Profile::Extended { // Adaptive clock gating can leave individual engine banks inaccessible // even while the GPU performance-state register says powered up. - dev_info!(self.dev.as_ref(), "M3: adaptive firmware fault evidence uses retained metadata; engine banks may be gated\n"); return Ok(()); } let sgx_guard = self.sgx.try_access().ok_or(ENODEV)?; diff --git a/drivers/gpu/drm/asahi/m3/m3_runtime.rs b/drivers/gpu/drm/asahi/m3/m3_runtime.rs index 40399a80a01f3..058c6610399ae 100644 --- a/drivers/gpu/drm/asahi/m3/m3_runtime.rs +++ b/drivers/gpu/drm/asahi/m3/m3_runtime.rs @@ -389,7 +389,7 @@ impl Inner { let index=found.unwrap_or(self.jobs.len()); if found.is_none() { let job=match control { - crate::m3::m3_submit::Command::Compute(c)=>NativeJob::Compute(crate::m3::m3_compute::Compute::new(&self.drm,&self.uat,&packet.vm,self.config.stats_region().map_err(early)?,c,self.device.soc().registers).map_err(early)?), + crate::m3::m3_submit::Command::Compute(c)=>NativeJob::Compute(crate::m3::m3_compute::Compute::new(&self.drm,&self.uat,&packet.vm,self.config.stats_region().map_err(early)?,c,self.device.soc().registers,self.device.firmware().profile()).map_err(early)?), crate::m3::m3_submit::Command::Render{..}=>NativeJob::Render(crate::m3::m3_render::Render::new(&self.drm,&self.uat,&packet.vm,self.config.stats_region().map_err(early)?,self.device.soc().clusters,self.device.soc().registers).map_err(early)?), }; self.jobs.push(job,GFP_KERNEL).map_err(|_|early(ENOMEM))?; @@ -469,6 +469,9 @@ impl Inner { crate::mem::sync(); self.t8122_verdict=false; let previous_events=self.config.completed_events; + if self.device.firmware().profile() == super::m3_profile::Profile::Extended { + packet.vm.mark_binding_published().map_err(early)?; + } let publication_tick=if crate::debug::debug_enabled(crate::debug::DebugFlags::M3PassTiming) {physical_counter()} else {0}; self.state.health.set_gpu_pending(true); self.gpu_pending = true; @@ -535,7 +538,7 @@ impl Inner { let control=first_compute.ok_or(EINVAL)?; if existing.is_none() { let job=crate::m3::m3_compute::Compute::new(&self.drm,&self.uat,&packet.vm, - self.config.stats_region()?,control,self.device.soc().registers)?; + self.config.stats_region()?,control,self.device.soc().registers,self.device.firmware().profile())?; self.jobs.push(NativeJob::Compute(job),GFP_KERNEL)?; } else { let NativeJob::Compute(j)=&mut self.jobs[compute_index] else {return Err(EINVAL);}; @@ -1059,7 +1062,7 @@ impl Runtime { // TTBAT of the admitted reservations, the firmware kernel root (`pagetables`) as // every context's upper half, and the host window (root entry 2) through the // shared L2. The RTKit is awake and no initdata or GPU job has been published. - let (uat,window)=adopted.insert(adopt_uat(&drm,device.soc(),&device.firmware().resources)?); + let (uat,window)=adopted.insert(adopt_uat(&drm,device.soc(),&device.firmware().resources,device.firmware().profile())?); let config=Config::new(&drm,uat,device.firmware(),&contents)?; // Link every kernel table the configuration created before the firmware sees an // address in it, then drop any translation cached before they existed. @@ -1103,7 +1106,8 @@ impl Runtime { // (7.1.12's M3 VM job lifetime, the shared `mmu::lifetime` on 7.2). let vm=self.inner.uat.new_vm(id, range)?.with_tracked_status()?; let mut reserved=KVec::new();reserved.push(reserved_range,GFP_KERNEL)?;reserved.push(0x100_8000_0000..0x101_0000_0000,GFP_KERNEL)?;reserved.push(0x10_0000_0000..0x10_0800_0000,GFP_KERNEL)?;reserved.push(0x10_7400_0000..0x10_7402_8000,GFP_KERNEL)?; - vm.install_driver_mappings(KVec::new(),reserved)?; + let mappings = self.inner.config.global_table_views(&vm)?; + vm.install_driver_mappings(mappings,reserved)?; Ok(vm) } pub(crate) fn map_timestamp(&mut self,mut bo:crate::gem::ObjectRef,range:core::ops::Range)->Result { @@ -1369,13 +1373,15 @@ pub(crate) fn physical_counter() -> u64 { /// (ttbs, pagetables, handoff, shared-l2) and the PPL exchange with its startup epoch, as the /// G15 backend does it on 7.2 (`g15::uat`). fn adopt_uat(drm: &driver::AsahiDrmDevice, soc: &'static crate::m3::m3_soc::Soc, - resources: &crate::m3::agx_resources::Resources) -> Result<(mmu::Uat, crate::g15::uat::Window)> { + resources: &crate::m3::agx_resources::Resources, profile: super::m3_profile::Profile) + -> Result<(mmu::Uat, crate::g15::uat::Window)> { let cfg = soc.hwcfg.ok_or(ENODEV)?; let [ttbs, tables, handoff, shared_l2, ..] = resources.regions; let range = |r: crate::m3::agx_resources::Region| r.base..r.base + r.size; let geometry = mmu::UatGeometry { ias: cfg.uat_ias.into(), oas: cfg.uat_oas }; let regions = mmu::AdoptedRegions { handoff: range(handoff), ttbs: range(ttbs), kernel_root: tables.base }; - let uat = mmu::Uat::new_adopted(drm, geometry, ®ions)?; + let uat = mmu::Uat::new_adopted_with_user_upper(drm, geometry, ®ions, + profile == super::m3_profile::Profile::Legacy)?; let mut window = crate::g15::uat::Window::from_regions(handoff.base, ttbs.base, tables.base, shared_l2.base)?; window.bootstrap(drm.as_ref().as_ref(), &uat)?; // 7.1.12 cleaned every reserved-table write to the point of coherency; these regions may be From ec30b227fefa70ee9bb6b962bceb2bf03495d791 Mon Sep 17 00:00:00 2001 From: Satya Benson Date: Fri, 30 Oct 2026 21:40:42 -0400 Subject: [PATCH 2/8] drm/asahi: enable J613 extended firmware render and retirement Encode extended tiler and fragment work, scene records, independent notifications and attachment visibility through the existing runtime. Retire native pools and enable only the exact J613 26.6.2 profile. Keep the public UAPI and physical G15 stream formats unchanged. Signed-off-by: Satya Benson --- drivers/gpu/drm/asahi/m3/m3_board.rs | 2 +- drivers/gpu/drm/asahi/m3/m3_compute.rs | 10 +- drivers/gpu/drm/asahi/m3/m3_config.rs | 12 + drivers/gpu/drm/asahi/m3/m3_drm.rs | 10 +- drivers/gpu/drm/asahi/m3/m3_firmware.rs | 10 +- .../gpu/drm/asahi/m3/m3_fragment_command.rs | 138 ++++++++++-- .../gpu/drm/asahi/m3/m3_parameter_layout.rs | 74 +++++++ drivers/gpu/drm/asahi/m3/m3_pass.rs | 4 +- drivers/gpu/drm/asahi/m3/m3_pass_layout.rs | 36 +++ drivers/gpu/drm/asahi/m3/m3_profile.rs | 21 ++ drivers/gpu/drm/asahi/m3/m3_render.rs | 206 +++++++++++++----- .../gpu/drm/asahi/m3/m3_render_sequence.rs | 168 ++++++++++++++ drivers/gpu/drm/asahi/m3/m3_runtime.rs | 4 +- drivers/gpu/drm/asahi/m3/m3_scene_layout.rs | 40 ++++ drivers/gpu/drm/asahi/m3/m3_shared_layout.rs | 26 +++ drivers/gpu/drm/asahi/m3/m3_submit.rs | 7 +- drivers/gpu/drm/asahi/m3/m3_sync_layout.rs | 55 +++++ drivers/gpu/drm/asahi/m3/m3_tiler_command.rs | 131 +++++++++-- drivers/gpu/drm/asahi/m3/t8122_admission.rs | 40 +++- tools/asahi/m3-runtime-tests.rs | 60 +++++ 20 files changed, 953 insertions(+), 101 deletions(-) diff --git a/drivers/gpu/drm/asahi/m3/m3_board.rs b/drivers/gpu/drm/asahi/m3/m3_board.rs index c44b2a42d99d5..5d4ea4277b618 100644 --- a/drivers/gpu/drm/asahi/m3/m3_board.rs +++ b/drivers/gpu/drm/asahi/m3/m3_board.rs @@ -13,7 +13,7 @@ use crate::m3::m3_resources::{ use crate::m3::m3_soc::Soc; /// Whether the root node's compatible list contains `compatible`. -fn board_is(compatible: &[u8]) -> bool { +pub(crate) fn board_is(compatible: &[u8]) -> bool { let Some(root) = kernel::of::root() else { return false; }; diff --git a/drivers/gpu/drm/asahi/m3/m3_compute.rs b/drivers/gpu/drm/asahi/m3/m3_compute.rs index 577510b7e9ecf..b6e9444ed5449 100644 --- a/drivers/gpu/drm/asahi/m3/m3_compute.rs +++ b/drivers/gpu/drm/asahi/m3/m3_compute.rs @@ -350,10 +350,12 @@ impl Compute { let barriers=self.barriers.as_mut().ok_or(EINVAL)?; let offset=self.slot*64; let mut bytes=[0;crate::m3::m3_sync_layout::BARRIER_SIZE]; - crate::m3::m3_sync_layout::Barrier { + let barrier=crate::m3::m3_sync_layout::Barrier { stamp:dependency.stamp,wait_value:dependency.value,event:dependency.event, self_value:value,uuid:0xc10000,dependency:crate::m3::m3_sync_layout::Dependency::PreviousPass, - }.encode(&mut bytes).map_err(|_|EINVAL)?; + }; + if self.profile==super::m3_profile::Profile::Extended {barrier.encode_extended(&mut bytes)} + else {barrier.encode(&mut bytes)}.map_err(|_|EINVAL)?; barriers.write(offset,&bytes)?; let address=barriers.va()+offset as u64; let (slots,next)=queue::insert_before_last(self.head,queue::QUALIFIED_CAPACITY).map_err(|_|EINVAL)?; @@ -382,8 +384,8 @@ impl Compute { /// Read only after stamps, queues, events and idle state prove retirement. pub(crate) fn batch_gpu_span(&mut self)->Result<[u64;2]> { if self.batch_count == 0 {return Err(EINVAL);} - Ok([self.objects[TIMESTAMPS[0]].read_u64(storage::slot_offset_for(self.profile,TIMESTAMPS[0],0).map_err(|_|EINVAL)?)?, - self.objects[TIMESTAMPS[1]].read_u64(storage::slot_offset_for(self.profile,TIMESTAMPS[1],self.batch_count-1).map_err(|_|EINVAL)?)?]) + Ok([self.profile.profiler_ticks(self.objects[TIMESTAMPS[0]].read_u64(storage::slot_offset_for(self.profile,TIMESTAMPS[0],0).map_err(|_|EINVAL)?)?), + self.profile.profiler_ticks(self.objects[TIMESTAMPS[1]].read_u64(storage::slot_offset_for(self.profile,TIMESTAMPS[1],self.batch_count-1).map_err(|_|EINVAL)?)?)]) } pub(crate) fn head(&self) -> u16 { self.head } pub(crate) fn ordinal(&self)->u64 {self.sequence} diff --git a/drivers/gpu/drm/asahi/m3/m3_config.rs b/drivers/gpu/drm/asahi/m3/m3_config.rs index ffd1bb963b905..02b847e210dc9 100644 --- a/drivers/gpu/drm/asahi/m3/m3_config.rs +++ b/drivers/gpu/drm/asahi/m3/m3_config.rs @@ -246,6 +246,18 @@ impl Config { self.thermal.busy(now); Ok(()) } + pub(crate) fn render_stats(&self) -> Result<[init::Region;2]> { + if self.profile==super::m3_profile::Profile::Legacy { + let value=self.stats_region()?;return Ok([value;2]); + } + let region=|index:usize,offset:usize| { + let object=&self.objects[index]; + init::Region::new(object.va().checked_add(offset as u64).ok_or(EOVERFLOW)?, + object.size().checked_sub(offset).ok_or(EINVAL)?).map_err(|_|EINVAL) + }; + Ok([region(UNKNOWN_C0,4)?,region(UNKNOWN_C1,8)?]) + } + pub(crate) fn stats_region(&self) -> Result { let owner = &self.objects[if self.profile == super::m3_profile::Profile::Extended { UNKNOWN_C3 diff --git a/drivers/gpu/drm/asahi/m3/m3_drm.rs b/drivers/gpu/drm/asahi/m3/m3_drm.rs index e1db1adf9e6a9..8aedd920f31dc 100644 --- a/drivers/gpu/drm/asahi/m3/m3_drm.rs +++ b/drivers/gpu/drm/asahi/m3/m3_drm.rs @@ -113,10 +113,12 @@ impl Backend { crate::m3::t8122_start::not_admitted(pdev.as_ref(), soc); })?; dev_info!(pdev.as_ref(), "M3: resource admission complete\n"); - // An armed T8122 start supplies the values its table leaves out. An unarmed T8122 - // needs a complete table, which it does not have: it is refused here. - let experiment = crate::m3::t8122_start::arm(pdev.as_ref(), soc)?; - if experiment.is_none() { + // Extended T8122 uses its dedicated initialization graph. Keep the + // legacy guarded-start and completeness checks for older interfaces. + let extended=crate::m3::t8122_start::is_t8122(soc) + && crate::m3::t8122_admission::loaded_profile(pdev)?==super::m3_profile::Profile::Extended; + let experiment = if extended {None} else {crate::m3::t8122_start::arm(pdev.as_ref(), soc)?}; + if experiment.is_none() && !extended { soc.require_complete(pdev.as_ref())?; } let experiment = experiment.as_ref(); diff --git a/drivers/gpu/drm/asahi/m3/m3_firmware.rs b/drivers/gpu/drm/asahi/m3/m3_firmware.rs index 6bc87a33aaa05..94a2d3384d08a 100644 --- a/drivers/gpu/drm/asahi/m3/m3_firmware.rs +++ b/drivers/gpu/drm/asahi/m3/m3_firmware.rs @@ -62,7 +62,7 @@ pub(crate) static KNOWN_IMAGES_T8122: [KnownImage; 1] = [KnownImage { initdata_magic: Some(G15_V14_8_3_INITDATA), }]; -/// Image record for the extended format. The SoC's runtime selection does not use it yet. +/// Exact normalized image identity for the extended firmware interface. pub(crate) static T8122_IMAGES_26: [KnownImage; 1] = [KnownImage { name: "T8122 C0 firmware-compat 26.6.2", uuid: None, @@ -164,7 +164,7 @@ pub(crate) static T8122_LAYOUT: Layout = Layout { pub(crate) const TEXT_SHA256: [u8; 32] = [ 0x11, 0xe4, 0x9f, 0x75, 0xb, 0x67, 0x1a, 0x2b, 0x13, 0xd0, 0x92, 0xdb, 0x1c, 0xbc, 0xa3, 0xa8, 0x77, 0x55, 0x86, 0xd6, 0xb1, 0xe3, 0xaa, 0xe8, 0xa6, 0xa4, 0x64, 0xa7, 0x80, 0x2, 0x93, 0xd5]; -/// T8122 C0, firmware-compat 26.6.2. Selection remains separate from runtime admission. +/// T8122 C0, firmware-compat 26.6.2 segment and loader-mutable bounds. pub(crate) static T8122_LAYOUT_26: Layout = Layout { text_size: 393216, data_size: 1228800, @@ -313,7 +313,10 @@ pub(crate) fn identify_loaded( prelude::*, }; - let layout = soc.firmware.ok_or(ENODEV)?; + let extended=crate::m3::t8122_start::is_t8122(soc) + && crate::m3::t8122_admission::loaded_profile(pdev)?==super::m3_profile::Profile::Extended; + if extended && experiment.is_some() {return Err(EINVAL);} + let layout = if extended {&T8122_LAYOUT_26} else {soc.firmware.ok_or(ENODEV)?}; if !Firmware::matching_layout(layout, &resources) { dev_err!(pdev.as_ref(), "M3 {}: unsupported firmware segment layout\n", soc.gpu_name); return Err(ENODEV); @@ -337,6 +340,7 @@ pub(crate) fn identify_loaded( // The T8122 start experiment identifies the same image, with its own InitData version. let images = match experiment { Some(_) => &KNOWN_IMAGES_T8122_EXPERIMENT[..], + None if extended => &T8122_IMAGES_26[..], None => soc.images, }; let image = crate::m3::m3_board::identify(pdev.as_ref(), bytes, &digest, images).ok_or(ENODEV)?; diff --git a/drivers/gpu/drm/asahi/m3/m3_fragment_command.rs b/drivers/gpu/drm/asahi/m3/m3_fragment_command.rs index 44671ee3cd0ab..e0aaf2584284a 100644 --- a/drivers/gpu/drm/asahi/m3/m3_fragment_command.rs +++ b/drivers/gpu/drm/asahi/m3/m3_fragment_command.rs @@ -3,6 +3,8 @@ use super::{agx_render::Geometry, agx_render_state::{Program,DepthStencil}, m3_compute_layout::{GpuVa,RegisterSet}, m3_init_layout::{Error,Region}, m3_queue_layout::FirmwareVa, m3_render_sequence::fragment}; pub(crate) const SIZE:usize=0xc73; +pub(crate) const EXTENDED_SIZE:usize=0xca0; +pub(crate) const EXTENDED_FLUSH:usize=0xbdc; pub(crate) const REGISTERS:usize=fragment::REGISTERS; /// The registers G15G does not take; the others keep the G15S order. pub(crate) const G15G_OMIT:[u32;4]=[0x160a8,0x160b8,0xa5a9,0xd429]; @@ -74,16 +76,30 @@ impl Command { /// Check references before touching output. The caller owns validation of /// client resource mappings; this layer serializes their semantic state. pub(crate) fn encode(&self,out:&mut[u8])->Result<(),Error> { - if out.len()Result<(),Error> { + self.encode_format(out,Some(dispatch)) + } + fn encode_format(&self,out:&mut[u8],dispatch:Option)->Result<(),Error> { + let extended=dispatch.is_some(); + let size=if extended {EXTENDED_SIZE} else {SIZE}; + if out.len()=1<<42 || self.register_set!=RegisterSet::G15G {return Err(Error::Address);} + } if self.context>=64 {return Err(Error::Address);} - let seq=self.sequence.at(0,fragment::LENGTH)?; + let sequence_size=if extended {0x370} else {fragment::LENGTH}; + let seq=self.sequence.at(0,sequence_size)?; let alias=self.gpu_alias.get(); - GpuVa::new(alias.checked_add((SIZE-3) as u64).ok_or(Error::Address)?).map_err(|_|Error::Address)?; + GpuVa::new(alias.checked_add(((size-1)&!3) as u64).ok_or(Error::Address)?).map_err(|_|Error::Address)?; let registers=alias+REGISTERS as u64; let s=&self.state;let g=&s.geometry; let layer_metadata=self.heap.get().checked_add(if g.layers>1 {0x200}else{0}).ok_or(Error::Address)?; GpuVa::new(layer_metadata).map_err(|_|Error::Address)?; - let user=super::agx_render_state::compact(self.scene_user.get().checked_add(0x40).ok_or(Error::Address)?) + let user=super::agx_render_state::compact(self.scene_user.get().checked_add(if extended {0} else {0x40}).ok_or(Error::Address)?) .map_err(|_|Error::Address)? & !15; let width=(g.pixels&0xffff)+1;let height=(g.pixels>>16)+1; let tx=width.div_ceil(32);let ty=height.div_ceil(32); @@ -116,43 +132,85 @@ impl Command { (0x101c1,0),(0xd469,0),(0x1a0f9,8),(0x1c838,0),(0x1ca28,user), (0xa5a9,0x6000400020),(0xd429,0x200000001), ]; - let bytes=&mut out[..SIZE];bytes.fill(0); + let bytes=&mut out[..size];bytes.fill(0); word(bytes,0,1); // fragment opcode; work counter remains zero word(bytes,0xc,self.context); for (offset,value) in [(0x14,seq),(0x20,self.notifier.get()),(0x28,self.manager.get()), (0x30,self.scene.get()),(0x38,self.empty.get()),(0x40,self.tilemap.get()),(0x780,registers)] {pointer(bytes,offset,value);} - word(bytes,SEQUENCE_LENGTH,fragment::LENGTH as u32); - // The qualified patch path updates register MSAA but retains these - // producer header values. Representation cleanup must not change that. - pointer(bytes,0x48,0x88);word(bytes,0x50,1); - bytes[0x54..0x56].copy_from_slice(&((g.y_blocks>>18) as u16).to_le_bytes()); - bytes[0x56..0x58].copy_from_slice(&((g.x_blocks>>18) as u16).to_le_bytes()); - word(bytes,0x68,s.merge_upper[0]);word(bytes,0x6c,s.merge_upper[1]); - pointer(bytes,0x78,u64::from(g.tiles)); + word(bytes,SEQUENCE_LENGTH,sequence_size as u32); + if extended { + pointer(bytes,4,dispatch.as_ref().map_or(0,|native|native.serial)); + word(bytes,0x50,1); + word(bytes,0x54,g.isp_blocks); + word(bytes,0x60,g.x_blocks); + word(bytes,0x64,g.y_blocks); + word(bytes,0x68,g.blocks); + word(bytes,0x78,g.tiles); + } else { + pointer(bytes,0x48,0x88);word(bytes,0x50,1); + bytes[0x54..0x56].copy_from_slice(&((g.y_blocks>>18) as u16).to_le_bytes()); + bytes[0x56..0x58].copy_from_slice(&((g.x_blocks>>18) as u16).to_le_bytes()); + word(bytes,0x68,s.merge_upper[0]);word(bytes,0x6c,s.merge_upper[1]); + pointer(bytes,0x78,u64::from(g.tiles)); + } let g15g=self.register_set==RegisterSet::G15G; let mut count=0; - for (register,value) in values.into_iter().filter(|(r,_)| !g15g || !G15G_OMIT.contains(r)) { - let n=count;count+=1; - word(bytes,REGISTERS+n*REGISTER_STRIDE,register);pointer(bytes,REGISTERS+n*REGISTER_STRIDE+4,value); + let mut emit=|register,value| { + word(bytes,REGISTERS+count*REGISTER_STRIDE,register); + pointer(bytes,REGISTERS+count*REGISTER_STRIDE+4,value); + count+=1; + }; + for (register,mut value) in values { + if extended { + if matches!(register,0x160b8|0x120a1|0x1c838|0x1ca28) {continue;} + if register==0x10791 {emit(0xa301,0);} + if register==0x16068 {value=(value&!0x3ffff)|0x3717f;} + if matches!(register,0x15109|0x15101) {value&=!3;} + } else if g15g && G15G_OMIT.contains(®ister) {continue;} + emit(register,value); + } + if let Some(native)=&dispatch { + for register in [0x160e0,0x1499,0xa341] {emit(register,fragment::UUID.into());} + emit(0x1c838,0); + emit(0x1ca28,user); + emit(0x10069,native.usc_window); } bytes[0x788..0x78a].copy_from_slice(&(count as u16).to_le_bytes()); bytes[0x78a..0x78c].copy_from_slice(&((count*REGISTER_STRIDE) as u16).to_le_bytes()); // Start3DJobParameters3: mirror records use their actual packed widths. for (offset,value) in [(0,s.depth_bias),(0x10,s.scissor),(0x20,s.query), - (0x168,s.partial_background.resources),(0x170,s.partial_background.address), + (0x168,if extended {s.background.resources}else{s.partial_background.resources}), + (0x170,if extended {s.background.address}else{s.partial_background.address}), (0x198,s.partial_background.resources),(0x1a0,s.partial_background.address), (0x1a8,s.zls_control),(0x1b8,s.depth.base),(0x1c0,s.depth.stride), (0x1c8,s.depth_compression.stride),(0x1e0,s.depth_compression.base), (0x1d0,s.depth.base),(0x1d8,s.depth.base), (0x1e8,s.stencil.base),(0x1f0,s.stencil.stride),(0x200,s.stencil.base),(0x208,s.stencil.base), (0x1f8,s.stencil_compression.stride),(0x210,s.stencil_compression.base), - (0x26c,s.partial_eot.address),(0x28c,s.partial_eot.address),(0x2c0,u64::from(s.depth_dimensions))] { - pointer(bytes,PARAMETERS+offset,value); + (0x26c,if extended {s.eot.address}else{s.partial_eot.address}),(0x28c,s.partial_eot.address),(0x2c0,u64::from(s.depth_dimensions))] { + pointer(bytes,PARAMETERS+offset+if extended && offset<=0x20 {0x10}else{0},value); } for (offset,value) in [(0x1b0,0x04040404),(0x228,s.mirror_tilebuffer_control), (0x230,s.isp_control),(0x238,width),(0x23c,height),(0x240,0x100000), - (0x248,s.mirror_tile_mode),(0x264,s.partial_eot.resources as u32),(0x284,s.partial_eot.resources as u32), + (0x248,s.mirror_tile_mode),(0x264,if extended {s.eot.resources as u32}else{s.partial_eot.resources as u32}),(0x284,s.partial_eot.resources as u32), (0x298,s.depth_clear),(0x29c,s.stencil_clear),(0x2a0,s.sample_size)] {word(bytes,PARAMETERS+offset,value);} + if let Some(native)=dispatch { + pointer(bytes,0xbc0,self.stamp.get()); + pointer(bytes,0xbc8,self.fw_stamp.get()); + word(bytes,0xbd0,self.stamp_value); + word(bytes,0xbd4,1); + word(bytes,EXTENDED_FLUSH,1); + word(bytes,0xbe0,fragment::UUID); + pointer(bytes,0xc10,self.timestamps[0].get()); + pointer(bytes,0xc18,self.timestamps[1].get()); + pointer(bytes,0xc20,self.user_timestamps[0].get()); + pointer(bytes,0xc28,self.user_timestamps[1].get()); + pointer(bytes,0xc46,self.pool.get()); + bytes[0xc4e]=1; + pointer(bytes,0xc57,native.firmware_aux.get()); + bytes[0xc5f]=native.incarnation; + bytes[0xc84]=1; + } else { word(bytes,0xb88,0x67); // EncoderParams.encoder_id word(bytes,0xb90,u32::MAX); // EncoderParams.unk_mask word(bytes,0xb94,u32::from(s.process_empty_tiles)); @@ -167,6 +225,46 @@ impl Command { pointer(bytes,fragment::COMMAND_TAIL+offset,value); } bytes[fragment::COMMAND_TAIL+0x36]=1;bytes[fragment::COMMAND_TAIL+0x4f]=1; + } Ok(()) } } + +#[cfg(test)] +mod extended_tests { + use super::*; + #[test] + fn native_fragment_uses_distinct_program_mirrors_and_completion_tail() { + let fw=|n:u64|FirmwareVa::new(0xffff_fc25_3000_0000+n*0x4000).unwrap(); + let gpu=|n:u64|GpuVa::new(0x10_0300_0000+n*0x4000).unwrap(); + let program=|n:u64|Program{address:0x1_0000_0000+n*0x1000,resources:n}; + let surface=DepthStencil{base:0,stride:0}; + let command=Command {context:3,register_set:RegisterSet::G15G, + sequence:Region::new(fw(0).get(),0x1000).unwrap(),notifier:fw(1),manager:fw(2),scene:fw(3),empty:fw(4), + gpu_alias:GpuVa::new(0x72_1000_4000).unwrap(),tilemap:gpu(0),heap:gpu(1),auxiliary:gpu(2),scene_user:gpu(3), + scratch:gpu(4),pool:fw(5),stamp:fw(6),fw_stamp:fw(7),stamp_value:0x3d000100, + timestamps:[fw(8),fw(9)],user_timestamps:[fw(10),fw(11)], + state:State {geometry:Geometry::new(64,64,super::super::agx_render::Utile::Pixels32, + super::super::agx_render::Utile::Pixels32,1).unwrap(),multisample:0,merge_upper:[0;2], + eot:program(1),background:program(2),partial_eot:program(3),partial_background:program(4), + scissor:0,depth_bias:0,query:0,isp_control:0xc000,tilebuffer_control:1,mirror_tilebuffer_control:1, + tile_mode:0x280,mirror_tile_mode:0x280,depth:surface,stencil:surface,depth_compression:surface, + stencil_compression:surface,zls_control:0,depth_dimensions:0,depth_clear:0,stencil_clear:0, + sample_size:4,process_empty_tiles:false}}; + let dispatch=||super::super::m3_tiler_command::Dispatch26 {incarnation:9,serial:1,peer_stamp:0, + usc_window:1<<32,firmware_aux:fw(12),scene_scratch:gpu(4)}; + let mut bytes=[0xa5;EXTENDED_SIZE+8]; + command.encode_extended(dispatch(),&mut bytes).unwrap(); + let word=|at|u32::from_le_bytes(bytes[at..at+4].try_into().unwrap()); + let pointer=|at|u64::from_le_bytes(bytes[at..at+8].try_into().unwrap()); + assert_eq!(word(0xbd0),command.stamp_value); + assert_eq!(word(EXTENDED_FLUSH),1); + assert_eq!(pointer(0x900),command.state.background.address); + assert_eq!(pointer(0x930),command.state.partial_background.address); + assert_eq!(bytes[0xc5f],9); + assert_eq!(&bytes[EXTENDED_SIZE..],&[0xa5;8]); + let mut short=[0xa5;EXTENDED_SIZE-1]; + assert!(command.encode_extended(dispatch(),&mut short).is_err()); + assert!(short.iter().all(|b|*b==0xa5)); + } +} diff --git a/drivers/gpu/drm/asahi/m3/m3_parameter_layout.rs b/drivers/gpu/drm/asahi/m3/m3_parameter_layout.rs index 684505615adf3..97c1367b29455 100644 --- a/drivers/gpu/drm/asahi/m3/m3_parameter_layout.rs +++ b/drivers/gpu/drm/asahi/m3/m3_parameter_layout.rs @@ -61,3 +61,77 @@ impl Manager { Ok(()) } } + +impl Manager { + pub(crate) const EXTENDED_SIZE: usize = 0xc0; + + /// J613's extended manager has both firmware and GPU page-list aliases. + /// The block ring is read through its canonical firmware mapping. + pub(crate) fn encode_extended(&self, pages_fw: FirmwareVa, blocks_fw: FirmwareVa, + hardware_id: u32, output: &mut [u8]) -> Result<(), Error> { + let capacity = self.block_count.checked_add(1).and_then(u32::checked_next_power_of_two) + .ok_or(Error::Size)?; + if hardware_id >= 32 || capacity as usize > self.blocks.size() / 8 { + return Err(Error::Size); + } + let bytes = output.get_mut(..Self::EXTENDED_SIZE).ok_or(Error::Size)?; + bytes.fill(0); + let word = |bytes: &mut [u8], offset: usize, value: u32| { + bytes[offset..offset + 4].copy_from_slice(&value.to_le_bytes()); + }; + let address = |bytes: &mut [u8], offset: usize, value: u64| { + bytes[offset..offset + 8].copy_from_slice(&value.to_le_bytes()); + }; + word(bytes, 8, hardware_id); + word(bytes, 12, hardware_id); + address(bytes, 0x1c, pages_fw.get()); + address(bytes, 0x24, self.pages.base()); + word(bytes, 0x2c, self.pages.size() as u32); + word(bytes, 0x30, self.page_count); + word(bytes, 0x34, capacity); + word(bytes, 0x38, self.block_count); + address(bytes, 0x40, blocks_fw.get()); + address(bytes, 0x48, self.control.get()); + word(bytes, 0x50, self.page_count - 1); + word(bytes, 0x54, BLOCK_SIZE as u32); + address(bytes, 0x60, self.counter.get()); + word(bytes, 0x78, self.page_count); + word(bytes, 0x7c, self.page_count); + Ok(()) + } +} + +#[cfg(test)] +mod extended_tests { + use super::*; + fn fixture() -> Manager { + Manager::new(GpuRegion::new(0x5000_0000, 0x4000).unwrap(), + GpuRegion::new(0x6000_0000, 0x4000).unwrap(), + FirmwareVa::new(0xffff_fc24_0010_0000).unwrap(), + FirmwareVa::new(0xffff_fc24_0020_0000).unwrap(), 32).unwrap() + } + #[test] fn native_manager_keeps_distinct_aliases_and_hardware_identity() { + let mut bytes = [0xa5; Manager::EXTENDED_SIZE + 8]; + let pages = FirmwareVa::new(0xffff_fc24_0030_0000).unwrap(); + let blocks = FirmwareVa::new(0xffff_fc24_0040_0000).unwrap(); + fixture().encode_extended(pages, blocks, 7, &mut bytes).unwrap(); + let word = |at| u32::from_le_bytes(bytes[at..at + 4].try_into().unwrap()); + let pointer = |at| u64::from_le_bytes(bytes[at..at + 8].try_into().unwrap()); + assert_eq!(word(8), 7); assert_eq!(word(12), 7); + assert_eq!(pointer(0x1c), pages.get()); + assert_eq!(pointer(0x24), 0x5000_0000); + assert_eq!(pointer(0x40), blocks.get()); + assert_eq!(word(0x34), 64); assert_eq!(word(0x38), 32); + assert_eq!(word(0x50), 127); + assert_eq!(pointer(0x80), 0); + assert_eq!(&bytes[Manager::EXTENDED_SIZE..], &[0xa5; 8]); + } + #[test] fn native_manager_rejects_short_storage_and_invalid_identity_before_mutation() { + let fw = FirmwareVa::new(0xffff_fc24_0010_0000).unwrap(); + let mut bytes = [0xa5; Manager::EXTENDED_SIZE]; + assert!(fixture().encode_extended(fw, fw, 32, &mut bytes).is_err()); + assert_eq!(bytes, [0xa5; Manager::EXTENDED_SIZE]); + assert!(fixture().encode_extended(fw, fw, 0, &mut bytes[..MANAGER_SIZE]).is_err()); + assert_eq!(bytes, [0xa5; Manager::EXTENDED_SIZE]); + } +} diff --git a/drivers/gpu/drm/asahi/m3/m3_pass.rs b/drivers/gpu/drm/asahi/m3/m3_pass.rs index 94c2172e33279..9c0eb070039cd 100644 --- a/drivers/gpu/drm/asahi/m3/m3_pass.rs +++ b/drivers/gpu/drm/asahi/m3/m3_pass.rs @@ -8,10 +8,10 @@ use layout::{Field,Space}; pub(crate) struct Pass {buffers:KVec} impl Pass { pub(crate) fn new(dev:&driver::AsahiDrmDevice,uat:&mmu::Uat,vm:&mmu::Vm, - slot:usize,clusters:u32,aliases:&mut KVec)->Result { + slot:usize,clusters:u32,profile:super::m3_profile::Profile,aliases:&mut KVec)->Result { let mut buffers=KVec::with_capacity(layout::COUNT,GFP_KERNEL)?; for field in layout::FIELDS { - let a=layout::board_allocation(slot,field,clusters).map_err(|_|EINVAL)?; + let a=layout::allocation_for(profile,slot,field,clusters).map_err(|_|EINVAL)?; let mut b=match a.space { Space::ClientGpu=>Buffer::at_prot(dev,uat.kernel_vm(),Some((vm,a.address)),None, a.size,prot::PROT_GPU_FW_SHARED_RW,prot::Prot::from_pte(a.access.pte()))?, diff --git a/drivers/gpu/drm/asahi/m3/m3_pass_layout.rs b/drivers/gpu/drm/asahi/m3/m3_pass_layout.rs index 5ace477a44fff..f88a2a2c9b4a5 100644 --- a/drivers/gpu/drm/asahi/m3/m3_pass_layout.rs +++ b/drivers/gpu/drm/asahi/m3/m3_pass_layout.rs @@ -219,3 +219,39 @@ mod tests { } } } + +pub(crate) fn allocation_for(profile: super::m3_profile::Profile, slot:usize, field:Field, clusters:u32) + -> Result { + let mut owner = board_allocation(slot,field,clusters)?; + if profile == super::m3_profile::Profile::Legacy { return Ok(owner); } + if owner.space == Space::Firmware { + owner.address = 0xffff_fc25_0000_0000 + (slot * COUNT + field as usize) as u64 * 0x10_0000; + } + match field { + Field::Scene => owner.size = 0x400, + Field::TilerCommand | Field::FragmentCommand => { + let fragment = field == Field::FragmentCommand; + owner.size = if fragment {0xca0} else {0x940}; + owner.gpu_alias = Some(0x72_1000_0000 + slot as u64 * 0x4_0000 + + u64::from(fragment) * 0x4000); + } + Field::TilerToFragment | Field::TilerDependency | Field::FragmentDependency => owner.size = 0x40, + _ => {} + } + Ok(owner) +} + +#[cfg(test)] +mod extended_tests { + use super::*; + #[test] fn extended_command_aliases_keep_whole_disjoint_pages() { + for slot in 0..SLOTS { + for field in [Field::TilerCommand,Field::FragmentCommand] { + let owner = allocation_for(super::super::m3_profile::Profile::Extended,slot,field,1).unwrap(); + assert_eq!(owner.address & 0x3fff,0); + assert_eq!(owner.gpu_alias.unwrap() & 0x3fff,0); + assert!(owner.size <= 0x4000); + } + } + } +} diff --git a/drivers/gpu/drm/asahi/m3/m3_profile.rs b/drivers/gpu/drm/asahi/m3/m3_profile.rs index 004ac2c7d1eef..04b872ed9ea80 100644 --- a/drivers/gpu/drm/asahi/m3/m3_profile.rs +++ b/drivers/gpu/drm/asahi/m3/m3_profile.rs @@ -32,6 +32,15 @@ impl Profile { } } + /// Keep the diagnostic AP/GPU timeline in its established 24 MHz units. + /// User timestamps and duration calculations retain native firmware units. + pub(crate) const fn profiler_ticks(self, timestamp: u64) -> u64 { + match self { + Self::Legacy => timestamp, + Self::Extended => (timestamp / 1000) * 24 + (timestamp % 1000) * 24 / 1000, + } + } + pub(crate) fn gpu_duration_ns(self, elapsed: u64) -> Option { match self { Self::Legacy => elapsed.checked_mul(1000).map(|value| value / 24), @@ -49,6 +58,18 @@ impl Profile { mod tests { use super::*; + #[test] + fn profiling_uses_one_clock_domain_without_changing_native_durations() { + for value in [0, 1, 999, 1000, 24_000_000, u64::MAX] { + assert_eq!(Profile::Legacy.profiler_ticks(value), value); + assert_eq!( + Profile::Extended.profiler_ticks(value), + ((value as u128) * 24 / 1000) as u64 + ); + } + assert_eq!(Profile::Extended.profiler_ticks(1_000_000_000), 24_000_000); + } + #[test] fn profile_acknowledgement_is_exact_and_file_query_covers_hal() { for profile in [Profile::Legacy, Profile::Extended] { diff --git a/drivers/gpu/drm/asahi/m3/m3_render.rs b/drivers/gpu/drm/asahi/m3/m3_render.rs index e7a02982e96f1..9a9f63d15b784 100644 --- a/drivers/gpu/drm/asahi/m3/m3_render.rs +++ b/drivers/gpu/drm/asahi/m3/m3_render.rs @@ -26,7 +26,7 @@ pub(crate) struct Render { passes:KVec, init_bm:Buffer, // All active mappings above must drop before their ASID binding. _binding:mmu::VmBind, - stats: Region, sequence_bytes: KVec, tiler_bytes: KVec, fragment_bytes: KVec, + stats: [Region;2], profile:super::m3_profile::Profile, sequence_bytes: KVec, tiler_bytes: KVec, fragment_bytes: KVec, last_progress_ns:i64, draw:u64, heads:[u16;2], cached_views:KVec, slot:usize, batch_count:usize, early_count:u32, @@ -67,37 +67,45 @@ const RENDER_QUEUES: [QueueObjects; 2] = [ scratch: s::FRAGMENT_SCRATCH, tiler: false, uuid: 0x3d0000 }, ]; impl Render { - pub(crate) fn new(dev:&driver::AsahiDrmDevice,uat:&mmu::Uat,vm:&mmu::Vm,stats:Region,clusters:u32, - registers:crate::m3::m3_compute_layout::RegisterSet)->Result { + pub(crate) fn new(dev:&driver::AsahiDrmDevice,uat:&mmu::Uat,vm:&mmu::Vm,stats:[Region;2],clusters:u32, + registers:crate::m3::m3_compute_layout::RegisterSet,profile:super::m3_profile::Profile)->Result { let lower_tlbi=crate::m3::m3_memory::LowerTlbi; let binding=uat.bind(vm)?; let mut objects=KVec::new();let mut aliases=KVec::new(); for index in 0..s::COUNT { - let a=s::allocation(index).map_err(|_|EINVAL)?; + let a=s::allocation_for(profile,index).map_err(|_|EINVAL)?; let protection=prot::Prot::from_pte(a.access.pte()); let b=match a.space { Space::ClientGpu=>Buffer::at_prot(dev,uat.kernel_vm(),Some((vm,a.address)),None, a.size,prot::PROT_GPU_FW_SHARED_RW,protection)?, - Space::Firmware=>Buffer::at_prot(dev,uat.kernel_vm(),None,Some(a.address), + Space::Firmware=>Buffer::at_prot(dev,uat.kernel_vm(), + a.gpu_alias.map(|address|(uat.kernel_lower_vm(),address)),Some(a.address), a.size,protection,prot::PROT_GPU_SHARED_RW)?, }; + let mut b=b; + if a.gpu_alias.is_some() { aliases.push(b.map_gpu_view(vm)?,GFP_KERNEL)?; } objects.push(b,GFP_KERNEL)?; } // InitBM is shared, not part of a pass. Preserve its qualified mapping. - let init_bm=Buffer::at_prot(dev,uat.kernel_vm(),None,Some(0xfffffc200062bfe0), + let init_bm=Buffer::at_prot(dev,uat.kernel_vm(),None,Some(if profile==super::m3_profile::Profile::Extended {0xffff_fc24_4000_0000} + else {0xfffffc200062bfe0}), sync::INIT_BM_SIZE,prot::PROT_FW_SHARED_RW,prot::PROT_GPU_SHARED_RW)?; let mut passes=KVec::with_capacity(pass_layout::SLOTS,GFP_KERNEL)?; - for slot in 0..pass_layout::SLOTS {passes.push(Pass::new(dev,uat,vm,slot,clusters,&mut aliases)?,GFP_KERNEL)?;} + for slot in 0..pass_layout::SLOTS {passes.push(Pass::new(dev,uat,vm,slot,clusters,profile,&mut aliases)?,GFP_KERNEL)?;} let mut sequence_bytes=KVec::new(); sequence_bytes.resize(sequence::STORAGE,0u8,GFP_KERNEL)?; let mut tiler_bytes=KVec::new(); - tiler_bytes.resize(tiler::SIZE,0u8,GFP_KERNEL)?; + tiler_bytes.resize(if profile==super::m3_profile::Profile::Extended {tiler::EXTENDED_SIZE} else {tiler::SIZE},0u8,GFP_KERNEL)?; let mut fragment_bytes=KVec::new(); - fragment_bytes.resize(fragment::SIZE,0u8,GFP_KERNEL)?; - let mut job=Self{passes,init_bm,stats,sequence_bytes,tiler_bytes,fragment_bytes,objects,_aliases:aliases,_binding:binding,last_progress_ns:0,draw:0,heads:[0;2],cached_views:KVec::new(),slot:0,batch_count:0,early_count:0,stage_timestamps:[true;2],base:0,checkpoint:(0,[0;2],0,0,0,0),clusters,registers,_lower_tlbi:lower_tlbi}; + fragment_bytes.resize(if profile==super::m3_profile::Profile::Extended {fragment::EXTENDED_SIZE} else {fragment::SIZE},0u8,GFP_KERNEL)?; + let mut job=Self{passes,init_bm,stats,profile,sequence_bytes,tiler_bytes,fragment_bytes,objects,_aliases:aliases,_binding:binding,last_progress_ns:0,draw:0,heads:[0;2],cached_views:KVec::new(),slot:0,batch_count:0,early_count:0,stage_timestamps:[true;2],base:0,checkpoint:(0,[0;2],0,0,0,0),clusters,registers,_lower_tlbi:lower_tlbi}; // Lists and the matching manager are built before InitBM publication. job.initialize_parameter_buffer()?; - for pool in &UMA_POOLS { pool.initialize(&mut job.objects)?; } + for (index,pool) in UMA_POOLS.iter().enumerate() { + if profile==super::m3_profile::Profile::Extended { + pool.initialize_extended(&mut job.objects,33+index as u8,false)?; + } else {pool.initialize(&mut job.objects)?;} + } job.initialize_state()?; job.initialize_queues()?; dev_info!(dev.as_ref(),"M3 native render storage initialized, PB={} MiB\n",PB_BLOCKS as usize*PB_BLOCK_SIZE/(1024*1024)); @@ -113,8 +121,14 @@ impl Render { let control = queue::FirmwareVa::new(self.objects[BLOCK_CONTROL].va()).map_err(|_| EINVAL)?; let counter = queue::FirmwareVa::new(self.objects[s::BM_COUNTER].va()).map_err(|_| EINVAL)?; let manager = parameter::Manager::new(pages, blocks, control, counter, PB_BLOCKS).map_err(|_| EINVAL)?; - let mut bytes = [0; parameter::MANAGER_SIZE]; - manager.encode(&mut bytes).map_err(|_| EINVAL)?; + let extended=self.profile==super::m3_profile::Profile::Extended; + let mut bytes = [0; parameter::Manager::EXTENDED_SIZE]; + let length=if extended { + manager.encode_extended(queue::FirmwareVa::new(self.objects[PARAMETER_PAGE_LIST].va()).map_err(|_|EINVAL)?, + queue::FirmwareVa::new(self.objects[PARAMETER_BLOCK_LIST].va()).map_err(|_|EINVAL)?,0,&mut bytes) + .map_err(|_|EINVAL)?; + parameter::Manager::EXTENDED_SIZE + } else {manager.encode(&mut bytes).map_err(|_|EINVAL)?;parameter::MANAGER_SIZE}; for group in 0..PB_GROUPS { let buffer = &self.objects[PB_FIRST + group]; let backing = parameter::Backing::new(GpuRegion::new(buffer.gpu_va()?, buffer.size()).map_err(|_| EINVAL)?) @@ -128,18 +142,30 @@ impl Render { backing.page(page).map_err(|_| EINVAL)?)?; } } - self.objects[BUFFER_MANAGER].write(0, &bytes) + self.objects[BUFFER_MANAGER].write(0, &bytes[..length]) } fn initialize_state(&mut self) -> Result { let address = |owner: usize| queue::FirmwareVa::new(self.objects[owner].va()).map_err(|_| EINVAL); let notification = state::Notifier { threshold: address(s::EVENT_COUNT)?, context: self._binding.slot() }; let list_owner = address(JOB_LIST)?; let mut context = [0; state::CONTEXT_SIZE]; - state::context(&mut context).map_err(|_| EINVAL)?; - self.objects[GPU_CONTEXT].write(0, &context)?; - let mut notifier = [0; state::NOTIFIER_SIZE]; - notification.encode(&mut notifier).map_err(|_| EINVAL)?; - self.objects[NOTIFIER].write(0, ¬ifier)?; + if self.profile==super::m3_profile::Profile::Extended { + state::context_extended(&mut context).map_err(|_|EINVAL)?; + self.objects[GPU_CONTEXT].write(0,&context[..state::EXTENDED_CONTEXT_SIZE])?; + for engine in 0..2 { + let value=state::Notifier {threshold:queue::FirmwareVa::new(self.objects[s::EVENT_COUNT].va()+engine as u64*8).map_err(|_|EINVAL)?, + context:self._binding.slot()}; + let mut notifier=[0;state::EXTENDED_NOTIFIER_SIZE]; + value.encode_extended(&mut notifier).map_err(|_|EINVAL)?; + self.objects[NOTIFIER].write(engine*state::EXTENDED_NOTIFIER_SIZE,¬ifier)?; + } + } else { + state::context(&mut context).map_err(|_| EINVAL)?; + self.objects[GPU_CONTEXT].write(0, &context)?; + let mut notifier = [0; state::NOTIFIER_SIZE]; + notification.encode(&mut notifier).map_err(|_| EINVAL)?; + self.objects[NOTIFIER].write(0, ¬ifier)?; + } let mut list = [0; state::JOB_LIST_SIZE]; state::job_list(&mut list, list_owner).map_err(|_| EINVAL)?; self.objects[JOB_LIST].write(0, &list)?; @@ -157,6 +183,10 @@ impl Render { state::counter(bytes, value).map_err(|_| EINVAL)?; self.objects[owner].write(0, bytes)?; } + if self.profile==super::m3_profile::Profile::Extended { + self.objects[s::EVENT_COUNT].fill(0)?; + self.objects[s::BM_COUNTER].fill(0)?; + } Ok(()) } fn initialize_queues(&mut self) -> Result { @@ -172,8 +202,13 @@ impl Render { }else{[self.passes[0].get(P::TilerToFragment).va(),self.passes[0].get(P::FragmentCommand).va()]}; let seeds=[queue::FirmwareVa::new(seed_addresses[0]).map_err(|_|EINVAL)?, queue::FirmwareVa::new(seed_addresses[1]).map_err(|_|EINVAL)?]; - let mut bytes = [0; queue::INFO_SIZE]; - info.encode(&mut bytes).map_err(|_| EINVAL)?; + let mut bytes=KVec::new(); + bytes.resize(if self.profile==super::m3_profile::Profile::Extended { + queue::QueueInfo::EXTENDED_SIZE + } else {queue::INFO_SIZE},0u8,GFP_KERNEL)?; + if self.profile==super::m3_profile::Profile::Extended { + info.encode_extended(address(owner.info)?,&mut bytes).map_err(|_|EINVAL)?; + } else {info.encode(&mut bytes).map_err(|_|EINVAL)?;} self.objects[owner.info].write(0, &bytes)?; let mut state = [0; queue::STATE_SIZE]; queue::RingState { capacity: queue::QUALIFIED_CAPACITY, done: 0, read: 0, write: 0 } @@ -187,9 +222,12 @@ impl Render { Ok(()) } fn write_barrier(&mut self, owner: P, command: sync::Barrier) -> Result { - let mut bytes = [0; sync::BARRIER_SIZE]; - command.encode(&mut bytes).map_err(|_| EINVAL)?; - self.passes[self.slot].get_mut(owner).write(0, &bytes) + let mut bytes=[0;sync::Barrier::EXTENDED_SIZE]; + let length=if self.profile==super::m3_profile::Profile::Extended { + command.encode_extended(&mut bytes).map_err(|_|EINVAL)?; + sync::Barrier::EXTENDED_SIZE + } else {command.encode(&mut bytes).map_err(|_|EINVAL)?;sync::BARRIER_SIZE}; + self.passes[self.slot].get_mut(owner).write(0,&bytes[..length]) } // Construct synchronization from the actual owners after resetting a pass, // before publishing its ring entries. InitBM is shared and dispatched only @@ -201,9 +239,11 @@ impl Render { let ta_stamp = address(s::TA_FW_STAMP)?; let fs_stamp = address(s::FRAGMENT_FW_STAMP)?; let mut init = [0; sync::INIT_BM_SIZE]; - sync::InitBufferManager { context: self._binding.slot(), slot: 0, - block_count: PB_BLOCKS, manager, stamp: timeline::stamp(s::STAMP_TA - 256, 1) } - .encode(&mut init).map_err(|_| EINVAL)?; + let value=sync::InitBufferManager { context: self._binding.slot(), slot: 0, + block_count: PB_BLOCKS, manager, stamp: timeline::stamp(s::STAMP_TA - 256, 1) }; + if self.profile==super::m3_profile::Profile::Extended { + value.encode_extended(&mut init).map_err(|_|EINVAL)?; + } else {value.encode(&mut init).map_err(|_|EINVAL)?;} self.init_bm.write(0, &init)?; let ta_value = timeline::stamp(s::STAMP_TA - 256, draw); let fs_value = timeline::stamp(s::STAMP_FRAGMENT - 256, draw); @@ -225,7 +265,7 @@ impl Render { } // Prepare each private command after its retired slot is reset. The // reusable host staging allocation is copied before the next slot is built. - fn prepare_tiler(&mut self,r:agx_uapi::UapiRenderCommand,draw:u64)->Result { + fn prepare_tiler(&mut self,r:agx_uapi::UapiRenderCommand,usc:u64,draw:u64)->Result { use crate::m3::m3_compute_layout::GpuVa; let pass=&self.passes[self.slot]; let shared=|index:usize| queue::FirmwareVa::new(self.objects[index].va()).map_err(|_|EINVAL); @@ -243,7 +283,7 @@ impl Render { let value=tiler::Command { context:self._binding.slot(),register_set:self.registers, notifier:shared(NOTIFIER)?,manager:shared(BUFFER_MANAGER)?, - scene:fw(P::Scene)?,empty:shared(547)?,gpu_alias:GpuVa::new(command.gpu_va()?).map_err(|_|EINVAL)?, + scene:fw(P::Scene)?,empty:if self.profile==super::m3_profile::Profile::Extended {queue::FirmwareVa::new(pass.get(P::Scene).va()+0x100).map_err(|_|EINVAL)?} else {shared(547)?},gpu_alias:GpuVa::new(command.gpu_va()?).map_err(|_|EINVAL)?, sequence:Region::new(seq.va(),seq.size()).map_err(|_|EINVAL)?,pool:shared(UMA_POOLS[0].manager)?, scratch:fw(P::TilerScratch)?,stamp:shared(s::TA_STAMP)?,fw_stamp:shared(s::TA_FW_STAMP)?, stamp_value:timeline::stamp(s::STAMP_TA - 256, draw),timestamps:[fw(P::TilerStart)?,fw(P::TilerEnd)?], @@ -253,7 +293,12 @@ impl Render { vdm:crate::m3::agx_render_state::compact(r.vdm_base).map_err(|_|EINVAL)?, multisample:r.multisample_control,ppp:r.ppp_control,process_empty_tiles:r.flags&2!=0, }; - value.encode(&mut self.tiler_bytes).map_err(|_|EINVAL)?; + if self.profile==super::m3_profile::Profile::Extended { + value.encode_extended(tiler::Dispatch26 {incarnation:self._binding.generation(),serial:draw.checked_mul(2).ok_or(EOVERFLOW)?, + peer_stamp:timeline::stamp(s::STAMP_FRAGMENT-256,draw),usc_window:usc, + firmware_aux:queue::FirmwareVa::new(self.objects[UMA_POOLS[0].manager].va()+0x100).map_err(|_|EINVAL)?, + scene_scratch:gpu(P::FragmentScratch)?},&mut self.tiler_bytes).map_err(|_|EINVAL)?; + } else {value.encode(&mut self.tiler_bytes).map_err(|_|EINVAL)?;} self.passes[self.slot].get_mut(P::TilerCommand).write(0,&self.tiler_bytes) } fn prepare_sequences(&mut self,draw:u64)->Result { @@ -266,14 +311,21 @@ impl Render { let notifier=&self.objects[NOTIFIER]; let owners=sequence::Owners { command:region(if fs{P::FragmentCommand}else{P::TilerCommand})?,sequence:region(target)?, - stats:self.stats,notifier:Region::new(notifier.va(),notifier.size()).map_err(|_|EINVAL)?, + stats:self.stats[usize::from(fs)],notifier:Region::new(notifier.va()+ + if fs && self.profile==super::m3_profile::Profile::Extended {0x200} else {0}, + if self.profile==super::m3_profile::Profile::Extended {0x200} else {notifier.size()}).map_err(|_|EINVAL)?, queue:shared(if fs{s::FRAGMENT_QUEUE}else{s::TA_QUEUE})?,scene:fw(P::Scene)?, manager:shared(BUFFER_MANAGER)?,pool:shared(UMA_POOLS[usize::from(fs)].manager)?, fw_stamp:shared(if fs{s::FRAGMENT_FW_STAMP}else{s::TA_FW_STAMP})?, context:self._binding.slot(),stamp:timeline::stamp((if fs{s::STAMP_FRAGMENT}else{s::STAMP_TA}) - 256, draw), }; let timed=self.stage_timestamps[usize::from(fs)]; - let length=if fs { + let length=if self.profile==super::m3_profile::Profile::Extended { + let pool=&self.objects[UMA_POOLS[usize::from(fs)].manager]; + let auxiliary=super::m3_compute_layout::GpuVa::new(pool.gpu_va()?+if fs{0x200}else{0x100}).map_err(|_|EINVAL)?; + let serial=draw.checked_mul(2).and_then(|v|v.checked_sub(u64::from(fs))).ok_or(EOVERFLOW)?; + owners.encode_extended(fs,auxiliary,serial,&mut self.sequence_bytes).map_err(|_|EINVAL)? + } else if fs { let b=pass.get(P::FragmentScratch); let scratch=GpuRegion::new(b.gpu_va()?,b.size()).map_err(|_|EINVAL)?; sequence::Fragment{owners,scratch,queue_command_count:timeline::previous(draw)}.encode_timestamps(&mut self.sequence_bytes,timed).map_err(|_|EINVAL)? @@ -290,6 +342,9 @@ impl Render { fn patch_context(&mut self)->Result { let context=self._binding.slot(); self.objects[NOTIFIER].u32(state::NOTIFIER_CONTEXT, context)?; + if self.profile==super::m3_profile::Profile::Extended { + self.objects[NOTIFIER].u32(0x200+state::NOTIFIER_CONTEXT,context)?; + } Ok(()) } /// Pass geometry; TPC storage is per tiler accelerator, one per GPU cluster. @@ -320,7 +375,8 @@ impl Render { let value=fragment::Command { context:self._binding.slot(),register_set:self.registers, sequence:Region::new(seq.va(),seq.size()).map_err(|_|EINVAL)?, - notifier:shared(NOTIFIER)?,manager:shared(BUFFER_MANAGER)?,scene:fw(P::Scene)?,empty:shared(547)?, + notifier:queue::FirmwareVa::new(self.objects[NOTIFIER].va()+ + if self.profile==super::m3_profile::Profile::Extended {0x200} else {0}).map_err(|_|EINVAL)?,manager:shared(BUFFER_MANAGER)?,scene:fw(P::Scene)?,empty:if self.profile==super::m3_profile::Profile::Extended {queue::FirmwareVa::new(pass.get(P::Scene).va()+0x100).map_err(|_|EINVAL)?} else {shared(547)?}, gpu_alias:GpuVa::new(command.gpu_va()?).map_err(|_|EINVAL)?,tilemap:gpu(P::Tilemap)?, heap:gpu(P::HeapMetadata)?,auxiliary:gpu(P::Auxiliary)?,scene_user:gpu(P::SceneUser)?, scratch:gpu(P::FragmentScratch)?,pool:shared(UMA_POOLS[1].manager)?,stamp:shared(s::FRAGMENT_STAMP)?, @@ -341,15 +397,34 @@ impl Render { stencil_clear:r.stencil_clear,sample_size:u32::from(r.sample_size),process_empty_tiles:r.flags&2!=0, }, }; - value.encode(&mut self.fragment_bytes).map_err(|_|EINVAL)?; + if self.profile==super::m3_profile::Profile::Extended { + value.encode_extended(tiler::Dispatch26 {incarnation:self._binding.generation(),serial:draw.checked_mul(2).and_then(|v|v.checked_sub(1)).ok_or(EOVERFLOW)?, + peer_stamp:0,usc_window:usc, + firmware_aux:queue::FirmwareVa::new(self.objects[UMA_POOLS[1].manager].va()+0x200).map_err(|_|EINVAL)?, + scene_scratch:gpu(P::FragmentScratch)?},&mut self.fragment_bytes).map_err(|_|EINVAL)?; + } else {value.encode(&mut self.fragment_bytes).map_err(|_|EINVAL)?;} self.passes[self.slot].get_mut(P::FragmentCommand).write(0,&self.fragment_bytes) } + pub(crate) fn set_attachments(&mut self,attachments:&[crate::m3::agx_attachments::Attachments;2])->Result { + // Preserve the legacy renderer's existing packet format. The native + // interface consumes separate attachment apertures for both stages. + if self.profile!=super::m3_profile::Profile::Extended {return Ok(());} + for (fragment,list) in attachments.iter().enumerate() { + let field=if fragment==0 {P::TilerSequence}else{P::FragmentSequence}; + let buffer=self.passes[self.slot].get_mut(field); + sequence::patch_extended_attachments(fragment!=0,list,|offset,data|buffer.write(offset,data))?; + } + crate::m3::agx_memory::publish();Ok(()) + } pub(crate) fn set_user_timestamps(&mut self,addresses:[[u64;2];2])->Result { if addresses.iter().enumerate().any(|(stage,pair)|*pair!=[0;2] && !self.stage_timestamps[stage]) {return Err(EINVAL);} for (stage,(command,sequence,offset,start,end)) in [ (P::TilerCommand,P::TilerSequence,sequence::ta::USER_PAIR,sequence::ta::TIMESTAMP_START,sequence::ta::TIMESTAMP_END), (P::FragmentCommand,P::FragmentSequence,sequence::fragment::USER_PAIR,sequence::fragment::TIMESTAMP_START,sequence::fragment::TIMESTAMP_END) ].into_iter().enumerate() { + let (offset,start,end)=if self.profile==super::m3_profile::Profile::Extended { + if stage==0 {(0x8d8,0x1cc,0x21c)} else {(0xc20,0x1e4,0x234)} + } else {(offset,start,end)}; let pair=addresses[stage]; if !self.stage_timestamps[stage] {continue;} self.passes[self.slot].get_mut(command).u64(offset,pair[0])?; @@ -365,19 +440,19 @@ impl Render { pub(crate) fn gpu_ns(&mut self,base:usize,count:usize)->Result { let start=self.passes[base].get_mut(P::TilerStart).read_u64(0)?; let end=self.passes[Self::at(base,count-1)].get_mut(P::FragmentEnd).read_u64(0)?; - Ok(if start!=0 && end>=start {(end-start)*1000/24} else {0}) + self.profile.gpu_duration_ns(if start!=0 && end>=start {end-start} else {0}).ok_or(EOVERFLOW) } /// The batch's first tiler start and last fragment end, in 24 MHz GPU timestamp ticks. pub(crate) fn gpu_ticks(&mut self,base:usize,count:usize)->Result<(u64,u64)> { - Ok((self.passes[base].get_mut(P::TilerStart).read_u64(0)?, - self.passes[Self::at(base,count-1)].get_mut(P::FragmentEnd).read_u64(0)?)) + Ok((self.profile.profiler_ticks(self.passes[base].get_mut(P::TilerStart).read_u64(0)?), + self.profile.profiler_ticks(self.passes[Self::at(base,count-1)].get_mut(P::FragmentEnd).read_u64(0)?))) } /// Read retained timestamps only after exact batch retirement. The first /// TA and final fragment delimit the batch in the shared 24 MHz counter. pub(crate) fn batch_gpu_span(&mut self)->Result<[u64;2]> { if self.batch_count == 0 {return Err(EINVAL);} - Ok([self.passes[self.base].get_mut(P::TilerStart).read_u64(0)?, - self.passes[Self::at(self.base,self.batch_count-1)].get_mut(P::FragmentEnd).read_u64(0)?]) + let (start,end)=self.gpu_ticks(self.base,self.batch_count)?; + Ok([start,end]) } pub(crate) fn stage_ns(&mut self,base:usize,count:usize)->Result<[u64;3]> { let mut total=[0;3]; @@ -390,7 +465,7 @@ impl Render { pub(crate) fn slot_stage_ns(&mut self,base:usize,count:usize,index:usize)->Result<[u64;3]> { if index>=count {return Err(EINVAL);} let slot=Self::at(base,index); - let duration=|a:u64,b:u64| if a!=0 && b>=a {(b-a)*1000/24} else {0}; + let duration=|a:u64,b:u64| self.profile.gpu_duration_ns(if a!=0 && b>=a {b-a} else {0}).unwrap_or(0); let fs_start=self.passes[slot].get_mut(P::FragmentStart).read_u64(0)?; let fs_end=self.passes[slot].get_mut(P::FragmentEnd).read_u64(0)?; let ta_start=self.passes[slot].get_mut(P::TilerStart).read_u64(0)?; @@ -415,7 +490,9 @@ impl Render { } /// Whether any draw has been published (the first one initializes the buffer manager). pub(crate) fn started(&self)->bool {self.draw>0} - pub(crate) fn first(&self)->bool {self.draw==self.batch_count as u64} + pub(crate) fn first(&self)->bool { + self.profile==super::m3_profile::Profile::Legacy && self.draw==self.batch_count as u64 + } /// Start a batch in pass slots `base..`. With `overlap`, earlier batches may still run: then /// the VM must already be bound, no storage may grow and the slots must not be in flight /// (the caller keeps the in-flight passes plus this batch within SLOTS). @@ -447,13 +524,15 @@ impl Render { let binding=uat.bind(vm)?; let mut views=KVec::new();let mut aliases=KVec::new(); for i in 0..s::COUNT { - if s::allocation(i).map_err(|_|EINVAL)?.space==Space::ClientGpu { + let a=s::allocation_for(self.profile,i).map_err(|_|EINVAL)?; + if a.gpu_alias.is_some() {aliases.push(self.objects[i].map_gpu_view(vm)?,GFP_KERNEL)?;} + if a.space==Space::ClientGpu { views.push((ViewOwner::Shared(i),self.objects[i].map_gpu_view(vm)?),GFP_KERNEL)?; } } for (slot,pass) in self.passes.iter_mut().enumerate() { for field in pass_layout::FIELDS { - let a=pass_layout::board_allocation(slot,field,self.clusters).map_err(|_|EINVAL)?; + let a=pass_layout::allocation_for(self.profile,slot,field,self.clusters).map_err(|_|EINVAL)?; if a.space==Space::ClientGpu {views.push((ViewOwner::Pass{slot,field},pass.get_mut(field).map_gpu_view(vm)?),GFP_KERNEL)?;} if a.gpu_alias.is_some() {aliases.push(pass.get_mut(field).map_gpu_view(vm)?,GFP_KERNEL)?;} } @@ -498,12 +577,22 @@ impl Render { self.base=base; Ok(()) } + fn publish_counts(&mut self, draw:u64) -> Result { + if self.profile==super::m3_profile::Profile::Extended { + for offset in [0,8] {self.objects[s::EVENT_COUNT].u64(offset,draw)?;} + for counter in [TA_POOL_COUNTER,FRAGMENT_POOL_COUNTER] {self.objects[counter].u64(0,draw)?;} + } else { + self.objects[s::BM_COUNTER].u32(0,draw as u32)?; + self.objects[s::EVENT_COUNT].u32(0,timeline::events(draw,2))?; + } + Ok(()) + } + /// Restore producer state only when no batch is in flight. Running firmware can /// consume append's WRITE updates without a new doorbell, so overlap cannot abort. pub(crate) fn abort_batch(&mut self)->Result { (self.draw,self.heads,self.batch_count,self.slot,self.early_count,self.base)=self.checkpoint; - self.objects[s::BM_COUNTER].u32(0,self.draw as u32)?; - self.objects[s::EVENT_COUNT].u32(0,timeline::events(self.draw,2))?; + self.publish_counts(self.draw)?; for (stage,pointers) in [s::TA_POINTERS,s::FRAGMENT_POINTERS].into_iter().enumerate() { self.objects[pointers].u32(queue::WRITE,u32::from(self.heads[stage]))?; } @@ -547,8 +636,20 @@ impl Render { self.draw=self.draw.checked_add(1).ok_or(EOVERFLOW)?; self.batch_count+=1; let draw=self.draw; - self.objects[s::BM_COUNTER].u32(0,draw as u32)?; - self.objects[s::EVENT_COUNT].u32(0,timeline::events(draw,2))?; + self.publish_counts(draw)?; + if self.profile==super::m3_profile::Profile::Extended { + let pass=&self.passes[self.slot]; + let scratch=pass.get(P::FragmentScratch); + let scratch_gpu=GpuRegion::new(scratch.gpu_va()?,scratch.size()).map_err(|_|EINVAL)?; + let scratch_fw=queue::FirmwareVa::new(scratch.va()).map_err(|_|EINVAL)?; + let list=pass.get(P::SceneUser); + let list=GpuRegion::new(list.gpu_va()?,list.size()).map_err(|_|EINVAL)?; + let metadata=pass.get(P::Scene); + let owner=Region::new(metadata.va(),metadata.size()).map_err(|_|EINVAL)?; + let mut bytes=[0;scene::EXTENDED_SIZE]; + scene::encode_extended(&mut bytes,owner,scratch_gpu,scratch_fw,list).map_err(|_|EINVAL)?; + self.passes[self.slot].get_mut(P::Scene).write(0,&bytes)?; + } else { let entry_offset = (self.draw % u64::from(MANAGER_SCENE_COUNT)) as usize * 4; let scene_list = &mut self.objects[s::BM_SCENES]; if entry_offset + 4 > scene_list.size() { return Err(ERANGE); } @@ -560,7 +661,8 @@ impl Render { let mut bytes = [0; scene::SIZE]; scene::Scene { manager_entry, manager_misc, user_buffer }.encode(&mut bytes).map_err(|_| EINVAL)?; self.passes[self.slot].get_mut(P::Scene).write(0, &bytes)?; - self.prepare_tiler(r,draw)?; + } + self.prepare_tiler(r,usc,draw)?; self.patch_context()?; self.prepare_fragment(r,usc,draw)?; self.prepare_sequences(draw)?; @@ -655,7 +757,13 @@ impl Render { if self.objects[stamp].read_u32(0)?!=value || self.objects[fw].read_u32(0)?!=value || self.objects[pointers].read_u32(queue::DONE)?!=u32::from(head) || self.objects[pointers].read_u32(queue::READ)?!=u32::from(head) {return Ok(false);} - } Ok(true) + } + if self.profile==super::m3_profile::Profile::Extended && self.draw!=0 { + for pool in UMA_POOLS { + if self.objects[pool.manager].read_u64(0x54)?!=self.draw {return Ok(false);} + } + } + Ok(true) } pub(crate) fn log(&mut self,dev:&driver::AsahiDrmDevice)->Result { // Retain allocator state as well as payloads: an exhausted parameter diff --git a/drivers/gpu/drm/asahi/m3/m3_render_sequence.rs b/drivers/gpu/drm/asahi/m3/m3_render_sequence.rs index f9a0483e77b21..3d421e9906692 100644 --- a/drivers/gpu/drm/asahi/m3/m3_render_sequence.rs +++ b/drivers/gpu/drm/asahi/m3/m3_render_sequence.rs @@ -184,3 +184,171 @@ impl Fragment { Ok(if timed {LENGTH} else {without_timestamps(bytes,TIMESTAMP_START,FINALIZE,LENGTH,0xa4)}) } } + +impl Owners { + pub(crate) fn encode_extended(&self, fragment:bool, + auxiliary:super::m3_compute_layout::GpuVa, serial:u64, output:&mut[u8]) -> Result { + if self.context>=64 || serial==0 || self.stamp==0 {return Err(Error::Address);} + let command=self.command.at(0,if fragment {0xca0}else{0x940})?; + let sequence=self.sequence.at(0,STORAGE)?; + let activity=self.stats.at(0,8)?; + let scene_tail=self.scene.get().checked_add(0x54).ok_or(Error::Address)?; + let (clock,pair,aux_time,elapsed)=if fragment {(0xc08,0xc10,0xc70,0xb84)}else{(0x8c0,0x8c8,0x928,0x83c)}; + let first=if fragment {0x1e4}else{0x1cc}; + let finalize=if fragment {0x280}else{0x268}; + let pool=if fragment {0x1b0}else{0x190}; + let length=if fragment {0x370}else{0x300}; + let uuid=if fragment {fragment::UUID}else{ta::UUID}; + let bytes=output.get_mut(..STORAGE).ok_or(Error::Size)?; + bytes.fill(0); + word(bytes,0,if fragment {7}else{5}); + pointer(bytes,0x14,command+if fragment {0x80}else{0x40}); + pointer(bytes,pool,self.pool.get()); + pointer(bytes,pool+0x18,auxiliary.get()); + pointer(bytes,pool+0x28,serial); + word(bytes,pool+0x30,u32::from(fragment)); + if fragment { + pointer(bytes,0x1c,self.scene.get()); + pointer(bytes,0x24,activity); + pointer(bytes,0x2c,command+0xb70); + pointer(bytes,0x34,scene_tail); + pointer(bytes,0x3c,command+0xbb0); + pointer(bytes,0x44,command+0xbb4); + pointer(bytes,0x4c,self.queue.get()); + pointer(bytes,0x54,command); + word(bytes,0x5c,self.context); + word(bytes,0x60,1); + pointer(bytes,0x7c,command+0xa58); + pointer(bytes,0x84,command+0xbf0); + word(bytes,0xa0,uuid); + } else { + pointer(bytes,0x1c,self.manager.get()); + pointer(bytes,0x24,self.scene.get()); + pointer(bytes,0x2c,activity); + pointer(bytes,0x34,self.queue.get()); + pointer(bytes,0x3c,command+0x85c); + word(bytes,0x44,self.context); + word(bytes,0x48,1); + pointer(bytes,0x64,command+0x784); + pointer(bytes,0x6c,command+0x8a8); + word(bytes,0x7c,uuid); + } + for (at,opcode,selected) in [(first,0x8000_0003,pair),(first+0x50,3,pair+8)] { + word(bytes,at,opcode); + pointer(bytes,at+4,command+clock); + pointer(bytes,at+0xc,command+pair); + pointer(bytes,at+0x14,command+selected); + pointer(bytes,at+0x1c,self.queue.get()); + pointer(bytes,at+0x2c,command+aux_time); + pointer(bytes,at+0x34,command+elapsed); + pointer(bytes,at+0x44,uuid.into()); + } + word(bytes,first+0x4c,1); + word(bytes,finalize,if fragment {8}else{6}); + if fragment { + word(bytes,finalize+4,uuid); + pointer(bytes,finalize+0xc,self.fw_stamp.get()); + word(bytes,finalize+0x14,self.stamp); + pointer(bytes,finalize+0x1c,self.scene.get()); + pointer(bytes,finalize+0x24,self.manager.get()); + word(bytes,finalize+0x2c,1); + pointer(bytes,finalize+0x30,activity); + pointer(bytes,finalize+0x38,command+0xbb0); + pointer(bytes,finalize+0x40,command+0xbb4); + pointer(bytes,finalize+0x48,command+0xb70); + pointer(bytes,finalize+0x50,self.queue.get()); + pointer(bytes,finalize+0x58,command); + word(bytes,finalize+0x60,self.context); + pointer(bytes,finalize+0x64,command+0xa58); + pointer(bytes,finalize+0x98,sequence+pool as u64); + word(bytes,finalize+0xa0,(-(finalize as i32))as u32); + pointer(bytes,finalize+0xa5,command+0xc60); + word(bytes,finalize+0xad,1); + bytes[finalize+0xb1]=1; + // J613 terminates this record after the 16-bit event field. + bytes[finalize+0xb2..finalize+0xb4].copy_from_slice(&1u16.to_le_bytes()); + word(bytes,finalize+0xb4,0x4000_0002); + } else { + pointer(bytes,finalize+4,self.scene.get()); + pointer(bytes,finalize+0xc,self.manager.get()); + pointer(bytes,finalize+0x14,activity); + pointer(bytes,finalize+0x1c,self.queue.get()); + pointer(bytes,finalize+0x24,command+0x85c); + word(bytes,finalize+0x2c,self.context); + pointer(bytes,finalize+0x34,command+0x784); + word(bytes,finalize+0x40,uuid); + pointer(bytes,finalize+0x48,self.fw_stamp.get()); + word(bytes,finalize+0x50,self.stamp); + pointer(bytes,finalize+0x78,sequence+pool as u64); + word(bytes,finalize+0x80,(-(finalize as i32))as u32); + pointer(bytes,finalize+0x85,command+0x918); + bytes[finalize+0x8d]=1; + word(bytes,finalize+0x94,0x4000_0002); + } + Ok(length) + } +} + +#[cfg(test)] +mod native_tests { + use super::*; + #[test] fn j613_fragment_ends_before_the_m4_tail_and_keeps_relative_restart() { + let fw=|index:u64|FirmwareVa::new(0xffff_fc25_2000_0000+index*0x4000).unwrap(); + let o=Owners {command:Region::new(fw(0).get(),0xca0).unwrap(),sequence:Region::new(fw(1).get(),STORAGE).unwrap(), + stats:Region::new(fw(2).get(),0x1000).unwrap(),notifier:Region::new(fw(3).get(),0x200).unwrap(), + queue:fw(4),scene:fw(5),manager:fw(6),pool:fw(7),fw_stamp:fw(8),context:3,stamp:0x3d000100}; + let mut output=[0xa5;STORAGE+8]; + let length=o.encode_extended(true,super::super::m3_compute_layout::GpuVa::new(0x72_3000_0200).unwrap(),1,&mut output).unwrap(); + assert_eq!(length,0x370); + let word=|at|u32::from_le_bytes(output[at..at+4].try_into().unwrap()); + assert_eq!(word(0x1e4),0x8000_0003); + assert_eq!(word(0x230),1); + assert_eq!(word(0x234),3); + assert_eq!(word(0x280),8); + assert_eq!(word(0x320),(-0x280i32)as u32); + assert_eq!(word(0x334),0x4000_0002); + assert_eq!(&output[STORAGE..],&[0xa5;8]); + } +} + +/// Native start records reserve one packed attachment list per engine. The +/// matching finalizer flag enables completion visibility for that same list. +pub(crate) const fn extended_attachment_offsets(fragment:bool)->(usize,usize) { + if fragment {(0xa8,0x324)} else {(0x88,0x2ec)} +} + +/// Send both pieces through the same bounded writer used for firmware storage. +#[cfg(any(test, CONFIG_DRM_ASAHI_M3_T8122))] +pub(crate) fn patch_extended_attachments(fragment:bool, + attachments:&super::agx_attachments::Attachments, + mut write:impl FnMut(usize,&[u8])->Result<(),E>)->Result<(),E> { + let (start,flag)=extended_attachment_offsets(fragment); + write(start,&attachments.0)?; + write(flag,&[u8::from(attachments.count()!=0)]) +} + +#[cfg(test)] +mod attachment_tests { + use super::*; + use super::super::agx_attachments::{Attachments,SIZE}; + #[test] fn each_engine_keeps_its_list_and_can_clear_a_previous_list() { + for (fragment,start,flag,pool) in [(false,0x88,0x2ec,0x190),(true,0xa8,0x324,0x1b0)] { + let mut bytes=[0xa5;STORAGE]; + let list=Attachments::new(&[(0x100_0040,0x180),(0x200_0000,0x4000)]).unwrap(); + let mut apply=|list:&Attachments|patch_extended_attachments(fragment,list,|at,data|{ + bytes.get_mut(at..at+data.len()).ok_or(Error::Size)?.copy_from_slice(data);Ok::<(),Error>(()) + }); + apply(&list).unwrap(); + assert_eq!(&bytes[start..start+SIZE],&list.0); + assert_eq!(bytes[flag],1); + assert_eq!(&bytes[pool..pool+8],&[0xa5;8]); + patch_extended_attachments(fragment,&Attachments::EMPTY,|at,data|{ + bytes[at..at+data.len()].copy_from_slice(data);Ok::<(),Error>(()) + }).unwrap(); + assert!(bytes[start..start+SIZE].iter().all(|b|*b==0)); + assert_eq!(bytes[flag],0); + assert_eq!(bytes[start-1],0xa5); + assert_eq!(bytes[start+SIZE],0xa5); + } + } +} diff --git a/drivers/gpu/drm/asahi/m3/m3_runtime.rs b/drivers/gpu/drm/asahi/m3/m3_runtime.rs index 058c6610399ae..3b1f3d6d08535 100644 --- a/drivers/gpu/drm/asahi/m3/m3_runtime.rs +++ b/drivers/gpu/drm/asahi/m3/m3_runtime.rs @@ -390,7 +390,7 @@ impl Inner { if found.is_none() { let job=match control { crate::m3::m3_submit::Command::Compute(c)=>NativeJob::Compute(crate::m3::m3_compute::Compute::new(&self.drm,&self.uat,&packet.vm,self.config.stats_region().map_err(early)?,c,self.device.soc().registers,self.device.firmware().profile()).map_err(early)?), - crate::m3::m3_submit::Command::Render{..}=>NativeJob::Render(crate::m3::m3_render::Render::new(&self.drm,&self.uat,&packet.vm,self.config.stats_region().map_err(early)?,self.device.soc().clusters,self.device.soc().registers).map_err(early)?), + crate::m3::m3_submit::Command::Render{..}=>NativeJob::Render(crate::m3::m3_render::Render::new(&self.drm,&self.uat,&packet.vm,self.config.render_stats().map_err(early)?,self.device.soc().clusters,self.device.soc().registers,self.device.firmware().profile()).map_err(early)?), }; self.jobs.push(job,GFP_KERNEL).map_err(|_|early(ENOMEM))?; } else { @@ -439,6 +439,7 @@ impl Inner { let timed=timestamps.map(|pair|policy::stage_timestamps(internal_timestamps,chip,pair)); j.append(command,usc,coalesce,timed)?; j.set_user_timestamps(timestamps)?; + j.set_attachments(&p.render_attachments[*next])?; coalesced_stamps+=u64::from(coalesce); } Ok(()) @@ -563,6 +564,7 @@ impl Inner { let timed=timestamps.map(|pair|policy::stage_timestamps(internal_timestamps,self.device.soc().chip_id,pair)); j.append_after(command,usc,false,predecessor,timed)?; j.set_user_timestamps(timestamps)?; + j.set_attachments(&p.render_attachments[*i])?; dependencies+=usize::from(predecessor.is_some()); previous=Some((true,j.completion_dependency()?)); }, diff --git a/drivers/gpu/drm/asahi/m3/m3_scene_layout.rs b/drivers/gpu/drm/asahi/m3/m3_scene_layout.rs index 880cf4c2b9a75..ea3c2064ef39e 100644 --- a/drivers/gpu/drm/asahi/m3/m3_scene_layout.rs +++ b/drivers/gpu/drm/asahi/m3/m3_scene_layout.rs @@ -39,3 +39,43 @@ impl Scene { Ok(()) } } + +pub(crate) const EXTENDED_SIZE: usize = 0x400; + +/// Encode a native scene header from its retained aliases. The local +/// statistics and parameter scratch remain part of this same owned record. +pub(crate) fn encode_extended(output: &mut [u8], owner: super::m3_init_layout::Region, + scratch_gpu: super::m3_pool_layout::GpuRegion, scratch_fw: FirmwareVa, + scene_list: super::m3_pool_layout::GpuRegion) -> Result<(), Error> { + let statistics = owner.at(0x200, 0x40).map_err(|_| Error::Size)?; + owner.at(0, EXTENDED_SIZE).map_err(|_| Error::Size)?; + if scratch_gpu.size() < 0x4000 || scene_list.size() < 0x4000 { return Err(Error::Size); } + let compact = super::agx_render_state::compact(scene_list.base()).map_err(|_| Error::Address)?; + let bytes = output.get_mut(..EXTENDED_SIZE).ok_or(Error::Size)?; + bytes.fill(0); + bytes[..8].copy_from_slice(&scratch_gpu.base().to_le_bytes()); + bytes[8..16].copy_from_slice(&scratch_fw.get().to_le_bytes()); + bytes[0x28..0x30].copy_from_slice(&compact.to_le_bytes()); + bytes[0x40..0x48].copy_from_slice(&statistics.to_le_bytes()); + Ok(()) +} + +#[cfg(test)] +mod extended_tests { + use super::*; + use super::super::{m3_init_layout::Region, m3_pool_layout::GpuRegion}; + #[test] fn native_scene_keeps_scratch_aliases_and_local_statistics_distinct() { + let owner = Region::new(0xffff_fc25_0010_0000, EXTENDED_SIZE).unwrap(); + let scratch = GpuRegion::new(0x10_0200_0000, 0x4000).unwrap(); + let scratch_fw = FirmwareVa::new(0xffff_fc25_0020_0000).unwrap(); + let list = GpuRegion::new(0x10_0300_0000, 0x4000).unwrap(); + let mut bytes = [0xa5; EXTENDED_SIZE + 8]; + encode_extended(&mut bytes, owner, scratch, scratch_fw, list).unwrap(); + let read = |at| u64::from_le_bytes(bytes[at..at + 8].try_into().unwrap()); + assert_eq!(read(0), scratch.base()); + assert_eq!(read(8), scratch_fw.get()); + assert_eq!(read(0x28), 0x300_0000); + assert_eq!(read(0x40), owner.at(0x200, 0x40).unwrap()); + assert_eq!(&bytes[EXTENDED_SIZE..], &[0xa5; 8]); + } +} diff --git a/drivers/gpu/drm/asahi/m3/m3_shared_layout.rs b/drivers/gpu/drm/asahi/m3/m3_shared_layout.rs index 9b8d503836d0a..0a2262858556d 100644 --- a/drivers/gpu/drm/asahi/m3/m3_shared_layout.rs +++ b/drivers/gpu/drm/asahi/m3/m3_shared_layout.rs @@ -100,3 +100,29 @@ pub(crate) fn fault_buffers()->impl Iterator { .chain((PB_FIRST..PB_FIRST+PB_GROUPS).map(|i|(i,256))) .chain([(TILEMAP_LABEL,1280),(HEAP_METADATA_LABEL,256)]) } + +pub(crate) fn allocation_for(profile: super::m3_profile::Profile, index: usize) -> Result { + let mut owner = allocation(index)?; + if profile == super::m3_profile::Profile::Legacy { return Ok(owner); } + if owner.space == Space::Firmware { + owner.address = 0xffff_fc24_0000_0000 + index as u64 * 0x10_0000; + } + match index { + // Extended firmware reads the uncached kernel aliases supplied to + // encode_extended; these lower client mappings are GPU-only. + PARAMETER_PAGE_LIST | PARAMETER_BLOCK_LIST => owner.access = Access::GpuCachedRw, + 8 | 20 => owner.size = 0x20000, + 9 | 21 => { owner.address &= !0x3fff; owner.size = 0x4000; } + 10 | 22 => { + owner.size = 0x4000; + owner.gpu_alias = Some(0x72_3000_0000 + u64::from(index == 22) * 0x4000); + } + GPU_CONTEXT => owner.size = 0x38, + NOTIFIER => owner.size = 0x400, + EVENT_COUNT => owner.size = 16, + BUFFER_MANAGER => owner.size = 0xc0, + FRAGMENT_QUEUE | TA_QUEUE => owner.size = 0x24c0, + _ => {} + } + Ok(owner) +} diff --git a/drivers/gpu/drm/asahi/m3/m3_submit.rs b/drivers/gpu/drm/asahi/m3/m3_submit.rs index 2aac0dfad36ff..29d2bc7de5f68 100644 --- a/drivers/gpu/drm/asahi/m3/m3_submit.rs +++ b/drivers/gpu/drm/asahi/m3/m3_submit.rs @@ -41,6 +41,7 @@ pub(crate) struct Packet { timestamps:KVec<[Option;4]>, pub(crate) attachments:KVec, pub(crate) wide_visibility:KVec, + pub(crate) render_attachments:KVec<[crate::m3::agx_attachments::Attachments;2]>, completion:UserFence, finish_claimed: AtomicBool, /// Set before a batch with any of this packet's commands can change firmware-visible state, @@ -196,7 +197,7 @@ impl queue::Queue for Queue { fn submit(&mut self,_reg_data:&crate::driver::AsahiRegData<'_>,_id:u64,mut syncs:KVec,in_sync_count:usize, raw:&[u8],objects:Pin<&xarray::XArray>>)->Result { let vm_job=self.vm.retain_job()?; - let mut parser=agx_uapi::UapiCommandParser::new(raw);let mut commands=KVec::new();let mut timestamps=KVec::new();let mut flushes=KVec::new();let mut visibility=KVec::new(); + let mut parser=agx_uapi::UapiCommandParser::new(raw);let mut commands=KVec::new();let mut timestamps=KVec::new();let mut flushes=KVec::new();let mut visibility=KVec::new();let mut render_attachments=KVec::new(); while let Some(command)=parser.next_hardware().map_err(|_|EINVAL)? { if let agx_uapi::ParsedHardwareCommand::Render{payload,vertex_attachments,fragment_attachments,..}=command { if let Err(e)=validate_render(payload,self.usc,&AddressSpace(&self.vm),self.early_tiling) { @@ -211,6 +212,7 @@ impl queue::Queue for Queue { Destination::resolve(objects,payload.fragment_timestamps.start)?, Destination::resolve(objects,payload.fragment_timestamps.end)?],GFP_KERNEL)?; flushes.push(crate::m3::agx_attachments::Attachments::EMPTY,GFP_KERNEL)?; + render_attachments.push([encode_attachments(&self.vm,&vertex_attachments)?,encode_attachments(&self.vm,&fragment_attachments)?],GFP_KERNEL)?; visibility.push(false,GFP_KERNEL)?; continue; } @@ -226,6 +228,7 @@ impl queue::Queue for Queue { // The common translator validates the original command/attachment // addresses; this M3 scheduling hint changes none of them. payload.flags &= !wide; + render_attachments.push([crate::m3::agx_attachments::Attachments::EMPTY;2],GFP_KERNEL)?; flushes.push(encode_attachments(&self.vm,&attachments)?,GFP_KERNEL)?; let c=agx_uapi::translate_compute_command(payload,attachments, agx_uapi::QueueUscWindow{base:self.usc,user_start:0x4000,user_end:(1u64<<42)-0x8000}, @@ -239,7 +242,7 @@ impl queue::Queue for Queue { commands.push(Command::Compute(crate::m3::agx_compute::Control{base:c.control_stream_base,end:c.control_stream_end,usc_base:self.usc}),GFP_KERNEL)?; } parser.finish().map_err(|_|EINVAL)?; - let packet=Arc::new(Packet{vm:self.vm.clone(),commands,timestamps,attachments:flushes,wide_visibility:visibility, + let packet=Arc::new(Packet{vm:self.vm.clone(),commands,timestamps,attachments:flushes,wide_visibility:visibility,render_attachments, completion:self.fences.new_fence(0,Completion)?.into(), finish_claimed:AtomicBool::new(false),published:AtomicBool::new(false),urgency:self.urgency,owner:Owner::current(), vm_job:KBox::pin_init(new_mutex!(Some(vm_job)),GFP_KERNEL)?},GFP_KERNEL)?; diff --git a/drivers/gpu/drm/asahi/m3/m3_sync_layout.rs b/drivers/gpu/drm/asahi/m3/m3_sync_layout.rs index 92d533efa8234..2cf5e18914043 100644 --- a/drivers/gpu/drm/asahi/m3/m3_sync_layout.rs +++ b/drivers/gpu/drm/asahi/m3/m3_sync_layout.rs @@ -91,3 +91,58 @@ impl Barrier { Ok(()) } } + +impl InitBufferManager { + /// The extended rebuild path requires a zero staged-block count. Appending + /// already-staged blocks would leave a previous hardware descriptor live. + pub(crate) fn encode_extended(&self, output: &mut [u8]) -> Result<(), Error> { + if self.context >= 64 || self.slot >= 32 { return Err(Error::Address); } + Self { context: self.context, slot: self.slot, block_count: 0, + manager: self.manager, stamp: self.stamp }.encode(output) + } +} + +impl Barrier { + pub(crate) const EXTENDED_SIZE: usize = 0x40; + + pub(crate) fn encode_extended(&self, output: &mut [u8]) -> Result<(), Error> { + if self.event >= 128 || self.wait_value == 0 || self.self_value == 0 { + return Err(Error::Address); + } + let bytes = output.get_mut(..Self::EXTENDED_SIZE).ok_or(Error::Size)?; + bytes.fill(0); + u32_at(bytes, 0, 4); + address_at(bytes, 4, self.stamp); + address_at(bytes, 12, self.stamp); + u32_at(bytes, 0x14, self.wait_value); + u32_at(bytes, 0x20, self.event); + u32_at(bytes, 0x24, self.self_value); + if matches!(self.dependency, Dependency::PreviousPass) { u32_at(bytes, 0x3c, 1); } + Ok(()) + } +} + +#[cfg(test)] +mod extended_tests { + use super::*; + #[test] fn full_rebuild_never_uses_the_legacy_staged_count() { + let mut bytes = [0xa5; INIT_BM_SIZE]; + InitBufferManager { context: 3, slot: 7, block_count: 2048, + manager: FirmwareVa::new(0xffff_fc24_0010_0000).unwrap(), stamp: 0x100 } + .encode_extended(&mut bytes).unwrap(); + assert_eq!(&bytes[12..20], &[0; 8]); + } + #[test] fn extended_dependencies_use_their_native_tail_and_preserve_trailing_storage() { + for previous in [false, true] { + let mut bytes = [0xa5; Barrier::EXTENDED_SIZE + 8]; + Barrier { stamp: FirmwareVa::new(0xffff_fc24_0010_0000).unwrap(), wait_value: 0x100, + event: 2, self_value: 0x200, uuid: 9, + dependency: if previous { Dependency::PreviousPass } else { Dependency::TilerToFragment } } + .encode_extended(&mut bytes).unwrap(); + assert_eq!(&bytes[0x18..0x20], &[0; 8]); + assert_eq!(&bytes[0x20..0x24], &[2, 0, 0, 0]); + assert_eq!(u32::from_le_bytes(bytes[0x3c..0x40].try_into().unwrap()), previous.into()); + assert_eq!(&bytes[Barrier::EXTENDED_SIZE..], &[0xa5; 8]); + } + } +} diff --git a/drivers/gpu/drm/asahi/m3/m3_tiler_command.rs b/drivers/gpu/drm/asahi/m3/m3_tiler_command.rs index eeda857a3ed88..caa61621e0b04 100644 --- a/drivers/gpu/drm/asahi/m3/m3_tiler_command.rs +++ b/drivers/gpu/drm/asahi/m3/m3_tiler_command.rs @@ -3,6 +3,17 @@ use super::{agx_render::Geometry, m3_compute_layout::{GpuVa,RegisterSet}, m3_init_layout::{Error,Region}, m3_queue_layout::FirmwareVa, m3_render_sequence::ta}; pub(crate) const SIZE:usize=0x92b; +pub(crate) const EXTENDED_SIZE:usize=0x940; +pub(crate) const EXTENDED_FLUSH:usize=0x894; +pub(crate) struct Dispatch26 { + pub(crate) incarnation:u8, + pub(crate) serial:u64, + pub(crate) peer_stamp:u32, + pub(crate) usc_window:u64, + pub(crate) firmware_aux:FirmwareVa, + pub(crate) scene_scratch:GpuVa, +} + pub(crate) const REGISTERS:usize=ta::REGISTERS; /// The registers G15G does not take; the others keep the G15S order. pub(crate) const G15G_OMIT:[u32;14]=[0x1c041,0x1c9d0,0x1c948,0x1c888,0x1c890,0x1c918,0x1c089,0x1c9e0,0x16c41,0x1ca40,0x1c9a8,0x1c920,0xa5a1,0xd419]; @@ -57,17 +68,32 @@ impl Command { /// All address checks precede mutation. Zero all reserved fields, then /// emit semantic registers and explicit packed command/tail fields. pub(crate) fn encode(&self,out:&mut[u8])->Result<(),Error> { - if out.len() Result<(),Error> { + self.encode_format(out, Some(dispatch)) + } + fn encode_format(&self, out: &mut [u8], dispatch: Option) -> Result<(),Error> { + let extended = dispatch.is_some(); + let size = if extended { EXTENDED_SIZE } else { SIZE }; + if out.len()= 1 << 42 || self.register_set != RegisterSet::G15G { + return Err(Error::Address); + } + } if self.context>=64 || self.page_count==0 || self.tpc_bytes==0 {return Err(Error::Address);} let registers=self.gpu_alias.get().checked_add(REGISTERS as u64).ok_or(Error::Address)?; - GpuVa::new(self.gpu_alias.get().checked_add((SIZE-3) as u64).ok_or(Error::Address)?) + GpuVa::new(self.gpu_alias.get().checked_add(((size-1)&!3) as u64).ok_or(Error::Address)?) .map_err(|_|Error::Address)?; - let seq=self.sequence.at(0,ta::LENGTH)?; + let sequence_size = if extended { 0x300 } else { ta::LENGTH }; + let seq=self.sequence.at(0,sequence_size)?; let tilemap=compact(self.tilemap.get())?; let tpc=compact(self.tpc.get())?; let heap=compact(self.heap.get())?; let preemption=[compact(self.preemption[0].get())?,compact(self.preemption[1].get())?,compact(self.preemption[2].get())?]; - let user=compact(self.scene_user.get().checked_add(0x40).ok_or(Error::Address)?)?; + let user=compact(self.scene_user.get().checked_add(if extended {0} else {0x40}).ok_or(Error::Address)?)?; let g=&self.geometry; let layers=g.layers; let layer_metadata=compact(self.heap.get().checked_add(if layers>1 {0x200}else{0}).ok_or(Error::Address)?)?; @@ -99,23 +125,67 @@ impl Command { (0x1c830,0),(0x1ca30,user),(0x16c39,user), (0x1c910,0xa0_00000000|(self.initial_scene_entry.get()&0xf_ffffffff)|1), ]; - let bytes=&mut out[..SIZE];bytes.fill(0); // TA opcode is zero - pointer(bytes,4,1); // qualified work-command counter remains one + let bytes=&mut out[..size];bytes.fill(0); // TA opcode is zero + pointer(bytes,4,dispatch.as_ref().map_or(1, |native| native.serial)); word(bytes,CONTEXT,self.context); - for (offset,value) in [(0x14,self.notifier.get()),(0x24,self.manager.get()), - (0x2c,self.scene.get()),(0x34,self.empty.get()),(0x740,registers), + for (offset,value) in [(if extended {0x10} else {0x14},self.notifier.get()), + (if extended {0x20} else {0x24},self.manager.get()), + (if extended {0x28} else {0x2c},self.scene.get()), + (if extended {0x30} else {0x34},self.empty.get()),(0x740,registers), (0x760,self.tpc.get()),(0x768,self.tpc_bytes),(0x770,seq)] {pointer(bytes,offset,value);} let g15g=self.register_set==RegisterSet::G15G; let mut count=0; - for (register,value) in values.into_iter().filter(|(r,_)| !g15g || !G15G_OMIT.contains(r)) { - let n=count;count+=1; - word(bytes,REGISTERS+n*REGISTER_STRIDE,register); - pointer(bytes,REGISTERS+n*REGISTER_STRIDE+4,value); + let mut emit = |register, value| { + word(bytes, REGISTERS + count * REGISTER_STRIDE, register); + pointer(bytes, REGISTERS + count * REGISTER_STRIDE + 4, value); + count += 1; + }; + for (register, mut value) in values { + if g15g && G15G_OMIT.contains(®ister) { continue; } + if extended && matches!(register, 0x12099 | 0x1c898) { continue; } + if let Some(native) = &dispatch { + if register == 0x101e1 { + emit(0xa309, 0); + emit(0x1c8e0, u64::MAX); + emit(0x1c8e8, u64::MAX); + emit(0x1c898, 0); + } + if register == 0x1c910 { + let address = native.scene_scratch.get(); + let escaped = (address & (1 << 42)) >> 3; + let bias = if escaped == 0 { 0x70_0000_0000 } else { 0 }; + value = ((address + bias) & 0x7f_ffff_fffe) | escaped | 1; + } + } + emit(register, value); + } + if let Some(native) = &dispatch { + emit(0xa5a1, 0x60_0040_0020); + emit(0xd419, 0x2_0000_0001); + for register in [0x1ca10, 0x14a1, 0xa349] { emit(register, ta::UUID.into()); } + emit(0x10061, native.usc_window); } bytes[0x748..0x74a].copy_from_slice(&(count as u16).to_le_bytes()); bytes[0x74a..0x74c].copy_from_slice(&((count*REGISTER_STRIDE) as u16).to_le_bytes()); - word(bytes,SEQUENCE_LENGTH,ta::LENGTH as u32); + word(bytes,SEQUENCE_LENGTH,sequence_size as u32); word(bytes,0x77c,1); // fragment event slot + if let Some(native) = dispatch { + word(bytes, STAMP_VALUE, native.peer_stamp); + pointer(bytes, 0x7a8, preemption[0]); + pointer(bytes, 0x878, self.stamp.get()); + pointer(bytes, 0x880, self.fw_stamp.get()); + word(bytes, 0x888, self.stamp_value); + word(bytes, EXTENDED_FLUSH, 1); + word(bytes, 0x898, ta::UUID); + pointer(bytes, 0x8c8, self.timestamps[0].get()); + pointer(bytes, 0x8d0, self.timestamps[1].get()); + pointer(bytes, 0x8d8, self.user_timestamps[0].get()); + pointer(bytes, 0x8e0, self.user_timestamps[1].get()); + pointer(bytes, 0x8fe, self.pool.get()); + bytes[0x906] = 1; + pointer(bytes, 0x90f, native.firmware_aux.get()); + bytes[0x917] = native.incarnation; + } else { word(bytes,STAMP_VALUE,self.stamp_value); pointer(bytes,ta::STRUCT3+0x24,preemption[0]); word(bytes,ta::STRUCT3+0xbc,0x67); // retained producer encoder_id @@ -132,6 +202,41 @@ impl Command { (0x47,self.scratch.get())] {pointer(bytes,ta::COMMAND_TAIL+offset,value);} bytes[ta::COMMAND_TAIL+0x36]=1; // unk_846 bytes[ta::COMMAND_TAIL+0x4f]=1; // unk_85f + } Ok(()) } } + +#[cfg(test)] +mod extended_tests { + use super::*; + fn command() -> Command { + let fw=|index:u64|FirmwareVa::new(0xffff_fc25_1000_0000+index*0x4000).unwrap(); + let gpu=|index:u64|GpuVa::new(0x10_0100_0000+index*0x4000).unwrap(); + Command {context:3,register_set:RegisterSet::G15G,notifier:fw(0),manager:fw(1),scene:fw(2),empty:fw(3), + gpu_alias:GpuVa::new(0x72_1000_0000).unwrap(),sequence:Region::new(fw(4).get(),0x1000).unwrap(), + pool:fw(5),scratch:fw(6),stamp:fw(7),fw_stamp:fw(8),stamp_value:0x7a000100, + timestamps:[fw(9),fw(10)],user_timestamps:[fw(11),fw(12)],preemption:[gpu(0),gpu(1),gpu(2)], + tilemap:gpu(3),tpc:gpu(4),tpc_bytes:0x4000,heap:gpu(5),scene_user:gpu(6), + initial_scene_entry:fw(13),geometry:Geometry::new(64,64,super::super::agx_render::Utile::Pixels32, + super::super::agx_render::Utile::Pixels32,1).unwrap(),page_count:128, + vdm:0x100000,multisample:0,ppp:0,process_empty_tiles:false} + } + #[test] fn native_tiler_keeps_peer_completion_and_usc_window_explicit() { + let value=command();let mut bytes=[0xa5;EXTENDED_SIZE+8]; + value.encode_extended(Dispatch26 {incarnation:9,serial:2,peer_stamp:0x3d000100, + usc_window:1<<32,firmware_aux:value.scratch, + scene_scratch:GpuVa::new(0x10_0200_0000).unwrap()},&mut bytes).unwrap(); + let word=|at|u32::from_le_bytes(bytes[at..at+4].try_into().unwrap()); + assert_eq!(word(0x780),0x3d000100); + assert_eq!(word(0x888),value.stamp_value); + assert_eq!(word(0x894),1); + assert_eq!(word(0x748)&0xffff,66); + assert_eq!(bytes[0x917],9); + assert_eq!(&bytes[EXTENDED_SIZE..],&[0xa5;8]); + let registers=&bytes[REGISTERS..REGISTERS+66*REGISTER_STRIDE]; + assert!(!registers.chunks_exact(12).any(|r|u32::from_le_bytes(r[..4].try_into().unwrap())==0x12099)); + let usc=registers.chunks_exact(12).find(|r|u32::from_le_bytes(r[..4].try_into().unwrap())==0x10061).unwrap(); + assert_eq!(u64::from_le_bytes(usc[4..].try_into().unwrap()),1<<32); + } +} diff --git a/drivers/gpu/drm/asahi/m3/t8122_admission.rs b/drivers/gpu/drm/asahi/m3/t8122_admission.rs index fdc41024899f1..fb280ebcde520 100644 --- a/drivers/gpu/drm/asahi/m3/t8122_admission.rs +++ b/drivers/gpu/drm/asahi/m3/t8122_admission.rs @@ -1,6 +1,6 @@ // SPDX-License-Identifier: GPL-2.0-only OR MIT -//! Read-only T8122 identity admission. Firmware startup remains disabled. +//! Read-only T8122 identity and firmware-interface admission. use super::m3_profile::Profile; @@ -53,6 +53,28 @@ pub(crate) fn validate_identity_for( }) } +/// Firmware-interface selection is exact and does not change physical GPU encodings. +pub(crate) fn profile_for(compat:&[u32],j613:bool)->Option { + match compat { + [14,8,3]=>Some(Profile::Legacy), + [26,6,2] if j613=>Some(Profile::Extended), + _=>None, + } +} + +#[cfg(not(test))] +pub(crate) fn loaded_profile(pdev:&kernel::platform::Device>) + ->kernel::error::Result { + use kernel::{c_str,prelude::*}; + let node=pdev.as_ref().of_node().ok_or(ENODEV)?; + let compat:KVec=node.get_property(c_str!("apple,firmware-compat"))?; + #[cfg(CONFIG_DRM_ASAHI_M3_T8122)] + let j613 = super::m3_board::board_is(b"apple,j613"); + #[cfg(not(CONFIG_DRM_ASAHI_M3_T8122))] + let j613 = false; + profile_for(&compat, j613).ok_or(ENODEV) +} + /// Shape of the voltage-sorted running-state tables: one entry per distinct millivolt, /// and the highest primary frequency in MHz. Voltages round up, matching `PwrConfig`. /// The first point is the off state. Both primary and secondary frequencies must increase @@ -108,7 +130,7 @@ pub(crate) fn opp_table_shape(opps: impl IntoIterator) -> Opt pub(crate) fn read_identity( pdev: &kernel::platform::Device>, ) -> kernel::error::Result { - read_identity_for(pdev, Profile::Legacy) + read_identity_for(pdev, loaded_profile(pdev)?) } #[cfg(not(test))] @@ -300,3 +322,17 @@ mod tests { assert!(validate_identity(&[14, 8, 3], 1, 1, G15G_C0, [0x7f, 0, 0]).is_none()); } } + +#[cfg(test)] +mod selection_tests { + use super::*; + #[test] fn extended_admission_requires_exact_firmware_and_j613() { + for board in [false,true] { + assert_eq!(profile_for(&[14,8,3],board),Some(Profile::Legacy)); + assert_eq!(profile_for(&[26,6,2],board),if board{Some(Profile::Extended)}else{None}); + for wrong in [&[][..],&[26,6][..],&[26,6,1][..],&[26,6,2,0][..],&[14,8,4][..]] { + assert_eq!(profile_for(wrong,board),None); + } + } + } +} diff --git a/tools/asahi/m3-runtime-tests.rs b/tools/asahi/m3-runtime-tests.rs index 458603366c954..91cbefd298fe2 100644 --- a/tools/asahi/m3-runtime-tests.rs +++ b/tools/asahi/m3-runtime-tests.rs @@ -46,3 +46,63 @@ mod m3_state_layout; #[path = "../../drivers/gpu/drm/asahi/m3/m3_compute_storage.rs"] mod m3_compute_storage; + +#[path = "../../drivers/gpu/drm/asahi/m3/m3_parameter_layout.rs"] +mod m3_parameter_layout; + +#[path = "../../drivers/gpu/drm/asahi/m3/m3_sync_layout.rs"] +mod m3_sync_layout; + +#[path = "../../drivers/gpu/drm/asahi/m3/agx_render.rs"] +mod agx_render; + +#[path = "../../drivers/gpu/drm/asahi/m3/agx_render_state.rs"] +mod agx_render_state; + +#[path = "../../drivers/gpu/drm/asahi/m3/m3_scene_layout.rs"] +mod m3_scene_layout; + +#[path = "../../drivers/gpu/drm/asahi/m3/m3_command_layout.rs"] +mod m3_command_layout; + +#[path = "../../drivers/gpu/drm/asahi/m3/m3_render_sequence.rs"] +mod m3_render_sequence; + +#[path = "../../drivers/gpu/drm/asahi/m3/m3_tiler_command.rs"] +mod m3_tiler_command; + +#[path = "../../drivers/gpu/drm/asahi/m3/m3_pass_layout.rs"] +mod m3_pass_layout; + +#[path = "../../drivers/gpu/drm/asahi/m3/m3_shared_layout.rs"] +mod m3_shared_layout; + +#[path = "../../drivers/gpu/drm/asahi/m3/m3_fragment_command.rs"] +mod m3_fragment_command; + +#[path = "../../drivers/gpu/drm/asahi/m3/agx_attachments.rs"] +mod agx_attachments; + +#[test] +fn extended_parameter_aliases_are_gpu_only_and_legacy_is_unchanged() { + use m3_pass_layout::{Access, Space}; + use m3_profile::Profile; + use m3_shared_layout as shared; + for index in 0..shared::COUNT { + let original = shared::allocation(index).unwrap(); + let legacy = shared::allocation_for(Profile::Legacy, index).unwrap(); + assert_eq!(legacy, original); + let extended = shared::allocation_for(Profile::Extended, index).unwrap(); + if [shared::PARAMETER_PAGE_LIST, shared::PARAMETER_BLOCK_LIST].contains(&index) { + assert_eq!(extended.space, Space::ClientGpu); + assert_eq!(extended.address, original.address); + assert_eq!(extended.size, original.size); + assert_eq!(extended.access, Access::GpuCachedRw); + assert_eq!(extended.access.pte(), 0xc0000000000080); + assert_eq!(original.access, Access::GpuFirmwareCachedRw); + } + if extended.space == Space::ClientGpu { + assert_ne!(extended.access, Access::GpuFirmwareCachedRw); + } + } +} From efcd402f4ccd5ecfb91745a944c328486af3745e Mon Sep 17 00:00:00 2001 From: Satya Benson Date: Tue, 3 Nov 2026 16:28:32 -0500 Subject: [PATCH 3/8] drm/asahi: admit authorized HIGH queues on Extended T8122 Signed-off-by: Satya Benson --- drivers/gpu/drm/asahi/file.rs | 10 ++++- drivers/gpu/drm/asahi/m3/m3_drm.rs | 5 +++ tools/asahi/queue-priority-tests.py | 69 +++++++++++++++++++++++++++++ 3 files changed, 82 insertions(+), 2 deletions(-) create mode 100644 tools/asahi/queue-priority-tests.py diff --git a/drivers/gpu/drm/asahi/file.rs b/drivers/gpu/drm/asahi/file.rs index 24e8db30e580e..a09fa33ed02a4 100644 --- a/drivers/gpu/drm/asahi/file.rs +++ b/drivers/gpu/drm/asahi/file.rs @@ -1350,9 +1350,15 @@ impl File { cls_pr_debug!(Errors, "queue_create: Invalid priority\n"); return Err(EINVAL); } - // HIGH has only been run on G17; G13 and G14 keep refusing it. + // Keep support tied to a backend and admitted profile. G13/G14 and + // unqualified M3 profiles retain their previous rejection. + let high_supported = reg_data.gpu.as_any().is::(); + #[cfg(CONFIG_DRM_ASAHI_M3_T8122)] + let high_supported = high_supported || reg_data.gpu.as_any() + .downcast_ref::() + .is_some_and(|gpu| gpu.supports_high_priority()); if data.priority == uapi::drm_asahi_priority_DRM_ASAHI_PRIORITY_HIGH - && !reg_data.gpu.as_any().is::() + && !high_supported { cls_pr_debug!( Errors, diff --git a/drivers/gpu/drm/asahi/m3/m3_drm.rs b/drivers/gpu/drm/asahi/m3/m3_drm.rs index 8aedd920f31dc..555edf7074a6b 100644 --- a/drivers/gpu/drm/asahi/m3/m3_drm.rs +++ b/drivers/gpu/drm/asahi/m3/m3_drm.rs @@ -96,6 +96,11 @@ pub(crate) struct Backend { } impl Backend { + /// HIGH promises host publication ordering, not firmware preemption. + pub(crate) fn supports_high_priority(&self) -> bool { + self.soc.chip_id == 0x8122 && self.profile == super::m3_profile::Profile::Extended + } + /// Admits and starts the M3 runtime on `soc` (the T8122), with the DRM device `drm` that the /// caller registers afterwards. pub(crate) fn start( diff --git a/tools/asahi/queue-priority-tests.py b/tools/asahi/queue-priority-tests.py new file mode 100644 index 0000000000000..5ecce0620588d --- /dev/null +++ b/tools/asahi/queue-priority-tests.py @@ -0,0 +1,69 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only OR MIT +"""Compile the production queue admission gate with controlled credentials/backends.""" +from pathlib import Path +import subprocess +import tempfile +root = Path(__file__).resolve().parents[2] +s = (root / 'drivers/gpu/drm/asahi/file.rs').read_text() +a = s.index(' if data.flags != 0', s.index('pub(crate) fn queue_create(')) +b = s.index(' let queues_xa', a) +gate = s[a:b] +predicate = (root / 'drivers/gpu/drm/asahi/m3/m3_drm.rs').read_text().split('pub(crate) fn supports_high_priority(&self) -> bool {', 1)[1].split('\n }', 1)[0] +unit = r''' +#![allow(dead_code, non_upper_case_globals)] +use std::any::Any; +const EINVAL:i32=22; const EACCES:i32=13; +macro_rules! cls_pr_debug { ($($arg:tt)*) => {} } +mod uapi { + pub const drm_asahi_priority_DRM_ASAHI_PRIORITY_REALTIME:u32=3; + pub const drm_asahi_priority_DRM_ASAHI_PRIORITY_HIGH:u32=2; +} +mod bindings { pub const CAP_SYS_NICE:u32=23; + pub unsafe fn capable(_:u32)->bool { CAP.with(|c|c.get()) } + thread_local! {pub static CAP:std::cell::Cell=const {std::cell::Cell::new(false)};} +} +mod g17 {pub struct Gpu;} +mod m3 {pub mod m3_drm { + #[derive(PartialEq)] pub enum Profile {Legacy, Extended} + pub struct Soc {pub chip_id:u32} + pub struct Backend {pub soc:Soc,pub profile:Profile} + impl Backend {pub fn supports_high_priority(&self)->bool { PREDICATE }} +}} +struct Gpu(Box); impl Gpu {fn as_any(&self)->&dyn Any {&*self.0}} +struct Reg {gpu:Gpu} +struct Data {flags:u32,priority:u32} +struct File(bool); impl File {fn is_current_master(&self)->bool {self.0}} +fn admit(reg_data:&Reg,data:&Data,file:&File)->Result<(),i32> { GATE Ok(()) } +fn main() { + use m3::m3_drm::*; + let mut n=0; + for backend in 0..5 { for cap in [false,true] { for master in [false,true] { + let gpu:Box=match backend { + 0=>Box::new(g17::Gpu), + 1=>Box::new(Backend{soc:Soc{chip_id:0x8122},profile:Profile::Extended}), + 2=>Box::new(Backend{soc:Soc{chip_id:0x8122},profile:Profile::Legacy}), + 3=>Box::new(Backend{soc:Soc{chip_id:0x6030},profile:Profile::Extended}), + _=>Box::new(())}; + let reg=Reg{gpu:Gpu(gpu)}; + bindings::CAP.with(|c|c.set(cap)); + for flags in [0,1,u32::MAX] { for priority in [0,1,2,3,4,u32::MAX] { + let expected=if flags!=0 || priority>=3 {Err(EINVAL)} + else if priority!=2 {Ok(())} + else if backend!=0 && !(backend==1 && cfg!(CONFIG_DRM_ASAHI_M3_T8122)) {Err(EINVAL)} + else if cap||master {Ok(())} else {Err(EACCES)}; + assert_eq!(admit(®,&Data{flags,priority},&File(master)),expected, + "backend={backend} cap={cap} master={master} flags={flags} priority={priority}"); n+=1; + }} + }}} + println!("{n} actual admission cases passed"); +} +'''.replace('PREDICATE', predicate.replace('super::m3_profile::Profile', 'Profile')).replace('GATE', gate) +with tempfile.TemporaryDirectory(prefix='queue-priority-') as d: + p = Path(d) / 'gate.rs' + p.write_text(unit) + for enabled in [False, True]: + subprocess.run(['rustc', '--edition=2021', *(['--cfg', 'CONFIG_DRM_ASAHI_M3_T8122'] if enabled else []), str(p), '-o', str(p.with_suffix(''))], check=True) + # Without M3 compiled only the G17 backend is admitted. + subprocess.run([str(p.with_suffix(''))], check=True) +print('M3 enabled/disabled gate compilation passed') From c1c0e23c6e4f50e03038a22a6cab7f8d72e39a54 Mon Sep 17 00:00:00 2001 From: Satya Benson Date: Tue, 3 Nov 2026 17:25:00 -0500 Subject: [PATCH 4/8] drm/asahi: test the M3 host priority mapping alongside admission Signed-off-by: Satya Benson --- tools/asahi/queue-priority-tests.py | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/tools/asahi/queue-priority-tests.py b/tools/asahi/queue-priority-tests.py index 5ecce0620588d..cfbbe8f36d4e6 100644 --- a/tools/asahi/queue-priority-tests.py +++ b/tools/asahi/queue-priority-tests.py @@ -10,6 +10,10 @@ b = s.index(' let queues_xa', a) gate = s[a:b] predicate = (root / 'drivers/gpu/drm/asahi/m3/m3_drm.rs').read_text().split('pub(crate) fn supports_high_priority(&self) -> bool {', 1)[1].split('\n }', 1)[0] +submit = (root / 'drivers/gpu/drm/asahi/m3/m3_submit.rs').read_text() +ordering = submit.split(' let urgency=', 1)[1].split(' Ok(Self', 1)[0] +ordering = 'let urgency=' + ordering +assert 'uapi::drm_asahi_priority_DRM_ASAHI_PRIORITY_REALTIME - data.priority' in s unit = r''' #![allow(dead_code, non_upper_case_globals)] use std::any::Any; @@ -35,6 +39,13 @@ struct Data {flags:u32,priority:u32} struct File(bool); impl File {fn is_current_master(&self)->bool {self.0}} fn admit(reg_data:&Reg,data:&Data,file:&File)->Result<(),i32> { GATE Ok(()) } +mod sched { + #[derive(Debug,PartialEq)] pub enum Priority {Low,Normal,High,Kernel} +} +fn ordering(priority:u32)->Result<(u8,sched::Priority),i32> { + ORDERING + Ok((urgency,priority)) +} fn main() { use m3::m3_drm::*; let mut n=0; @@ -56,9 +67,13 @@ "backend={backend} cap={cap} master={master} flags={flags} priority={priority}"); n+=1; }} }}} - println!("{n} actual admission cases passed"); + for (uapi,expected) in [(0,sched::Priority::Low),(1,sched::Priority::Normal),(2,sched::Priority::High)] { + assert_eq!(ordering(3-uapi),Ok((uapi as u8,expected))); + } + assert_eq!(ordering(4),Err(EINVAL)); + println!("{n} actual admission cases and LOW/MEDIUM/HIGH host mappings passed"); } -'''.replace('PREDICATE', predicate.replace('super::m3_profile::Profile', 'Profile')).replace('GATE', gate) +'''.replace('PREDICATE', predicate.replace('super::m3_profile::Profile', 'Profile')).replace('GATE', gate).replace('ORDERING', ordering) with tempfile.TemporaryDirectory(prefix='queue-priority-') as d: p = Path(d) / 'gate.rs' p.write_text(unit) From ba872b99a0039f15a82fa0f91ed7b8ab4e5905c1 Mon Sep 17 00:00:00 2001 From: Satya Benson Date: Tue, 3 Nov 2026 20:36:32 -0500 Subject: [PATCH 5/8] ci: register the M3 queue priority admission tests Signed-off-by: Satya Benson --- tools/asahi/run-host-tests.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/asahi/run-host-tests.py b/tools/asahi/run-host-tests.py index 5dfda83574ea6..4ebce94ccb47d 100755 --- a/tools/asahi/run-host-tests.py +++ b/tools/asahi/run-host-tests.py @@ -53,6 +53,7 @@ def mutations(flag, names, base, expect=KILL): # test file -> [(arguments, expectation)]. '{out}' is a fresh directory. ASAHI_TESTS = { + 'queue-priority-tests.py': [([], PASS)], 'test-m3-power.py': [([], PASS)], 'test-m3-uapi.py': [([], PASS)], 'test-m3-mailbox.py': [([], PASS)], From bd92c01562acb283b2138ba90843606a4b924164 Mon Sep 17 00:00:00 2001 From: Satya Benson Date: Tue, 3 Nov 2026 17:39:31 -0500 Subject: [PATCH 6/8] drm/asahi: back off interrupt-assisted M3 completion waits Signed-off-by: Satya Benson --- drivers/gpu/drm/asahi/m3/m3_completion.rs | 26 +++---- .../gpu/drm/asahi/m3/m3_completion_policy.rs | 77 +++++++++++++++++++ drivers/gpu/drm/asahi/m3/m3_drm.rs | 2 +- drivers/gpu/drm/asahi/m3/m3_rtkit.rs | 14 ++-- drivers/gpu/drm/asahi/m3/mod.rs | 2 + tools/asahi/m3-fence-latency.py | 63 +++++++++++++++ tools/asahi/m3-runtime-tests.rs | 3 + 7 files changed, 166 insertions(+), 21 deletions(-) create mode 100644 drivers/gpu/drm/asahi/m3/m3_completion_policy.rs create mode 100644 tools/asahi/m3-fence-latency.py diff --git a/drivers/gpu/drm/asahi/m3/m3_completion.rs b/drivers/gpu/drm/asahi/m3/m3_completion.rs index e06fcded9047e..387aaeca1b6c6 100644 --- a/drivers/gpu/drm/asahi/m3/m3_completion.rs +++ b/drivers/gpu/drm/asahi/m3/m3_completion.rs @@ -24,10 +24,9 @@ impl WorkItem for Completion { type Pointer = Arc; fn run(owner: Arc) { use crate::m3::m3_runtime::Service; - // Snapshots after a notification before sleeping for the next one: the completion - // stamps can trail the firmware's event message. - const TRAILING_POLLS: u32 = 4; - let (mut trailing, mut waited_at) = (0, u64::MAX); + // No deliberate spin: fsleep(5) selects udelay, not a scheduler sleep. + // Check immediately on IRQ, with backoff for trailing coherent witnesses. + let mut policy = super::m3_completion_policy::WaitPolicy::new(); loop { let (outcome, events) = { let mut guard = owner.shared.lock(); @@ -37,23 +36,20 @@ impl WorkItem for Completion { // Preserve the root-only CPU-overlap diagnostic: disabled waits retain // the runtime lock so VM/timestamp preparation cannot overlap the GPU. if let Service::Waiting(messages)=outcome { - if !events.cpu_overlap() {events.wait_past(messages);continue;} + if !events.cpu_overlap() { + events.wait_past(messages, policy.next_wait_us(events.wait_ceiling_us())); + continue; + } } (outcome,events) }; match outcome { Service::Idle => break, - Service::Progress => { trailing = 0; waited_at = u64::MAX; } + Service::Progress => policy.progress(), Service::Waiting(messages) => { - if messages != waited_at { waited_at = messages; trailing = 0; } - if trailing < TRAILING_POLLS { - trailing += 1; - kernel::time::delay::fsleep(kernel::time::Delta::from_micros(5)); - } else { - // Until the next firmware event, or a bounded 100 us resnapshot. - events.wait_past(messages); - trailing = 0; - } + // IRQs and timer expiry alone are not retirement progress. + // Keep backing off until service publishes or retires work. + events.wait_past(messages, policy.next_wait_us(events.wait_ceiling_us())); } } } diff --git a/drivers/gpu/drm/asahi/m3/m3_completion_policy.rs b/drivers/gpu/drm/asahi/m3/m3_completion_policy.rs new file mode 100644 index 0000000000000..9c0da8c3246e3 --- /dev/null +++ b/drivers/gpu/drm/asahi/m3/m3_completion_policy.rs @@ -0,0 +1,77 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Bounded fallback for witnesses which can become visible after the last IRQ. +//! +//! Notifications request a fresh snapshot, but do not prove witness progress. +//! Only publication or retirement resets this policy. In particular, neither a +//! timer wake nor a stream of early firmware notifications renews a spin budget. + +pub(crate) struct WaitPolicy { + next_us: u64, +} + +impl WaitPolicy { + pub(crate) const fn new() -> Self { + Self { next_us: 25 } + } + + pub(crate) fn progress(&mut self) { + self.next_us = 25; + } + + /// No deliberate busy delay: IRQs still wake each sleep immediately. + /// The caller supplies the root-selected ceiling (25..=1000 us). + pub(crate) fn next_wait_us(&mut self, ceiling_us: u64) -> u64 { + let wait = self.next_us.min(ceiling_us.clamp(25, 1000)); + self.next_us = (wait * 2).min(1000); + wait + } +} + +#[cfg_attr(not(test), kernel::macros::kunit_tests(asahi_m3_completion_policy))] +mod tests { + use super::WaitPolicy; + + #[test] + fn no_progress_backs_off_and_stays_bounded() { + let mut policy = WaitPolicy::new(); + for expected in [25, 50, 100, 200, 400, 800, 1000, 1000] { + assert_eq!(policy.next_wait_us(1000), expected); + } + for _ in 0..100_000 { + assert_eq!(policy.next_wait_us(1000), 1000); + } + } + + #[test] + fn publication_or_retirement_restarts_fast_detection() { + let mut policy = WaitPolicy::new(); + for _ in 0..8 { policy.next_wait_us(1000); } + policy.progress(); + assert_eq!(policy.next_wait_us(1000), 25); + assert_eq!(policy.next_wait_us(1000), 50); + } + + #[test] + fn live_ceiling_changes_apply_to_the_next_wait() { + let mut policy = WaitPolicy::new(); + for _ in 0..8 { policy.next_wait_us(1000); } + assert_eq!(policy.next_wait_us(25), 25); + assert_eq!(policy.next_wait_us(1000), 50); + assert_eq!(policy.next_wait_us(100), 100); + assert_eq!(policy.next_wait_us(100), 100); + } + + #[test] + fn fallback_keeps_timeout_checks_frequent_without_any_irq() { + let mut policy = WaitPolicy::new(); + let mut elapsed_us = 0; + while elapsed_us < 2_000_000 { + let wait = policy.next_wait_us(1000); + assert!(wait <= 1000); + elapsed_us += wait; + } + // This bounds the requested delay, not scheduler dispatch latency. + assert!(elapsed_us < 2_001_000); + } +} diff --git a/drivers/gpu/drm/asahi/m3/m3_drm.rs b/drivers/gpu/drm/asahi/m3/m3_drm.rs index 555edf7074a6b..04759396c7757 100644 --- a/drivers/gpu/drm/asahi/m3/m3_drm.rs +++ b/drivers/gpu/drm/asahi/m3/m3_drm.rs @@ -38,7 +38,7 @@ struct CompletionView(Arc); impl kernel::debugfs::Writer for CompletionView { fn write(&self, f: &mut kernel::fmt::Formatter<'_>) -> kernel::fmt::Result { let (us, irqs, sleeps, timers, preparations) = self.0.snapshot(); - writeln!(f, "version=1 wait_us={} notifications={} sleeps={} timer_wakes={} cpu_preparations={}", + writeln!(f, "version=2 policy=adaptive wait_us={} notifications={} sleeps={} timer_wakes={} cpu_preparations={}", us, irqs, sleeps, timers, preparations) } } diff --git a/drivers/gpu/drm/asahi/m3/m3_rtkit.rs b/drivers/gpu/drm/asahi/m3/m3_rtkit.rs index be81c4a4e9d92..ba8ad365ce20a 100644 --- a/drivers/gpu/drm/asahi/m3/m3_rtkit.rs +++ b/drivers/gpu/drm/asahi/m3/m3_rtkit.rs @@ -38,7 +38,7 @@ impl EventWait { Arc::pin_init( pin_init!(EventWait { count: AtomicU64::new(0), - wait_us: AtomicU64::new(100), + wait_us: AtomicU64::new(1000), overlap_cpu: AtomicBool::new(crate::m3::m3_params::unlocked_wait()), cpu_preparations: AtomicU64::new(0), sleeps: AtomicU64::new(0), @@ -55,7 +55,7 @@ impl EventWait { self.arrived.complete(); } - /// Diagnostic tuning changes only the bounded resnapshot cadence, never + /// Diagnostic tuning sets the adaptive fallback ceiling in microseconds, never /// stamps, pipe retirement, event counts, fences or the two-second timeout. /// Atomic publication permits root to compare policies in the same boot. pub(crate) fn set_wait_us(&self, us: u64) -> Result { @@ -64,6 +64,10 @@ impl EventWait { Ok(()) } + pub(crate) fn wait_ceiling_us(&self) -> u64 { + self.wait_us.load(Ordering::Relaxed) + } + pub(crate) fn cpu_overlap(&self) -> bool { self.overlap_cpu.load(Ordering::Relaxed) } @@ -82,11 +86,11 @@ impl EventWait { } /// Sleep until a notification later than `seen` has been recorded or - /// the selected 25..1000 us elapse (default 100 us). A one-jiffy fallback alone takes up to 4 ms at HZ=250, + /// the selected 25..1000 us fallback elapses. A one-jiffy fallback alone takes up to 4 ms at HZ=250, /// even when the GPU stamp arrives just after its notification. The timer /// adds a bounded resnapshot without replacing IRQ wakeups or busy-waiting. /// Only the single completion worker may call this method. - pub(crate) fn wait_past(self: &Arc, seen: u64) -> bool { + pub(crate) fn wait_past(self: &Arc, seen: u64, wait_us: u64) -> bool { // Events can arrive while the worker is polling. Discard stale wakeup // tokens before checking the sequence. A concurrent event either moves // the sequence or leaves a token, including between the check and wait. @@ -98,7 +102,7 @@ impl EventWait { // on every return, before a subsequent wait can arm the timer. self.sleeps.fetch_add(1, Ordering::Relaxed); let timer = self.clone().start(Delta::from_micros( - self.wait_us.load(Ordering::Relaxed) as i64)); + wait_us.clamp(25, 1000) as i64)); // Retain one tick as a backup bound; normally either the real // interrupt or our high-resolution timer supplies the token. self.arrived.wait_for_completion_timeout(1); diff --git a/drivers/gpu/drm/asahi/m3/mod.rs b/drivers/gpu/drm/asahi/m3/mod.rs index d885da1a8647c..eafd5732bceac 100644 --- a/drivers/gpu/drm/asahi/m3/mod.rs +++ b/drivers/gpu/drm/asahi/m3/mod.rs @@ -133,6 +133,8 @@ mod t8122_knobs; #[cfg(CONFIG_DRM_ASAHI_M3_T8122)] mod t8122_start; +mod m3_completion_policy; + mod m3_compute_layout; mod m3_compute_sequence; diff --git a/tools/asahi/m3-fence-latency.py b/tools/asahi/m3-fence-latency.py new file mode 100644 index 0000000000000..c8428d65dbf0d --- /dev/null +++ b/tools/asahi/m3-fence-latency.py @@ -0,0 +1,63 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only OR MIT +"""Join existing M3 timing logs. Input must be one boot, captured without gaps.""" +import collections +import json +import math +import re +import sys + +milestones = {} +packets = collections.defaultdict(list) +fences = [] +invalid = collections.Counter() +for line in open(sys.argv[1]): + fields = {k: int(v, 0) for k, v in re.findall(r'(\w+)=(-?0x[0-9a-fA-F]+|-?\d+)', line)} + if 'M3_BATCH_MILESTONE ' in line: + key = (fields['vm'], fields['publication_tick']) + if key in milestones: + invalid['duplicate_milestone'] += 1 + milestones[key] = fields + elif 'M3_BATCH_PACKET ' in line: + packets[(fields['vm'], fields['packet'])].append((fields['publication_tick'], fields['command'])) + elif 'M3_PACKET_FENCE ' in line: + fences.append(fields) +values = [] +previous_signal = {} +for f in sorted(fences, key=lambda f: f['signal_before_tick']): + identity = (f['vm'], f['packet']) + previous = previous_signal.get(identity, 0) + previous_signal[identity] = f['signal_after_tick'] + if f['status'] != 0: + invalid['error_fence'] += 1 + continue + linked = packets.get((f['vm'], f['packet']), []) + # Pointer reuse is disambiguated by the completed fence's physical counter. + candidates = [(tick, command) for tick, command in linked if previous < tick <= f['signal_before_tick'] and command == f['commands'] - 1] + if not candidates: + invalid['missing_final_command'] += 1 + continue + final_tick, _ = max(candidates) + batch = milestones.get((f['vm'], final_tick)) + if not batch or batch['mixed'] or batch['render_end_tick'] <= 0: + invalid['missing_or_non_render_end'] += 1 + continue + end = batch['render_end_tick'] + before, after = f['signal_before_tick'], f['signal_after_tick'] + if not (final_tick <= end <= before <= after): + invalid['invalid_counter_order'] += 1 + continue + # GPU execution end is earlier than full coherent retirement; this includes + # trailing witnesses and dispatch/retirement, rather than claiming an IRQ is done. + values.append(((before - end) / 24, (after - end) / 24)) + +def percentiles(data): + ordered = sorted(data) + return {str(p): ordered[max(0, math.ceil(len(ordered) * p / 100) - 1)] for p in (50, 95, 99, 100)} if ordered else {} + +print(json.dumps({'render_fences': len(values), 'invalid': invalid, + 'gpu_end_to_signal_before_us': percentiles([a for a, b in values]), + 'gpu_end_to_signal_after_us': percentiles([b for a, b in values]), + 'note': '24 MHz physical counter; interval brackets signal(); includes trailing witnesses. Not presentation latency.'}, indent=2)) +if not values: + sys.exit('No valid render fences: do not report latency from this capture') diff --git a/tools/asahi/m3-runtime-tests.rs b/tools/asahi/m3-runtime-tests.rs index 91cbefd298fe2..0d6d4e5841194 100644 --- a/tools/asahi/m3-runtime-tests.rs +++ b/tools/asahi/m3-runtime-tests.rs @@ -106,3 +106,6 @@ fn extended_parameter_aliases_are_gpu_only_and_legacy_is_unchanged() { } } } + +#[path = "../../drivers/gpu/drm/asahi/m3/m3_completion_policy.rs"] +mod m3_completion_policy; From 9f3b31006dd1ac17b751b73d9feceba550675f48 Mon Sep 17 00:00:00 2001 From: Satya Benson Date: Tue, 3 Nov 2026 17:59:21 -0500 Subject: [PATCH 7/8] drm/asahi: omit no-op M3 module pins without unload support Signed-off-by: Satya Benson --- drivers/gpu/drm/asahi/m3/m3_runtime.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/drivers/gpu/drm/asahi/m3/m3_runtime.rs b/drivers/gpu/drm/asahi/m3/m3_runtime.rs index 3b1f3d6d08535..8e25e2c3c602f 100644 --- a/drivers/gpu/drm/asahi/m3/m3_runtime.rs +++ b/drivers/gpu/drm/asahi/m3/m3_runtime.rs @@ -1079,6 +1079,8 @@ impl Runtime { crate::m3::t8122_start::prepare_verdict(pdev.as_ref(),device.soc(),started,e); state.health.mark_failed(); if device.stop_asc().is_err() { + // Without module unloading, the C helper is an unbound no-op. + #[cfg(CONFIG_MODULE_UNLOAD)] unsafe {kernel::bindings::__module_get(crate::THIS_MODULE.as_ptr())}; core::mem::forget(transport);core::mem::forget(state); core::mem::forget(adopted); @@ -1348,6 +1350,7 @@ impl Drop for Runtime { for batch in self.inner.active.iter() {Inner::finish_all(&batch.entries,Err(ENODEV));} if policy::retain_runtime(self.inner.device.stop_asc().is_err(),self.inner.gpu_pending) { // SAFETY: a live module reference pins callbacks with retained DMA. + #[cfg(CONFIG_MODULE_UNLOAD)] unsafe { kernel::bindings::__module_get(crate::THIS_MODULE.as_ptr()) }; return; } From 40c0f578fa188d83dbe58f0cdd0780aff05696fa Mon Sep 17 00:00:00 2001 From: Satya Benson Date: Wed, 4 Nov 2026 21:13:17 -0500 Subject: [PATCH 8/8] selftests: asahi: match the 7.3 capability argument type Signed-off-by: Satya Benson --- tools/asahi/queue-priority-tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/asahi/queue-priority-tests.py b/tools/asahi/queue-priority-tests.py index cfbbe8f36d4e6..1c5b4695c85f7 100644 --- a/tools/asahi/queue-priority-tests.py +++ b/tools/asahi/queue-priority-tests.py @@ -24,7 +24,7 @@ pub const drm_asahi_priority_DRM_ASAHI_PRIORITY_HIGH:u32=2; } mod bindings { pub const CAP_SYS_NICE:u32=23; - pub unsafe fn capable(_:u32)->bool { CAP.with(|c|c.get()) } + pub unsafe fn capable(_:i32)->bool { CAP.with(|c|c.get()) } thread_local! {pub static CAP:std::cell::Cell=const {std::cell::Cell::new(false)};} } mod g17 {pub struct Gpu;}