drm/asahi: kill G17 contexts left with only executed render work

drivers/gpu/drm/asahi/g17/queue/compute/exit.rs

@@ -111,6 +111,10 @@ impl Queue {
                     CANCELLED_DELAY_NS
                 }
     }
+    /// A kill of `context` was published through this queue.
+    pub(crate) fn exit_published(&self, context: &Arc<Context>) -> bool {
+        self.exit.is_some() && self.context().is_some_and(|bound| Arc::ptr_eq(bound, context))
+    }
     pub(crate) fn exit_owner(&self, context: &Arc<Context>) -> Option<u64> {
         if !self
             .active

drivers/gpu/drm/asahi/g17/runtime.rs

@@ -2380,6 +2380,8 @@ impl Registry {
         let waits = || self.compute.iter().flatten().filter_map(|entry| entry.queue.as_deref())
             .filter_map(|queue| queue.kill_waiting(now));
         compute_exit::write_kill_waits(f, waits().count(), waits().max().unwrap_or(0))?;
+        compute_exit::write_render_kills(f, self.teardown.kill_waits(now).count(),
+                                         self.teardown.kill_waits(now).max().unwrap_or(0))?;
         writeln!(f, "compute_slots used={} total={} reusable={} building={}",
                  self.compute.iter().flatten().count(), self.compute.len(),
                  self.compute.iter().flatten().filter_map(|entry| entry.queue.as_deref()).filter(|queue| queue.reusable()).count(),

drivers/gpu/drm/asahi/g17/runtime/compute_exit.rs

@@ -31,6 +31,47 @@ static PRESSED_KILLS: AtomicU32 = AtomicU32::new(0);
 static PRESSED_KILL_NS: AtomicU64 = AtomicU64::new(0);
 const PRESSED_KILL_INTERVAL_NS: u64 = 250_000_000;
 
+/// Render-only kills (see `Firmware::publish_render_kill`): published, acknowledged, and
+/// contexts refused (each counted once, by its first reason) because the scheduler page names
+/// a queue outside the context, because the held pairs have no single owner, or because
+/// admission could not hold them.
+static RENDER_KILLS: [AtomicU32; 5] = [const { AtomicU32::new(0) }; 5];
+const RENDER_KILL_PUBLISHED: usize = 0;
+const RENDER_KILL_ACKNOWLEDGED: usize = 1;
+const RENDER_KILL_SCOPE: usize = 2;
+const RENDER_KILL_OWNER: usize = 3;
+const RENDER_KILL_HOLD: usize = 4;
+/// A render-only candidate is left this long after close, as a quarantined compute queue is,
+/// so that ordinary retirement goes first.
+const RENDER_KILL_DELAY_NS: u64 = 250_000_000;
+
+/// The render queue a render-only kill names: one of the context's own render queues that
+/// its scheduler page lists, never the reserved queues 0 and 1.
+fn render_kill_target(listed: u128, own: u128) -> Option<u8> {
+    let candidates = listed & own & !0b11;
+    (candidates != 0).then(|| candidates.trailing_zeros() as u8)
+}
+
+/// The one admission owner of every pair a kill holds, if there is one.
+fn hold_owner(owners: impl Iterator<Item = Option<u64>>) -> Option<u64> {
+    let mut owner = None;
+    for slot_owner in owners {
+        match (owner, slot_owner) {
+            (_, None) | (_, Some(0)) => return None,
+            (None, Some(next)) => owner = Some(next),
+            (Some(current), Some(next)) if current != next => return None,
+            _ => {}
+        }
+    }
+    owner
+}
+
+fn refuse_render_kill(teardown: &mut crate::g17::teardown::Pending, index: usize, reason: usize) {
+    if teardown.refuse_render_kill(index) {
+        RENDER_KILLS[reason].fetch_add(1, Ordering::Relaxed);
+    }
+}
+
 /// GPU memory is within an eighth of the device limit. A failed context holds its memory
 /// until its kill is acknowledged, so kills waiting for other contexts' compute to pause can
 /// push the device to its limit, and then every client's allocations fail: on Neo,
@@ -63,6 +104,15 @@ pub(in crate::g17) fn write_kill_waits(f: &mut kernel::fmt::Formatter<'_>, waiti
     writeln!(f, "failed_kills waiting={} longest_wait_ms={}", waiting, longest / 1_000_000)
 }
 
+pub(in crate::g17) fn write_render_kills(f: &mut kernel::fmt::Formatter<'_>, waiting: usize, longest: u64)
+    -> kernel::fmt::Result
+{
+    let count = |index: usize| RENDER_KILLS[index].load(Ordering::Relaxed);
+    writeln!(f, "render_kills published={} acknowledged={} refused_scope={} refused_owner={} refused_hold={} waiting={} longest_wait_ms={}",
+             count(RENDER_KILL_PUBLISHED), count(RENDER_KILL_ACKNOWLEDGED), count(RENDER_KILL_SCOPE),
+             count(RENDER_KILL_OWNER), count(RENDER_KILL_HOLD), waiting, longest / 1_000_000)
+}
+
 impl super::Registry {
     /// ACK storage belongs to the already-retained transaction, so a full event
     /// batch cannot overflow a separate cache. No allocation or wait is needed.
@@ -74,6 +124,10 @@ impl super::Registry {
                 }
             }
         }
+        if self.teardown.acknowledge_render_kill(cookie) {
+            RENDER_KILLS[RENDER_KILL_ACKNOWLEDGED].fetch_add(1, Ordering::Relaxed);
+            crate::g17::trace::teardown(4, cookie, 0x100, 0);
+        }
     }
 }
 
@@ -101,6 +155,8 @@ impl Firmware {
                 let _ = queue.announce_exit(now, || self.primary.notify(MSG_CONTROL_NOTIFY));
             }
         }
+        let primary = &mut self.primary;
+        self.queues.teardown.announce_render_kills(|| primary.notify(MSG_CONTROL_NOTIFY));
         // Under memory pressure one kill skips the wait, at most one per interval.
         let last = PRESSED_KILL_NS.load(Ordering::Relaxed);
         let mut pressed =
@@ -109,14 +165,24 @@ impl Firmware {
             let Some((context, age)) = self.queues.teardown.closed(index, now) else {
                 continue;
             };
-            // A failed context's executed pairs keep their pool leases until the kill is
-            // acknowledged (`Pair::firmware_drained`), so its kill cannot wait for them.
-            if !self.queues.render.teardown_idle(&context)?
-                || (!self.compute_failed(&context) && !self.queues.render.teardown_pools_released(&context))
-            {
+            if !self.queues.render.teardown_idle(&context)? {
+                self.queues.teardown.clear_kill_wait(index);
                 continue;
             }
-            if let Err(error) = self.publish_compute_exit(&context, age, now, &mut pressed) {
+            // A failed context's executed pairs keep their pool leases until the kill is
+            // acknowledged (`Pair::firmware_drained`), so its kill cannot wait for them.
+            let result = if self.compute_failed(&context) || self.queues.render.teardown_pools_released(&context) {
+                // Only a render-only kill's wait lives on the entry; a compute kill keeps its
+                // own on the queue.
+                self.queues.teardown.clear_kill_wait(index);
+                self.publish_compute_exit(&context, age, now, &mut pressed)
+            } else if self.render_kill_candidate(index, &context, age) {
+                self.publish_render_kill(index, &context, now, &mut pressed)
+            } else {
+                self.queues.teardown.clear_kill_wait(index);
+                Ok(())
+            };
+            if let Err(error) = result {
                 dev_err!(
                     self.primary.state.shared.dev.as_ref(),
                     "Scheduler {} exit request failed: {:?}\n",
@@ -243,6 +309,102 @@ impl Firmware {
         Ok(())
     }
 
+    /// A closed context whose compute side is clean, with render pairs whose commands all ran
+    /// but were never stamped (`Pair::executed_for_kill`). Its failed compute command was
+    /// retired on the firmware's fault record, so no compute queue is left to carry a kill,
+    /// yet those pairs are released only after one: without this the context, its pairs and
+    /// its memory stay until the processors stop.
+    fn render_kill_candidate(&self, index: usize, context: &Arc<Context>, age: u64) -> bool {
+        age >= RENDER_KILL_DELAY_NS
+            && !self.queues.teardown.render_kill_published(index)
+            && self.queues.render.executed_slots(context) != 0
+            && self.queues.compute.iter().flatten().filter_map(|entry| entry.queue.as_deref()).all(|queue| {
+                queue.idle_for_context(context) && !queue.switching_from(context) && !queue.exit_published(context)
+            })
+    }
+
+    /// Kill a render-only candidate (`render_kill_candidate`). The rules of a failed
+    /// context's kill apply: the pairs are held for the kill and their memory is released only
+    /// once the scheduler page reports it, the kill waits for a moment without other contexts'
+    /// compute unless GPU memory is pressed, and it names only queues of this context.
+    fn publish_render_kill(&mut self, index: usize, context: &Arc<Context>, now: u64, pressed: &mut bool) -> Result {
+        if self.render_control_backpressured() {
+            return Ok(());
+        }
+        let beside = self.foreign_compute_running(context, now);
+        if beside && !*pressed {
+            let (waited, crossed) = self.queues.teardown.note_kill_wait(index, now, SLOW_KILL_WAIT_NS);
+            let last = SLOW_KILL_WARNED_NS.load(Ordering::Relaxed);
+            if crossed
+                && (last == 0 || now.saturating_sub(last) >= SLOW_KILL_WAIT_NS)
+                && SLOW_KILL_WARNED_NS
+                    .compare_exchange(last, now.max(1), Ordering::Relaxed, Ordering::Relaxed)
+                    .is_ok()
+            {
+                dev_warn!(
+                    self.primary.state.shared.dev.as_ref(),
+                    "Scheduler {} kill has waited {} ms for other contexts' compute to pause\n",
+                    context.id(),
+                    waited / 1_000_000
+                );
+            }
+            return Ok(());
+        }
+        let (mut own, mut foreign, slots) = self.queues.render.exit_scope(context);
+        for entry in self.queues.compute.iter().flatten() {
+            if let Some(queue) = entry.queue.as_deref() {
+                let bit = 1u128 << queue.qid();
+                if queue.context().is_some_and(|bound| Arc::ptr_eq(bound, context)) {
+                    own |= bit;
+                } else {
+                    foreign |= bit;
+                }
+            }
+        }
+        let state = exit::State::read(context)?;
+        let Some(qids) = render_kill_target(state.qids, own).and_then(|target| state.scope(target, own, foreign)) else {
+            refuse_render_kill(&mut self.queues.teardown, index, RENDER_KILL_SCOPE);
+            return Ok(());
+        };
+        let held = |slot: u8| slots & (1u64 << slot) != 0 && self.queues.render.qids_for_slot(slot) & qids != 0;
+        let range = 0..crate::hw::t8140::queues::RENDER_SLOTS as u8;
+        let hold = range.clone().filter(|slot| held(*slot)).fold(0u64, |hold, slot| hold | 1u64 << slot);
+        let owner = hold_owner(range.filter(|slot| held(*slot)).map(|slot| self.queues.render.slot_owner(slot)));
+        let Some(owner) = owner.filter(|_| hold != 0) else {
+            refuse_render_kill(&mut self.queues.teardown, index, RENDER_KILL_OWNER);
+            return Ok(());
+        };
+        if !self
+            .queues
+            .admission
+            .hold_render_slots_for_exit_kill(owner, hold, self.queues.render.executed_slots(context) & hold)
+        {
+            refuse_render_kill(&mut self.queues.teardown, index, RENDER_KILL_HOLD);
+            return Ok(());
+        }
+        let cookie = exit::next_cookie()?;
+        let record = ControlRecord::ContextKill(ContextKill::new(context.scheduler_va(), cookie));
+        if let Err(error) = Rings::publish_control(&self.init, &record) {
+            self.queues.admission.release_exit_kill_hold(hold);
+            return if error == EAGAIN { Ok(()) } else { Err(error) };
+        }
+        // Publication is irrevocable. The held pairs stay held until this context's
+        // scheduler release is consumed (`release_killed_slots` at detach).
+        self.queues.teardown.record_render_kill(index, cookie, now)?;
+        record_kill_wait(self.queues.teardown.take_kill_wait(index, now));
+        if beside {
+            *pressed = false;
+            PRESSED_KILLS.fetch_add(1, Ordering::Relaxed);
+            PRESSED_KILL_NS.store(now.max(1), Ordering::Relaxed);
+        }
+        RENDER_KILLS[RENDER_KILL_PUBLISHED].fetch_add(1, Ordering::Relaxed);
+        crate::g17::trace::teardown(7, cookie, u64::from(context.id()), hold | u64::from(beside));
+        fence(Ordering::SeqCst);
+        let primary = &mut self.primary;
+        self.queues.teardown.announce_render_kills(|| primary.notify(MSG_CONTROL_NOTIFY));
+        Ok(())
+    }
+
     /// Another context's compute command is in flight. On Neo, kills of failed contexts beside
     /// other clients' compute occasionally made the firmware recover the GPU (twice in ten
     /// minutes of a five-lane load with 67 faults), even with nothing freed before the kill's
@@ -304,6 +466,9 @@ impl Firmware {
         {
             return true;
         }
+        if self.queues.teardown.render_kill_polling(now, exit::POLL_NS) {
+            return true;
+        }
         (0..self.queues.teardown.len()).any(|index| {
             self.queues
                 .teardown
@@ -321,3 +486,27 @@ impl Firmware {
         })
     }
 }
+
+#[kernel::macros::kunit_tests(asahi_g17_render_kill_target)]
+mod render_kill_target_tests {
+    use super::{hold_owner, render_kill_target};
+
+    #[test]
+    fn a_render_kill_holds_pairs_of_one_owner_only() {
+        assert_eq!(hold_owner([Some(5), Some(5)].into_iter()), Some(5));
+        assert_eq!(hold_owner([Some(5), Some(6)].into_iter()), None);
+        assert_eq!(hold_owner([Some(5), None].into_iter()), None);
+        assert_eq!(hold_owner([Some(0)].into_iter()), None);
+        assert_eq!(hold_owner(core::iter::empty()), None);
+    }
+
+    #[test]
+    fn a_render_kill_names_an_own_listed_queue_never_0_or_1() {
+        // Listed by the scheduler page but not the context's own: no target.
+        assert_eq!(render_kill_target(1 << 40, 1 << 41), None);
+        // The reserved queues are never named.
+        assert_eq!(render_kill_target(0b11, 0b11), None);
+        assert_eq!(render_kill_target(0b111 | 1 << 40, 0b111 | 1 << 40), Some(2));
+        assert_eq!(render_kill_target(1 << 40 | 1 << 90, 1 << 90), Some(90));
+    }
+}

drivers/gpu/drm/asahi/g17/runtime/render/exit.rs

@@ -39,6 +39,14 @@ impl RenderState {
         slots
     }
 
+    /// The admission owner whose identities the pair in `slot` was built with.
+    pub(in crate::g17::runtime) fn slot_owner(&self, slot: u8) -> Option<u64> {
+        self.entries
+            .get(usize::from(slot))
+            .and_then(Option::as_ref)
+            .map(|entry| entry.reservation.owner)
+    }
+
     pub(in crate::g17::runtime) fn qids_for_slot(&self, slot: u8) -> u128 {
         self.entries
             .get(usize::from(slot))