diff --git a/drivers/gpu/drm/asahi/g17/feed.rs b/drivers/gpu/drm/asahi/g17/feed.rs index 760f352046192..bd1228a82e26f 100644 --- a/drivers/gpu/drm/asahi/g17/feed.rs +++ b/drivers/gpu/drm/asahi/g17/feed.rs @@ -44,6 +44,8 @@ struct Detector { busy: AtomicU64, streak: AtomicU32, until: AtomicU64, + /// Until when the GPU counts as the bottleneck (see `Feed::gpu_bound`). + bound_until: AtomicU64, active: AtomicBool, pending: AtomicBool, stopping: AtomicBool, @@ -57,6 +59,7 @@ impl Detector { busy: AtomicU64::new(0), streak: AtomicU32::new(0), until: AtomicU64::new(0), + bound_until: AtomicU64::new(0), active: AtomicBool::new(false), pending: AtomicBool::new(false), stopping: AtomicBool::new(false), @@ -108,17 +111,26 @@ impl Detector { Err(_) => 0, } }; - if streak >= STREAK_ON && idle_sum.saturating_mul(4) >= busy_sum { + if streak < STREAK_ON { + return; + } + if idle_sum.saturating_mul(4) >= busy_sum { self.until.store(now + LINGER_NS, Ordering::Release); if !self.active.swap(true, Ordering::AcqRel) { self.pending.store(true, Ordering::Release); } + } else { + self.bound_until.store(now + LINGER_NS, Ordering::Release); } } fn wanted(&self, now: u64) -> bool { !self.stopping.load(Ordering::Acquire) && now < self.until.load(Ordering::Acquire) } + + fn bound(&self, now: u64) -> bool { + !self.stopping.load(Ordering::Acquire) && now < self.bound_until.load(Ordering::Acquire) + } } #[cfg(CONFIG_CPU_FREQ)] @@ -416,6 +428,17 @@ impl Feed { } } + /// Whether the GPU itself is the bottleneck: passes follow each other with refill gaps + /// under a quarter of their busy time, and the feed does not see host-refill-bound work. + /// Lingers like streaming, so a pause of more than a few milliseconds ends it. + pub(crate) fn gpu_bound(&self) -> bool { + if !cfg!(CONFIG_CPU_FREQ) { + return false; + } + let now = Self::now(); + self.detector.bound(now) && !self.detector.wanted(now) + } + /// Called after a firmware event pass, before queuing the single feed work item. pub(crate) fn take_pending(&self) -> bool { self.detector.pending.swap(false, Ordering::AcqRel) @@ -498,3 +521,51 @@ impl PinnedDrop for Feed { self.shutdown(); } } + +#[kernel::macros::kunit_tests(asahi_g17_feed_detector)] +mod detector_tests { + use super::*; + + /// Notes `passes` passes of `busy` ticks, each `gap` ticks after the previous, 1 ms apart. + fn run(detector: &Detector, passes: u64, busy: u64, gap: u64) -> u64 { + let mut start = 1000; + let mut now = 1_000_000; + for _ in 0..passes { + detector.note(start, start + busy, now); + start += busy + gap; + now += 1_000_000; + } + now + } + + #[test] + fn back_to_back_long_passes_are_gpu_bound() { + let detector = Detector::new(); + // 2 ms passes with 40 us refill gaps, as a deep heavy queue or a heavy row at depth 1. + let now = run(&detector, 16, CLOCK_HZ / 500, CLOCK_HZ / 25_000); + assert!(detector.bound(now)); + assert!(!detector.wanted(now)); + assert!(!detector.bound(now + LINGER_NS)); + } + + #[test] + fn refill_gaps_stream_instead() { + let detector = Detector::new(); + // 20 us passes with 60 us refill gaps: bound by host refills. + let now = run(&detector, 16, CLOCK_HZ / 50_000, CLOCK_HZ / 16_667); + assert!(detector.wanted(now)); + assert!(!detector.bound(now)); + } + + #[test] + fn paced_or_short_runs_are_neither() { + let detector = Detector::new(); + // Each pass starts after a pause of at least the refill limit. + let now = run(&detector, 16, CLOCK_HZ / 500, MAX_REFILL_TICKS); + assert!(!detector.bound(now) && !detector.wanted(now)); + // Fewer back-to-back passes than the streak needs. + let detector = Detector::new(); + let now = run(&detector, u64::from(STREAK_ON) - 1, CLOCK_HZ / 500, 0); + assert!(!detector.bound(now) && !detector.wanted(now)); + } +} diff --git a/drivers/gpu/drm/asahi/g17/mod.rs b/drivers/gpu/drm/asahi/g17/mod.rs index 1bb7a30e8e033..6495ad2d98c13 100644 --- a/drivers/gpu/drm/asahi/g17/mod.rs +++ b/drivers/gpu/drm/asahi/g17/mod.rs @@ -440,6 +440,16 @@ impl Mailbox { fn send(&self, endpoint: u8, message: u64) -> Result { self.with(|rtkit| rtkit.send_message(endpoint, message)) } + + /// A handle on the receive thread's minimum utilization, usable from any context. + /// + /// # Safety + /// + /// The caller must not use the handle after this mailbox is dropped. + unsafe fn rx_util_min_handle(&self) -> rtkit::RxUtilMin { + // SAFETY: The caller's obligation is the handle's. + self.with(|rtkit| unsafe { rtkit.rx_util_min_handle() }) + } } /// A running firmware coprocessor. @@ -1386,7 +1396,10 @@ impl Gpu { secondary, primary, bridge, - port: port::Port::new(power::Demand::new(pdev.as_ref().as_raw())?)?, + port: port::Port::new( + power::Demand::new(pdev.as_ref().as_raw())?, + shared.feed.clone(), + )?, seen_submissions: 0, rings: channel::Rings::new()?, recovery: recovery::State::new(), diff --git a/drivers/gpu/drm/asahi/g17/port.rs b/drivers/gpu/drm/asahi/g17/port.rs index 243fbfce07f9e..8c405c8cfb90f 100644 --- a/drivers/gpu/drm/asahi/g17/port.rs +++ b/drivers/gpu/drm/asahi/g17/port.rs @@ -22,11 +22,20 @@ use core::sync::atomic::{fence, AtomicBool, AtomicU32, AtomicU64, Ordering}; use kernel::{ new_mutex, new_spinlock, prelude::*, + soc::apple::rtkit, sync::{Arc, Mutex, MutexGuard, SetOnce, SpinLock}, }; const PRIORITIES: usize = Policy::PRIORITIES as usize; +/// Minimum utilization of the primary coprocessor's receive thread, which handles every +/// completion, while the GPU itself is the bottleneck (see `Feed::gpu_bound`). Otherwise the +/// thread keeps the scheduler's RT default, which takes its CPU cluster to the top frequency for +/// each completion: the completion may be on the critical path, of a waiter, of an idle GPU or of +/// work bound by host refills. While the GPU is the bottleneck the burst gains nothing, and at the +/// SoC's thermal limit the power it takes comes out of the GPU's clock. +const RX_UTIL_MIN_GPU_BOUND: u32 = 256; + /// One priority class's compute work ring: its slot array and the cursors of its two firmware /// consumers and of the host producer. #[derive(Clone, Copy)] @@ -89,6 +98,12 @@ pub(super) struct Port { rings: SpinLock<[u32; PRIORITIES]>, #[pin] qos: SpinLock, + /// Sets the completion receive thread's minimum utilization, from `feed`'s view of the GPU. + rx: SetOnce, + feed: Arc, + /// The request last made, `None` before the first. + #[pin] + rx_applied: SpinLock>>, /// Serializes every render publication, with or without the device mutex: the tiling /// and fragment work rings have one host producer at a time, and same-VM attachment /// overlaps are ordered by publication order (see hazard.rs). @@ -144,13 +159,19 @@ impl Drop for OuterClaim<'_> { } impl Port { - pub(super) fn new(demand: super::power::Demand) -> Result> { + pub(super) fn new( + demand: super::power::Demand, + feed: Arc, + ) -> Result> { Arc::pin_init( pin_init!(Self { views: SetOnce::new(), regs: SetOnce::new(), rings <- new_spinlock!([0; PRIORITIES], "g17::Port::rings"), qos <- new_spinlock!(qos::Accounting::new(), "g17::Port::qos"), + rx: SetOnce::new(), + feed, + rx_applied <- new_spinlock!(None, "g17::Port::rx_applied"), render <- new_mutex!((), "g17::Port::render"), generation: AtomicU64::new(0), publications: AtomicU64::new(0), @@ -167,7 +188,10 @@ impl Port { /// Called once while the session is built, before any publisher can see the port. pub(super) fn attach(&self, views: Views, regs: crate::regs::G17Resources) -> Result { - if self.views.populate(views) && self.regs.populate(regs) { + // SAFETY: The session's primary coprocessor owns the mailbox and outlives the port, and + // with it every use of the handle. + let rx = unsafe { (*views.mailbox).rx_util_min_handle() }; + if self.views.populate(views) && self.regs.populate(regs) && self.rx.populate(rx) { Ok(()) } else { Err(EBUSY) @@ -393,7 +417,9 @@ impl Port { policy: Policy, ) -> Result { let view = self.qos_view()?; - self.qos.lock().publish(&view, owner, scheduler, policy, qos::clock) + let publication = self.qos.lock().publish(&view, owner, scheduler, policy, qos::clock)?; + self.rx_update(); + Ok(publication) } /// Publishes the two owners of a render command, fragment then tiling. @@ -404,7 +430,10 @@ impl Port { policy: Policy, ) -> Result<[qos::Publication; 2]> { let view = self.qos_view()?; - self.qos.lock().publish_owners(&view, owners, scheduler, policy, qos::clock) + let publications = + self.qos.lock().publish_owners(&view, owners, scheduler, policy, qos::clock)?; + self.rx_update(); + Ok(publications) } pub(super) fn qos_cancel(&self, publication: qos::Publication) -> Result { @@ -414,7 +443,24 @@ impl Port { pub(super) fn qos_complete(&self, owners: &[qos::Owner]) -> Result { let view = self.qos_view()?; - self.qos.lock().complete(&view, owners) + self.qos.lock().complete(&view, owners)?; + self.rx_update(); + Ok(()) + } + + /// Requests the receive thread's minimum utilization for the feed's current view of the GPU. + /// Called at every publication and completion: the feed's state lingers for a few + /// milliseconds, so after a pause the publication that ends it restores the default before + /// its completion arrives. + fn rx_update(&self) { + let Some(rx) = self.rx.as_ref() else { + return; + }; + let want = self.feed.gpu_bound().then_some(RX_UTIL_MIN_GPU_BOUND); + let mut applied = self.rx_applied.lock(); + if *applied != Some(want) && rx.set(want).is_ok() { + *applied = Some(want); + } } /// Rings the primary coprocessor's doorbell. diff --git a/drivers/soc/apple/mailbox-g17-test.h b/drivers/soc/apple/mailbox-g17-test.h index 33365864f167a..10690d1d27bf7 100644 --- a/drivers/soc/apple/mailbox-g17-test.h +++ b/drivers/soc/apple/mailbox-g17-test.h @@ -54,9 +54,49 @@ static void apple_g17_tx_stall_bounded_test(struct kunit *test) KUNIT_EXPECT_EQ(test, apple_mbox_send(&mbox, msg, false), -ENODEV); } +/* The receive thread's minimum utilization: request validation, and applying + * it to an RT thread (the test thread stands in) without losing its policy. + */ +static void apple_g17_rx_util_min_test(struct kunit *test) +{ + struct apple_mbox mbox = { + .rx_util_min = -1, .rx_util_min_applied = -1, + }; + + KUNIT_EXPECT_EQ(test, apple_mbox_set_rx_util_min(&mbox, 0), -EOPNOTSUPP); + mbox.ascwrap_v6.role = APPLE_MBOX_ASCWRAP_V6_ROLE_GFX; + if (!IS_ENABLED(CONFIG_UCLAMP_TASK)) { + KUNIT_EXPECT_EQ(test, apple_mbox_set_rx_util_min(&mbox, 0), -EOPNOTSUPP); + kunit_skip(test, "needs CONFIG_UCLAMP_TASK"); + } + KUNIT_EXPECT_EQ(test, apple_mbox_set_rx_util_min(&mbox, -2), -EINVAL); + KUNIT_EXPECT_EQ(test, apple_mbox_set_rx_util_min(&mbox, SCHED_CAPACITY_SCALE + 1), -EINVAL); + KUNIT_EXPECT_EQ(test, mbox.rx_util_min, -1); +#ifdef CONFIG_UCLAMP_TASK + sched_set_fifo(current); + KUNIT_ASSERT_EQ(test, apple_mbox_set_rx_util_min(&mbox, 256), 0); + apple_mbox_apply_rx_util_min(&mbox); + KUNIT_EXPECT_EQ(test, mbox.rx_util_min_applied, 256); + KUNIT_EXPECT_EQ(test, current->policy, SCHED_FIFO); + KUNIT_EXPECT_EQ(test, current->rt_priority, MAX_RT_PRIO / 2); + KUNIT_EXPECT_EQ(test, (unsigned int)current->uclamp_req[UCLAMP_MIN].value, 256U); + KUNIT_EXPECT_TRUE(test, current->uclamp_req[UCLAMP_MIN].user_defined); + + /* -1 hands the clamp back to the scheduler's RT default. */ + KUNIT_ASSERT_EQ(test, apple_mbox_set_rx_util_min(&mbox, -1), 0); + apple_mbox_apply_rx_util_min(&mbox); + KUNIT_EXPECT_EQ(test, mbox.rx_util_min_applied, -1); + KUNIT_EXPECT_EQ(test, current->policy, SCHED_FIFO); + KUNIT_EXPECT_EQ(test, current->rt_priority, MAX_RT_PRIO / 2); + KUNIT_EXPECT_FALSE(test, current->uclamp_req[UCLAMP_MIN].user_defined); + sched_set_normal(current, 0); +#endif +} + static struct kunit_case apple_g17_stall_cases[] = { KUNIT_CASE(apple_g17_rx_flood_bounded_test), KUNIT_CASE(apple_g17_tx_stall_bounded_test), + KUNIT_CASE(apple_g17_rx_util_min_test), { } }; static struct kunit_suite apple_g17_stall_suite = { diff --git a/drivers/soc/apple/mailbox.c b/drivers/soc/apple/mailbox.c index a85e5b585f319..bc327c8f91e3a 100644 --- a/drivers/soc/apple/mailbox.c +++ b/drivers/soc/apple/mailbox.c @@ -29,9 +29,11 @@ #include #include #include +#include #include #include #include +#include #define APPLE_ASC_MBOX_CONTROL_FULL BIT(16) #define APPLE_ASC_MBOX_CONTROL_EMPTY BIT(17) @@ -451,12 +453,49 @@ static int apple_mbox_poll_locked(struct apple_mbox *mbox) return ret; } +/* + * Threaded receive handler only: apply the consumer's minimum utilization to + * this thread. As an RT thread it otherwise runs with the RT default, which + * makes schedutil raise its CPU cluster to the top frequency on every message. + */ +static void apple_mbox_apply_rx_util_min(struct apple_mbox *mbox) +{ + int util_min = READ_ONCE(mbox->rx_util_min); + struct sched_attr attr = { + .sched_policy = current->policy, + .sched_priority = current->rt_priority, + .sched_flags = SCHED_FLAG_KEEP_ALL | SCHED_FLAG_UTIL_CLAMP_MIN, + }; + + if (util_min == mbox->rx_util_min_applied) + return; + /* -1 resets the clamp to the default, which follows the RT sysctl. */ + attr.sched_util_min = util_min; + if (!sched_setattr_nocheck(current, &attr)) + mbox->rx_util_min_applied = util_min; +} + +int apple_mbox_set_rx_util_min(struct apple_mbox *mbox, int util_min) +{ + if (!IS_ENABLED(CONFIG_UCLAMP_TASK) || !apple_mbox_is_g17(mbox)) + return -EOPNOTSUPP; + if (util_min < -1 || util_min > SCHED_CAPACITY_SCALE) + return -EINVAL; + WRITE_ONCE(mbox->rx_util_min, util_min); + return 0; +} +EXPORT_SYMBOL_GPL(apple_mbox_set_rx_util_min); + static irqreturn_t apple_mbox_recv_irq(int irq, void *data) { struct apple_mbox *mbox = data; unsigned long flags; int consumed = 0; + /* Only the G17 receive handler is threaded (see apple_mbox_probe()). */ + if (apple_mbox_is_g17(mbox)) + apple_mbox_apply_rx_util_min(mbox); + spin_lock_irqsave(&mbox->rx_lock, flags); if (likely(!READ_ONCE(mbox->removing))) consumed = apple_mbox_poll_locked(mbox); @@ -784,6 +823,8 @@ static int apple_mbox_probe(struct platform_device *pdev) if (mbox->irq_send_empty < 0 && mbox->irq_send_empty != -ENXIO) return mbox->irq_send_empty; + mbox->rx_util_min = -1; + mbox->rx_util_min_applied = -1; spin_lock_init(&mbox->rx_lock); spin_lock_init(&mbox->tx_lock); mutex_init(&mbox->lifecycle_lock); diff --git a/drivers/soc/apple/rtkit.c b/drivers/soc/apple/rtkit.c index a84c1bdc34a63..0107908a2e3ff 100644 --- a/drivers/soc/apple/rtkit.c +++ b/drivers/soc/apple/rtkit.c @@ -1147,6 +1147,12 @@ void apple_rtkit_resume_rx(struct apple_rtkit *rtk) } EXPORT_SYMBOL_GPL(apple_rtkit_resume_rx); +int apple_rtkit_set_rx_util_min(struct apple_rtkit *rtk, int util_min) +{ + return apple_mbox_set_rx_util_min(rtk->mbox, util_min); +} +EXPORT_SYMBOL_GPL(apple_rtkit_set_rx_util_min); + void apple_rtkit_flush_rx(struct apple_rtkit *rtk) { flush_workqueue(rtk->wq); diff --git a/include/linux/soc/apple/mailbox.h b/include/linux/soc/apple/mailbox.h index fcb17eaf2d5a3..b5b723f0a215e 100644 --- a/include/linux/soc/apple/mailbox.h +++ b/include/linux/soc/apple/mailbox.h @@ -77,6 +77,14 @@ struct apple_mbox { void (*rx)(struct apple_mbox *mbox, struct apple_mbox_msg msg, void *cookie); void *cookie; + /* + * Minimum utilization (uclamp_min) a consumer asked the threaded receive + * handler to run with, and the value it last applied to itself; -1 is the + * scheduler's default. See apple_mbox_set_rx_util_min(). + */ + int rx_util_min; + int rx_util_min_applied; + /* * Protected by tx_lock: set while irq_send_empty is unmasked, so that * the IRQ handler, a failed send and provider removal mask it exactly @@ -97,6 +105,12 @@ int apple_mbox_poll(struct apple_mbox *mbox); */ void apple_mbox_hold_rx(struct apple_mbox *mbox); void apple_mbox_release_rx(struct apple_mbox *mbox); +/* + * Set the minimum utilization (0..1024) the threaded receive handler runs with, + * or -1 to restore the scheduler's default; the handler applies it to itself + * when it next runs. Only mailboxes with a threaded receive handler support it. + */ +int apple_mbox_set_rx_util_min(struct apple_mbox *mbox, int util_min); int apple_mbox_send(struct apple_mbox *mbox, struct apple_mbox_msg msg, bool atomic); diff --git a/include/linux/soc/apple/rtkit.h b/include/linux/soc/apple/rtkit.h index 85a4a10f0dbb3..65bd28a6f0240 100644 --- a/include/linux/soc/apple/rtkit.h +++ b/include/linux/soc/apple/rtkit.h @@ -278,6 +278,12 @@ int apple_rtkit_poll(struct apple_rtkit *rtk); void apple_rtkit_hold_rx(struct apple_rtkit *rtk); void apple_rtkit_resume_rx(struct apple_rtkit *rtk); +/* + * Set the minimum utilization (0..1024, or -1 for the scheduler's default) of + * the coprocessor's receive thread; see apple_mbox_set_rx_util_min(). + */ +int apple_rtkit_set_rx_util_min(struct apple_rtkit *rtk, int util_min); + /* * Wait until the messages already received have been handled, including the * recv_message, crashed and shmem_setup callbacks the worker thread ran for diff --git a/rust/kernel/soc/apple/rtkit.rs b/rust/kernel/soc/apple/rtkit.rs index 16946dacfaa1d..95c57d2736774 100644 --- a/rust/kernel/soc/apple/rtkit.rs +++ b/rust/kernel/soc/apple/rtkit.rs @@ -353,6 +353,57 @@ impl RtKit { pub fn has_endpoint(self: Pin<&mut Self>, endpoint: u8) -> bool { unsafe { bindings::apple_rtkit_has_endpoint(self.rtk, endpoint) } } + + /// Sets the minimum utilization (0..=1024) the coprocessor's receive thread runs with, or + /// restores the scheduler's default with `None`. + /// + /// The receive thread applies it to itself when it next runs. Returns `EOPNOTSUPP` for a + /// mailbox without a threaded receive handler. + pub fn set_rx_util_min(&self, util_min: Option) -> Result { + // INVARIANT: `rtk` is valid per the type invariant. + RxUtilMin { rtk: self.rtk }.set(util_min) + } + + /// Returns a handle that sets the receive thread's minimum utilization (see + /// [`RtKit::set_rx_util_min`]) without borrowing this [`RtKit`], from any context. + /// + /// # Safety + /// + /// The caller must not use the handle after this [`RtKit`] is dropped. + pub unsafe fn rx_util_min_handle(&self) -> RxUtilMin { + // INVARIANT: The caller keeps this `RtKit`, and so `rtk`, alive while using the handle. + RxUtilMin { rtk: self.rtk } + } +} + +/// Sets the minimum utilization of an RTKit coprocessor's receive thread. +/// +/// The request is a single store that the receive thread applies to itself when it next runs, +/// so it may be made from any context. +/// +/// # Invariants +/// +/// `rtk` is valid while the handle is used. +#[derive(Clone, Copy)] +pub struct RxUtilMin { + rtk: *mut bindings::apple_rtkit, +} + +// SAFETY: The request is one store into the mailbox, valid from any thread. +unsafe impl Send for RxUtilMin {} +// SAFETY: As above; the handle has no state of its own. +unsafe impl Sync for RxUtilMin {} + +impl RxUtilMin { + /// See [`RtKit::set_rx_util_min`]. + pub fn set(&self, util_min: Option) -> Result { + let util_min = match util_min { + Some(v) => i32::try_from(v).map_err(|_| EINVAL)?, + None => -1, + }; + // SAFETY: `rtk` is valid per the type invariant; the call only stores the request. + to_result(unsafe { bindings::apple_rtkit_set_rx_util_min(self.rtk, util_min) }) + } } // SAFETY: `RtKit` operations require a mutable reference