From b670fd8de0ec3a550c54714ada3c388f1992c8cf Mon Sep 17 00:00:00 2001 From: Krishna Chaitanya Chundru Date: Thu, 17 Sep 2026 16:31:15 +0530 Subject: [PATCH 1/2] PCI/bwctrl: Add on-demand link speed scaling Add a devfreq-based PCIe Link Speed controller that uses downstream I/O activity to scale the link speed through the simple_ondemand governor. Track submitted and completed request bytes for participating endpoints, aggregate activity across PCIe bridge hierarchies, and account for in-flight requests when calculating link utilization. Protect link retraining with a minimum downscale dwell time and limit downscaling to one supported speed step at a time. Require every endpoint below a port to opt in before changing its link speed, preventing non-participating devices from being affected by bandwidth scaling. Add the Qualcomm PCIe Link Bandwidth Notification Capability bit required for this functionality. Integrate NVMe data-queue activity reporting while excluding admin-queue traffic, so link speed follows actual storage I/O rather than driver housekeeping. Signed-off-by: Krishna Chaitanya Chundru --- drivers/nvme/host/pci.c | 27 ++ drivers/pci/controller/dwc/pcie-qcom.c | 8 + drivers/pci/pcie/Kconfig | 16 + drivers/pci/pcie/bwctrl.c | 548 ++++++++++++++++++++++++- include/linux/pci-bwctrl.h | 39 ++ include/linux/pci.h | 1 + 6 files changed, 636 insertions(+), 3 deletions(-) diff --git a/drivers/nvme/host/pci.c b/drivers/nvme/host/pci.c index 69932d640b537..a6de6d5aa1054 100644 --- a/drivers/nvme/host/pci.c +++ b/drivers/nvme/host/pci.c @@ -21,6 +21,7 @@ #include #include #include +#include #include #include #include @@ -393,6 +394,25 @@ struct nvme_queue { struct completion delete_done; }; +/* + * Report request byte transitions to bwctrl's on-demand PCIe Link Speed + * scaling. Admin-queue traffic (keep-alives, log page/identify, etc.) is + * excluded so link speed tracks actual data I/O, not driver housekeeping. + */ +static void nvme_pci_note_activity(struct nvme_queue *nvmeq, struct request *req, + bool submit) +{ + s64 bytes = blk_rq_bytes(req); + + if (!nvmeq->qid) + return; + + if (!submit) + bytes = -bytes; + + pcie_bwctrl_note_activity(to_pci_dev(nvmeq->dev->dev), bytes); +} + /* bits for iod->flags */ enum nvme_iod_flags { /* this command has been aborted by the timeout handler */ @@ -1457,6 +1477,7 @@ static blk_status_t nvme_queue_rq(struct blk_mq_hw_ctx *hctx, return ret; spin_lock(&nvmeq->sq_lock); nvme_sq_copy_cmd(nvmeq, &iod->cmd); + nvme_pci_note_activity(nvmeq, req, true); nvme_write_sq_db(nvmeq, bd->last); spin_unlock(&nvmeq->sq_lock); return BLK_STS_OK; @@ -1474,6 +1495,7 @@ static void nvme_submit_cmds(struct nvme_queue *nvmeq, struct rq_list *rqlist) struct nvme_iod *iod = blk_mq_rq_to_pdu(req); nvme_sq_copy_cmd(nvmeq, &iod->cmd); + nvme_pci_note_activity(nvmeq, req, true); } nvme_write_sq_db(nvmeq, true); spin_unlock(&nvmeq->sq_lock); @@ -1518,10 +1540,13 @@ static void nvme_queue_rqs(struct rq_list *rqlist) static __always_inline void nvme_pci_unmap_rq(struct request *req) { + struct nvme_queue *nvmeq = req->mq_hctx->driver_data; + if (blk_integrity_rq(req)) nvme_unmap_metadata(req); if (blk_rq_nr_phys_segments(req)) nvme_unmap_data(req); + nvme_pci_note_activity(nvmeq, req, false); } static void nvme_pci_complete_rq(struct request *req) @@ -3753,6 +3778,8 @@ static int nvme_probe(struct pci_dev *pdev, const struct pci_device_id *id) if (IS_ERR(dev)) return PTR_ERR(dev); + pcie_bwctrl_register(pdev); + result = nvme_add_ctrl(&dev->ctrl); if (result) goto out_put_ctrl; diff --git a/drivers/pci/controller/dwc/pcie-qcom.c b/drivers/pci/controller/dwc/pcie-qcom.c index 19daadee65f7c..137a84748bfc6 100644 --- a/drivers/pci/controller/dwc/pcie-qcom.c +++ b/drivers/pci/controller/dwc/pcie-qcom.c @@ -374,6 +374,10 @@ static void qcom_pcie_set_slot_nccs(struct dw_pcie *pci) val |= PCI_EXP_SLTCAP_NCCS; writel(val, pci->dbi_base + offset + PCI_EXP_SLTCAP); + val = readl(pci->dbi_base + offset + PCI_EXP_LNKCAP); + val |= PCI_EXP_LNKCAP_LBNC; + writel(val, pci->dbi_base + offset + PCI_EXP_LNKCAP); + dw_pcie_dbi_ro_wr_dis(pci); } @@ -1290,6 +1294,10 @@ static int qcom_pcie_post_init_2_9_0(struct qcom_pcie *pcie) val &= ~PCI_EXP_LNKCAP_ASPMS; writel(val, pci->dbi_base + offset + PCI_EXP_LNKCAP); + val = readl(pci->dbi_base + offset + PCI_EXP_LNKCAP); + val |= PCI_EXP_LNKCAP_LBNC; + writel(val, pci->dbi_base + offset + PCI_EXP_LNKCAP); + writel(PCI_EXP_DEVCTL2_COMP_TMOUT_DIS, pci->dbi_base + offset + PCI_EXP_DEVCTL2); diff --git a/drivers/pci/pcie/Kconfig b/drivers/pci/pcie/Kconfig index 207c2deae35ff..1faec0d93cb87 100644 --- a/drivers/pci/pcie/Kconfig +++ b/drivers/pci/pcie/Kconfig @@ -146,3 +146,19 @@ config PCIE_EDR the PCI Firmware Specification r3.2. Enable this if you want to support hybrid DPC model which uses both firmware and OS to implement DPC. + +config PCIE_BW_ONDEMAND + bool "PCIe on-demand link speed scaling" + depends on PCIEPORTBUS && PM_DEVFREQ + default y + help + Enable devfreq-based on-demand PCIe Link Speed scaling. Downstream + device drivers report activity via pcie_bwctrl_note_activity(); + bwctrl uses this to scale Link Speed up or down through the + simple_ondemand governor. + + Changing Link Speed requires link retraining, which briefly + interrupts I/O and may affect other devices sharing the same Root + Port/switch. + + If unsure, say N. diff --git a/drivers/pci/pcie/bwctrl.c b/drivers/pci/pcie/bwctrl.c index c4c8d260bf969..a6126c4407f68 100644 --- a/drivers/pci/pcie/bwctrl.c +++ b/drivers/pci/pcie/bwctrl.c @@ -20,16 +20,24 @@ #define dev_fmt(fmt) "bwctrl: " fmt #include +#include #include #include #include +#include #include +#include #include +#include +#include +#include #include #include #include +#include #include #include +#include #include #include "../pci.h" @@ -39,10 +47,44 @@ * struct pcie_bwctrl_data - PCIe bandwidth controller * @set_speed_mutex: Serializes link speed changes * @cdev: Thermal cooling device associated with the port + * @devfreq: On-demand Link Speed scaling devfreq device, if enabled + * @profile: devfreq profile backing @devfreq + * @ondemand_data: simple_ondemand governor tuning backing @devfreq + * @freq_table: Link Speeds usable as devfreq frequencies + * @nr_freqs: Number of valid entries in @freq_table (and matching OPPs) + * @activity_lock: Serializes @bytes_in_flight/@bytes_completed_total/@window_start_t + * @bytes_in_flight: Sum of currently-outstanding downstream request sizes + * @bytes_completed_total: Cumulative bytes completed since registration, never reset + * @window_start_t: Start of the current polling window + * @last_agg_total: Snapshot of the aggregated total as of the last poll + * @last_bytes: Bytes completed in the most recently closed window (for logging) + * @last_elapsed_us: Duration of the most recently closed window (for logging) + * @last_capacity_mbps: Link capacity during the most recently closed window (for logging) + * @last_change_t: Time of the most recent Link Speed change, for the dwell-time guard + * @held_for_optin: Set once "holding, not opted in" has been logged, until it clears */ struct pcie_bwctrl_data { struct mutex set_speed_mutex; struct thermal_cooling_device *cdev; +#ifdef CONFIG_PCIE_BW_ONDEMAND + struct devfreq *devfreq; + struct devfreq_dev_profile profile; + struct devfreq_simple_ondemand_data ondemand_data; + unsigned long freq_table[BITS_PER_TYPE(u8)]; + unsigned int nr_freqs; + + spinlock_t activity_lock; + s64 bytes_in_flight; + u64 bytes_completed_total; + ktime_t window_start_t; + u64 last_agg_total; + + u64 last_bytes; + u64 last_elapsed_us; + u32 last_capacity_mbps; + ktime_t last_change_t; + bool held_for_optin; +#endif }; /* Prevent port removal during Link Speed changes. */ @@ -178,6 +220,491 @@ int pcie_set_target_speed(struct pci_dev *port, enum pci_bus_speed speed_req, return ret; } +EXPORT_SYMBOL_GPL(pcie_set_target_speed); + +#ifdef CONFIG_PCIE_BW_ONDEMAND +static enum pci_bus_speed pcie_bwctrl_cur_speed(struct pci_dev *port) +{ + struct pci_bus *bus = port->subordinate; + + return bus ? bus->cur_bus_speed : PCI_SPEED_UNKNOWN; +} + +/* + * Current negotiated link capacity in Mb/s (encoded bandwidth, i.e. link + * signaling rate reduced by 128b/130b or 8b/10b encoding overhead, times + * negotiated width) -- the real achievable throughput ceiling at the link's + * current speed, not the raw signaling rate. + */ +static u32 pcie_bwctrl_link_capacity_mbps(struct pci_dev *port) +{ + enum pci_bus_speed speed = pcie_bwctrl_cur_speed(port); + u16 lnksta; + u32 width; + int ret; + + ret = pcie_capability_read_word(port, PCI_EXP_LNKSTA, &lnksta); + if (ret != PCIBIOS_SUCCESSFUL) { + pci_err(port, "bwctrl: failed to read LNKSTA for capacity calc: %d\n", + ret); + return 0; + } + + width = FIELD_GET(PCI_EXP_LNKSTA_NLW, lnksta); + + return width * PCIE_SPEED2MBS_ENC(speed); +} + +/* + * Running totals gathered while walking a subtree: @completed_total sums + * cumulative completed bytes (see pcie_bwctrl_note_activity()), and + * @in_flight sums bytes currently outstanding (submitted, not yet + * completed) across every descendant bridge. + */ +struct pcie_bwctrl_agg { + u64 completed_total; + s64 in_flight; +}; + +/* + * Sum bytes_completed_total and bytes_in_flight across every bwctrl-tracked + * bridge found beneath the walked bus. Each endpoint's traffic is credited + * exactly once, at its immediate upstream bridge (see + * pcie_bwctrl_note_activity()), so summing across the whole subtree -- + * however deep -- picks up that credit exactly once without needing + * per-child bookkeeping at every intermediate level. + */ +static int pcie_bwctrl_agg_cb(struct pci_dev *dev, void *userdata) +{ + struct pcie_bwctrl_agg *agg = userdata; + struct pcie_bwctrl_data *cdata; + + if (!pci_is_bridge(dev)) + return 0; + + cdata = READ_ONCE(dev->link_bwctrl); + if (!cdata) + return 0; + + scoped_guard(spinlock_irqsave, &cdata->activity_lock) { + agg->completed_total += cdata->bytes_completed_total; + agg->in_flight += cdata->bytes_in_flight; + } + + return 0; +} + +/* + * Aggregate bytes completed and bytes in flight for @port itself plus + * every descendant bridge reachable from @port's subordinate bus. Runs + * from devfreq_monitor()'s workqueue context (holding devfreq->lock, a + * mutex), so it is safe to sleep here -- pci_walk_bus() takes + * down_read(&pci_bus_sem). + */ +static void pcie_bwctrl_aggregate(struct pci_dev *port, struct pcie_bwctrl_data *data, + struct pcie_bwctrl_agg *agg) +{ + scoped_guard(spinlock_irqsave, &data->activity_lock) { + agg->completed_total = data->bytes_completed_total; + agg->in_flight = data->bytes_in_flight; + } + + if (port->subordinate) + pci_walk_bus(port->subordinate, pcie_bwctrl_agg_cb, agg); +} + +static int pcie_bwctrl_devfreq_get_status(struct device *dev, + struct devfreq_dev_status *stat) +{ + struct pci_dev *port = to_pci_dev(dev); + struct pcie_bwctrl_data *data = port->link_bwctrl; + struct pcie_bwctrl_agg agg; + u64 elapsed_us, bytes, capacity_bytes, credited_in_flight; + u32 capacity_mbps; + ktime_t curr_t; + + memset(stat, 0, sizeof(*stat)); + + if (!data) + return -ENODEV; + + pcie_bwctrl_aggregate(port, data, &agg); + + scoped_guard(spinlock_irqsave, &data->activity_lock) { + curr_t = ktime_get(); + if (!data->window_start_t) { + data->window_start_t = curr_t; + data->last_agg_total = agg.completed_total; + return 0; + } + + elapsed_us = ktime_us_delta(curr_t, data->window_start_t); + bytes = agg.completed_total - data->last_agg_total; + + data->window_start_t = curr_t; + data->last_agg_total = agg.completed_total; + } + + capacity_mbps = pcie_bwctrl_link_capacity_mbps(port); + + stat->current_frequency = pcie_bwctrl_cur_speed(port); + stat->total_time = (u64)capacity_mbps * elapsed_us; + + /* + * A request that stays outstanding across an entire poll window + * (e.g. a large transfer that takes longer than polling_ms to + * complete) contributes nothing to @bytes above until it finally + * completes -- which both hides the link being busy while it's + * still in flight, and produces an oversized spike in whichever + * window the completion happens to land in. Credit each window + * with however much of the link's own capacity is still + * outstanding, capped at that window's real capacity so this can + * never claim more throughput than physically possible. + */ + capacity_bytes = stat->total_time / 8; + credited_in_flight = min_t(u64, max_t(s64, agg.in_flight, 0), capacity_bytes); + + stat->busy_time = min_t(u64, (bytes + credited_in_flight) * 8, stat->total_time); + + scoped_guard(spinlock_irqsave, &data->activity_lock) { + data->last_bytes = bytes; + data->last_elapsed_us = elapsed_us; + data->last_capacity_mbps = capacity_mbps; + } + + return 0; +} + +/* + * Minimum time a newly-reached Link Speed must be held before scaling back + * down. Sustained throughput that sits between two adjacent speeds' + * capacities (over-saturating the lower one, under-using the higher one) + * otherwise makes simple_ondemand flip-flop every poll, and each flip costs + * a real link retrain. Scaling up is never held back by this guard, since + * that would risk starving a real burst. + */ +#define PCIE_BWCTRL_DOWNSCALE_DWELL_MS 1000 + +/* + * Look for any endpoint beneath the walked bus that has not opted into + * bwctrl bandwidth scaling. Bridges are skipped -- only leaf (non-bridge) + * devices are required to opt in, since a bridge itself has no traffic of + * its own to gate on. Returning non-zero stops the walk early. + */ +static int pcie_bwctrl_optin_cb(struct pci_dev *dev, void *userdata) +{ + if (pci_is_bridge(dev)) + return 0; + + return dev->bwctrl_participate ? 0 : 1; +} + +/* + * True only if every endpoint beneath @port's subordinate bus has called + * pcie_bwctrl_register(). A port with no subordinate bus, or an empty one, + * trivially has nothing that could withhold consent. + */ +static bool pcie_bwctrl_all_opted_in(struct pci_dev *port) +{ + int not_opted_in = 0; + + if (port->subordinate) + pci_walk_bus(port->subordinate, pcie_bwctrl_optin_cb, ¬_opted_in); + + return !not_opted_in; +} + +/* + * Clamp a governor-requested scale-DOWN to at most one freq_table step + * below the current speed, so idle-driven decisions retrain one generation + * at a time rather than dropping straight to the slowest supported speed. + * Scale-up is intentionally left unclamped: simple_ondemand's up-scale + * requests DEVFREQ_MAX_FREQ once busy_time crosses upthreshold, and real + * sustained traffic should reach a speed that can actually accommodate it + * immediately, not climb through intermediate generations first. + */ +static enum pci_bus_speed pcie_bwctrl_clamp_step(struct pcie_bwctrl_data *data, + enum pci_bus_speed cur_speed, + enum pci_bus_speed speed_req) +{ + int cur_idx = -1, req_idx = -1; + + if (speed_req >= cur_speed) + return speed_req; + + for (unsigned int i = 0; i < data->nr_freqs; i++) { + if (data->freq_table[i] == cur_speed) + cur_idx = i; + if (data->freq_table[i] == speed_req) + req_idx = i; + } + + /* Have not observed the current speed in the table; do not clamp. */ + if (cur_idx < 0) + return speed_req; + + if (req_idx >= 0 && req_idx < cur_idx - 1) + return data->freq_table[cur_idx - 1]; + + return speed_req; +} + +static int pcie_bwctrl_devfreq_target(struct device *dev, unsigned long *freq, + u32 flags) +{ + struct pci_dev *port = to_pci_dev(dev); + struct pcie_bwctrl_data *data = port->link_bwctrl; + enum pci_bus_speed speed_req = (enum pci_bus_speed)*freq; + enum pci_bus_speed cur_speed = pcie_bwctrl_cur_speed(port); + int ret = 0; + + if (data) + speed_req = pcie_bwctrl_clamp_step(data, cur_speed, speed_req); + + if (data && speed_req < cur_speed) { + s64 since_change_ms; + + scoped_guard(spinlock_irqsave, &data->activity_lock) + since_change_ms = data->last_change_t ? + ktime_ms_delta(ktime_get(), data->last_change_t) : + S64_MAX; + + if (since_change_ms < PCIE_BWCTRL_DOWNSCALE_DWELL_MS) + speed_req = cur_speed; + } + + if (speed_req != cur_speed && data && !pcie_bwctrl_all_opted_in(port)) { + if (!data->held_for_optin) { + pci_dbg(port, + "bwctrl: holding at %u, not every endpoint has opted in\n", + cur_speed); + data->held_for_optin = true; + } + speed_req = cur_speed; + } else if (data) { + data->held_for_optin = false; + } + + if (speed_req != cur_speed) { + u64 last_bytes = 0, last_elapsed_us = 0; + u32 last_capacity_mbps = 0; + + if (data) { + scoped_guard(spinlock_irqsave, &data->activity_lock) { + last_bytes = data->last_bytes; + last_elapsed_us = data->last_elapsed_us; + last_capacity_mbps = data->last_capacity_mbps; + } + } + + pci_dbg(port, + "bwctrl: speed change %u -> %u (last window: %llu bytes / %llu us, capacity %u Mb/s)\n", + cur_speed, speed_req, last_bytes, last_elapsed_us, + last_capacity_mbps); + + ret = pcie_set_target_speed(port, speed_req, true); + if (ret && ret != -EAGAIN) + pci_err(port, "failed to set link speed to %u: %d\n", + speed_req, ret); + if (ret == -EAGAIN) + ret = 0; + + if (data) { + scoped_guard(spinlock_irqsave, &data->activity_lock) + data->last_change_t = ktime_get(); + } + } + + *freq = pcie_bwctrl_cur_speed(port); + + return ret; +} + +static int pcie_bwctrl_devfreq_get_cur_freq(struct device *dev, + unsigned long *freq) +{ + *freq = pcie_bwctrl_cur_speed(to_pci_dev(dev)); + + return 0; +} + +/** + * pcie_bwctrl_note_activity - Report I/O byte transitions to bwctrl + * @pdev: Downstream PCIe device (e.g. an NVMe controller's own pci_dev) + * @bytes: Positive request size at submit, negative (matching) size at completion + * + * May be called from IRQ/atomic context (e.g. NVMe completion handlers), so + * this must not sleep: unlike pcie_set_target_speed(), it does not take + * pcie_bwctrl_setspeed_rwsem. @port->link_bwctrl is only ever cleared (never + * freed) while holding that rwsem for writing, and @data itself is + * devm-allocated against the bwctrl service device, so a lock-free read here + * is safe -- the worst case is observing a stale non-NULL @data pointer + * whose ->devfreq is already NULL, which the check below turns into a no-op. + * + * See the kdoc in for the calling convention. + */ +void pcie_bwctrl_note_activity(struct pci_dev *pdev, s64 bytes) +{ + struct pci_dev *port = pci_upstream_bridge(pdev); + struct pcie_bwctrl_data *data; + + if (!port) + return; + + data = READ_ONCE(port->link_bwctrl); + if (!data) + return; + + guard(spinlock_irqsave)(&data->activity_lock); + + if (!data->devfreq) + return; + + data->bytes_in_flight += bytes; + + if (bytes < 0) + data->bytes_completed_total += -bytes; +} +EXPORT_SYMBOL_GPL(pcie_bwctrl_note_activity); + +/** + * pcie_bwctrl_register - Opt an endpoint into bwctrl bandwidth scaling + * @pdev: Downstream PCIe device (e.g. an NVMe controller's own pci_dev) + * + * See the kdoc in for the calling convention. + */ +void pcie_bwctrl_register(struct pci_dev *pdev) +{ + pdev->bwctrl_participate = 1; +} +EXPORT_SYMBOL_GPL(pcie_bwctrl_register); + +static void pcie_bwctrl_devfreq_init(struct pci_dev *port, struct pcie_bwctrl_data *data) +{ + unsigned long bit, supported_speeds = port->supported_speeds; + unsigned int n = 0, added = 0; + int ret; + + if (hweight8(supported_speeds) <= 1) { + pci_err(port, "bwctrl devfreq: only one supported speed (0x%02lx), skipping\n", + supported_speeds); + return; + } + + /* + * supported_speeds is a PCIe Supported Link Speeds Vector: bit N + * (N >= 1) set means PCIE_SPEED_2_5GT + (N - 1) is supported. + */ + for_each_set_bit(bit, &supported_speeds, BITS_PER_TYPE(u8)) { + if (bit == 0) + continue; + data->freq_table[n++] = PCIE_SPEED_2_5GT + (bit - 1); + } + + pci_err(port, "bwctrl devfreq: supported_speeds=0x%02lx, %u freq_table entries\n", + supported_speeds, n); + + /* + * devfreq_add_device() resolves min/max scaling frequency through + * the OPP framework regardless of profile->freq_table, so an OPP + * must exist for every entry or registration fails with -EINVAL. + */ + for (added = 0; added < n; added++) { + ret = dev_pm_opp_add(&port->dev, data->freq_table[added], 0); + if (ret) { + pci_err(port, "failed to add OPP for speed %lu: %d\n", + data->freq_table[added], ret); + goto err_remove_opps; + } + pci_err(port, "bwctrl devfreq: added OPP for speed %lu\n", + data->freq_table[added]); + } + + data->ondemand_data.upthreshold = 80; + data->ondemand_data.downdifferential = 5; + + data->profile.polling_ms = 100; + data->profile.timer = DEVFREQ_TIMER_DELAYED; + data->profile.target = pcie_bwctrl_devfreq_target; + data->profile.get_dev_status = pcie_bwctrl_devfreq_get_status; + data->profile.get_cur_freq = pcie_bwctrl_devfreq_get_cur_freq; + data->profile.freq_table = data->freq_table; + data->profile.max_state = n; + data->profile.initial_freq = pcie_bwctrl_cur_speed(port); + + pci_err(port, "bwctrl devfreq: registering, initial_freq=%u\n", + data->profile.initial_freq); + + data->devfreq = devfreq_add_device(&port->dev, &data->profile, + DEVFREQ_GOV_SIMPLE_ONDEMAND, + &data->ondemand_data); + if (IS_ERR(data->devfreq)) { + pci_err(port, "failed to register bwctrl devfreq: %ld\n", + PTR_ERR(data->devfreq)); + data->devfreq = NULL; + goto err_remove_opps; + } + + data->nr_freqs = n; + pci_err(port, "bwctrl devfreq: registered successfully\n"); + + return; + +err_remove_opps: + for (unsigned int i = 0; i < added; i++) + dev_pm_opp_remove(&port->dev, data->freq_table[i]); +} + +static void pcie_bwctrl_devfreq_remove(struct pci_dev *port, struct pcie_bwctrl_data *data) +{ + struct devfreq *devfreq; + unsigned int nr_freqs; + + scoped_guard(spinlock_irqsave, &data->activity_lock) { + devfreq = data->devfreq; + data->devfreq = NULL; + nr_freqs = data->nr_freqs; + data->nr_freqs = 0; + } + + if (!devfreq) + return; + + pci_err(port, "bwctrl devfreq: removing\n"); + devfreq_remove_device(devfreq); + + for (unsigned int i = 0; i < nr_freqs; i++) + dev_pm_opp_remove(&port->dev, data->freq_table[i]); +} + +static void pcie_bwctrl_devfreq_suspend(struct pcie_bwctrl_data *data) +{ + if (data->devfreq) + devfreq_suspend_device(data->devfreq); +} + +static void pcie_bwctrl_devfreq_resume(struct pcie_bwctrl_data *data) +{ + if (data->devfreq) + devfreq_resume_device(data->devfreq); +} +#else +static void pcie_bwctrl_devfreq_init(struct pci_dev *port, struct pcie_bwctrl_data *data) +{ +} + +static void pcie_bwctrl_devfreq_remove(struct pci_dev *port, struct pcie_bwctrl_data *data) +{ +} + +static void pcie_bwctrl_devfreq_suspend(struct pcie_bwctrl_data *data) +{ +} + +static void pcie_bwctrl_devfreq_resume(struct pcie_bwctrl_data *data) +{ +} +#endif /* CONFIG_PCIE_BW_ONDEMAND */ static void pcie_bwnotif_enable(struct pcie_device *srv) { @@ -266,13 +793,17 @@ static int pcie_bwnotif_probe(struct pcie_device *srv) if (ret) return ret; +#ifdef CONFIG_PCIE_BW_ONDEMAND + spin_lock_init(&data->activity_lock); +#endif + scoped_guard(rwsem_write, &pcie_bwctrl_setspeed_rwsem) { - port->link_bwctrl = data; + WRITE_ONCE(port->link_bwctrl, data); ret = request_irq(srv->irq, pcie_bwnotif_irq, IRQF_SHARED, "PCIe bwctrl", srv); if (ret) { - port->link_bwctrl = NULL; + WRITE_ONCE(port->link_bwctrl, NULL); return ret; } @@ -286,6 +817,8 @@ static int pcie_bwnotif_probe(struct pcie_device *srv) if (IS_ERR(port->link_bwctrl->cdev)) port->link_bwctrl->cdev = NULL; + pcie_bwctrl_devfreq_init(port, data); + return 0; } @@ -293,6 +826,13 @@ static void pcie_bwnotif_remove(struct pcie_device *srv) { struct pcie_bwctrl_data *data = srv->port->link_bwctrl; + /* + * Stop devfreq polling (synchronously) before anything below tears + * down port->link_bwctrl, so no devfreq callback can ever observe a + * stale or NULL pcie_bwctrl_data. + */ + pcie_bwctrl_devfreq_remove(srv->port, data); + pcie_cooling_device_unregister(data->cdev); scoped_guard(rwsem_write, &pcie_bwctrl_setspeed_rwsem) { @@ -300,12 +840,13 @@ static void pcie_bwnotif_remove(struct pcie_device *srv) free_irq(srv->irq, srv); - srv->port->link_bwctrl = NULL; + WRITE_ONCE(srv->port->link_bwctrl, NULL); } } static int pcie_bwnotif_suspend(struct pcie_device *srv) { + pcie_bwctrl_devfreq_suspend(srv->port->link_bwctrl); pcie_bwnotif_disable(srv->port); return 0; } @@ -313,6 +854,7 @@ static int pcie_bwnotif_suspend(struct pcie_device *srv) static int pcie_bwnotif_resume(struct pcie_device *srv) { pcie_bwnotif_enable(srv); + pcie_bwctrl_devfreq_resume(srv->port->link_bwctrl); return 0; } diff --git a/include/linux/pci-bwctrl.h b/include/linux/pci-bwctrl.h index cee07127455b3..cebbc8bc9a1fa 100644 --- a/include/linux/pci-bwctrl.h +++ b/include/linux/pci-bwctrl.h @@ -25,4 +25,43 @@ static inline void pcie_cooling_device_unregister(struct thermal_cooling_device } #endif +/** + * pcie_bwctrl_register - Opt an endpoint into bwctrl bandwidth scaling + * @pdev: Downstream PCIe device (e.g. an NVMe controller's own pci_dev) + * + * Call once from the endpoint driver's probe(). Every endpoint beneath a + * given ancestor Switch/Root Port must call this before that ancestor's + * Link Speed is ever scaled up or down; a single non-participating + * endpoint (including one whose driver never calls this at all) holds + * every ancestor up to the Root Port at its current speed. Safe to call + * unconditionally; a no-op when CONFIG_PCIE_BW_ONDEMAND is disabled. + */ +#ifdef CONFIG_PCIE_BW_ONDEMAND +void pcie_bwctrl_register(struct pci_dev *pdev); +#else +static inline void pcie_bwctrl_register(struct pci_dev *pdev) +{ +} +#endif + +/** + * pcie_bwctrl_note_activity - Report I/O byte transitions to bwctrl + * @pdev: Downstream PCIe device (e.g. an NVMe controller's own pci_dev) + * @bytes: Positive request size at submit, negative (matching) size at completion + * + * Callers call this once at submit with the request's size, and once at + * completion with the negated size. bwctrl uses the resulting bytes-per-poll + * throughput, together with the devfreq simple_ondemand governor, to scale + * the upstream Root Port's Link Speed up or down. Safe to call + * unconditionally; a no-op when CONFIG_PCIE_BW_ONDEMAND is disabled or the + * port has no bwctrl instance. + */ +#ifdef CONFIG_PCIE_BW_ONDEMAND +void pcie_bwctrl_note_activity(struct pci_dev *pdev, s64 bytes); +#else +static inline void pcie_bwctrl_note_activity(struct pci_dev *pdev, s64 bytes) +{ +} +#endif + #endif diff --git a/include/linux/pci.h b/include/linux/pci.h index 09134d0a559fd..d9fdeb7b1e176 100644 --- a/include/linux/pci.h +++ b/include/linux/pci.h @@ -428,6 +428,7 @@ struct pci_dev { unsigned int clear_retrain_link:1; /* Need to clear Retrain Link bit manually */ unsigned int no_bw_notif:1; /* BW notifications may cause issues */ + unsigned int bwctrl_participate:1; /* Driver opted into bwctrl bandwidth scaling */ unsigned int d3hot_delay; /* D3hot->D0 transition time in ms */ unsigned int d3cold_delay; /* D3cold->D0 transition time in ms */ From 31b048b5896af2f6d34586ff4a2c707d23bbd735 Mon Sep 17 00:00:00 2001 From: Krishna Chaitanya Chundru Date: Wed, 19 Aug 2026 18:55:52 +0530 Subject: [PATCH 2/2] PCI/bwctrl: Set host bridge OPP and optionally disable ASPM around link retraining PCIe host bridge controllers may need their operating point raised before retraining to a higher link speed so that hardware resources (e.g., RPMh votes on Qualcomm platforms) are available at the requested data rate. After retraining, the operating point must be updated to reflect the actual negotiated speed. Add pcie_set_opp() to look up an OPP on the host bridge parent device using a key of (per-lane frequency in kHz, LNKCTL2 Target Link Speed level). Keying by generation rather than total bandwidth lets OPP tables remain width-independent. In pcie_set_target_speed(), call pcie_set_opp() before retraining only when upscaling (speed_req > cur_bus_speed), since only raising the operating point requires pre-staging hardware. After retraining, call pcie_set_opp() unconditionally with the actual cur_bus_speed to settle the votes. Both calls are skipped for downstream ports of PCIe switches, as those are outside the host controller's scope. Some controllers also require ASPM to be disabled around link retraining. Add a disable_aspm_for_retrain flag to pci_host_bridge; when set, pcie_set_target_speed() saves the child device's ASPM state, disables all ASPM link states before retraining, and restores them afterward. Signed-off-by: Krishna Chaitanya Chundru --- drivers/pci/pcie/bwctrl.c | 62 ++++++++++++++++++++++++++++++++++++++- include/linux/pci.h | 1 + 2 files changed, 62 insertions(+), 1 deletion(-) diff --git a/drivers/pci/pcie/bwctrl.c b/drivers/pci/pcie/bwctrl.c index a6126c4407f68..0f4c654be9229 100644 --- a/drivers/pci/pcie/bwctrl.c +++ b/drivers/pci/pcie/bwctrl.c @@ -39,6 +39,7 @@ #include #include #include +#include #include "../pci.h" #include "portdrv.h" @@ -162,6 +163,38 @@ static int pcie_bwctrl_change_speed(struct pci_dev *port, u16 target_speed, bool return pcie_retrain_link(port, use_lt); } +static int pcie_set_opp(struct pci_dev *pdev, struct pci_host_bridge *host, + enum pci_bus_speed speed) +{ + struct device *dev = host->dev.parent; + struct dev_pm_opp_key key = {}; + int ret, freq_mbps, width; + unsigned long freq_kbps; + struct dev_pm_opp *opp; + u16 lnksta; + + pcie_capability_read_word(pdev, PCI_EXP_LNKSTA, &lnksta); + width = FIELD_GET(PCI_EXP_LNKSTA_NLW, lnksta); + + freq_mbps = pcie_dev_speed_mbps(speed); + if (freq_mbps < 0) + return -EINVAL; + + freq_kbps = freq_mbps * KILO; + key.freq = freq_kbps; + key.level = pci_bus_speed2lnkctl2(speed); + key.bw = 0; + opp = dev_pm_opp_find_key_exact(dev, &key, true); + if (!IS_ERR(opp)) { + ret = dev_pm_opp_set_opp(dev, opp); + if (ret) + dev_err(dev, "Failed to set OPP for freq (%lu): %d\n", + freq_kbps * width, ret); + dev_pm_opp_put(opp); + } + return 0; +} + /** * pcie_set_target_speed - Set downstream Link Speed for PCIe Port * @port: PCIe Port @@ -182,9 +215,12 @@ static int pcie_bwctrl_change_speed(struct pci_dev *port, u16 target_speed, bool int pcie_set_target_speed(struct pci_dev *port, enum pci_bus_speed speed_req, bool use_lt) { + struct pci_host_bridge *host = pci_find_host_bridge(port->bus); + bool is_rootbus = pci_is_root_bus(port->bus); struct pci_bus *bus = port->subordinate; + struct pci_dev *child = NULL; + int aspm_state = 0, ret; u16 target_speed; - int ret; if (WARN_ON_ONCE(!pcie_valid_speed(speed_req))) return -EINVAL; @@ -194,6 +230,24 @@ int pcie_set_target_speed(struct pci_dev *port, enum pci_bus_speed speed_req, target_speed = pcie_bwctrl_select_speed(port, speed_req); + /* + * The host bridge driver may need to be scaled for targeted speed + * otherwise link might not come up at requested speed. + */ + if (is_rootbus && host && bus) { + /* Get function 0 of downstream device */ + list_for_each_entry(child, &bus->devices, bus_list) + if (PCI_FUNC(child->devfn) == 0) + break; + + if (child && host->disable_aspm_for_retrain) { + aspm_state = pcie_aspm_enabled(child); + // pci_disable_link_state_locked(child, PCIE_LINK_STATE_ALL); + } + if (speed_req > bus->cur_bus_speed) + pcie_set_opp(port, host, speed_req); + } + scoped_guard(rwsem_read, &pcie_bwctrl_setspeed_rwsem) { struct pcie_bwctrl_data *data = port->link_bwctrl; @@ -218,6 +272,12 @@ int pcie_set_target_speed(struct pci_dev *port, enum pci_bus_speed speed_req, !list_empty(&bus->devices)) ret = -EAGAIN; + if (bus && is_rootbus && host) { + // if (child && host->disable_aspm_for_retrain) + // pci_enable_link_state_locked(child, aspm_state); + pcie_set_opp(port, host, bus->cur_bus_speed); + } + return ret; } EXPORT_SYMBOL_GPL(pcie_set_target_speed); diff --git a/include/linux/pci.h b/include/linux/pci.h index d9fdeb7b1e176..f793f9066a201 100644 --- a/include/linux/pci.h +++ b/include/linux/pci.h @@ -665,6 +665,7 @@ struct pci_host_bridge { unsigned int msi_domain:1; /* Bridge wants MSI domain */ unsigned int broken_l1ss_resume:1; /* Resuming from L1SS during system suspend is broken */ + unsigned int disable_aspm_for_retrain:1; /* Disable ASPM before link retain */ /* Resource alignment requirements */ resource_size_t (*align_resource)(struct pci_dev *dev,