diff --git a/drivers/nvme/host/pci.c b/drivers/nvme/host/pci.c index 69932d640b537..a6de6d5aa1054 100644 --- a/drivers/nvme/host/pci.c +++ b/drivers/nvme/host/pci.c @@ -21,6 +21,7 @@ #include #include #include +#include #include #include #include @@ -393,6 +394,25 @@ struct nvme_queue { struct completion delete_done; }; +/* + * Report request byte transitions to bwctrl's on-demand PCIe Link Speed + * scaling. Admin-queue traffic (keep-alives, log page/identify, etc.) is + * excluded so link speed tracks actual data I/O, not driver housekeeping. + */ +static void nvme_pci_note_activity(struct nvme_queue *nvmeq, struct request *req, + bool submit) +{ + s64 bytes = blk_rq_bytes(req); + + if (!nvmeq->qid) + return; + + if (!submit) + bytes = -bytes; + + pcie_bwctrl_note_activity(to_pci_dev(nvmeq->dev->dev), bytes); +} + /* bits for iod->flags */ enum nvme_iod_flags { /* this command has been aborted by the timeout handler */ @@ -1457,6 +1477,7 @@ static blk_status_t nvme_queue_rq(struct blk_mq_hw_ctx *hctx, return ret; spin_lock(&nvmeq->sq_lock); nvme_sq_copy_cmd(nvmeq, &iod->cmd); + nvme_pci_note_activity(nvmeq, req, true); nvme_write_sq_db(nvmeq, bd->last); spin_unlock(&nvmeq->sq_lock); return BLK_STS_OK; @@ -1474,6 +1495,7 @@ static void nvme_submit_cmds(struct nvme_queue *nvmeq, struct rq_list *rqlist) struct nvme_iod *iod = blk_mq_rq_to_pdu(req); nvme_sq_copy_cmd(nvmeq, &iod->cmd); + nvme_pci_note_activity(nvmeq, req, true); } nvme_write_sq_db(nvmeq, true); spin_unlock(&nvmeq->sq_lock); @@ -1518,10 +1540,13 @@ static void nvme_queue_rqs(struct rq_list *rqlist) static __always_inline void nvme_pci_unmap_rq(struct request *req) { + struct nvme_queue *nvmeq = req->mq_hctx->driver_data; + if (blk_integrity_rq(req)) nvme_unmap_metadata(req); if (blk_rq_nr_phys_segments(req)) nvme_unmap_data(req); + nvme_pci_note_activity(nvmeq, req, false); } static void nvme_pci_complete_rq(struct request *req) @@ -3753,6 +3778,8 @@ static int nvme_probe(struct pci_dev *pdev, const struct pci_device_id *id) if (IS_ERR(dev)) return PTR_ERR(dev); + pcie_bwctrl_register(pdev); + result = nvme_add_ctrl(&dev->ctrl); if (result) goto out_put_ctrl; diff --git a/drivers/pci/controller/dwc/pcie-qcom.c b/drivers/pci/controller/dwc/pcie-qcom.c index 19daadee65f7c..137a84748bfc6 100644 --- a/drivers/pci/controller/dwc/pcie-qcom.c +++ b/drivers/pci/controller/dwc/pcie-qcom.c @@ -374,6 +374,10 @@ static void qcom_pcie_set_slot_nccs(struct dw_pcie *pci) val |= PCI_EXP_SLTCAP_NCCS; writel(val, pci->dbi_base + offset + PCI_EXP_SLTCAP); + val = readl(pci->dbi_base + offset + PCI_EXP_LNKCAP); + val |= PCI_EXP_LNKCAP_LBNC; + writel(val, pci->dbi_base + offset + PCI_EXP_LNKCAP); + dw_pcie_dbi_ro_wr_dis(pci); } @@ -1290,6 +1294,10 @@ static int qcom_pcie_post_init_2_9_0(struct qcom_pcie *pcie) val &= ~PCI_EXP_LNKCAP_ASPMS; writel(val, pci->dbi_base + offset + PCI_EXP_LNKCAP); + val = readl(pci->dbi_base + offset + PCI_EXP_LNKCAP); + val |= PCI_EXP_LNKCAP_LBNC; + writel(val, pci->dbi_base + offset + PCI_EXP_LNKCAP); + writel(PCI_EXP_DEVCTL2_COMP_TMOUT_DIS, pci->dbi_base + offset + PCI_EXP_DEVCTL2); diff --git a/drivers/pci/pcie/Kconfig b/drivers/pci/pcie/Kconfig index 207c2deae35ff..1faec0d93cb87 100644 --- a/drivers/pci/pcie/Kconfig +++ b/drivers/pci/pcie/Kconfig @@ -146,3 +146,19 @@ config PCIE_EDR the PCI Firmware Specification r3.2. Enable this if you want to support hybrid DPC model which uses both firmware and OS to implement DPC. + +config PCIE_BW_ONDEMAND + bool "PCIe on-demand link speed scaling" + depends on PCIEPORTBUS && PM_DEVFREQ + default y + help + Enable devfreq-based on-demand PCIe Link Speed scaling. Downstream + device drivers report activity via pcie_bwctrl_note_activity(); + bwctrl uses this to scale Link Speed up or down through the + simple_ondemand governor. + + Changing Link Speed requires link retraining, which briefly + interrupts I/O and may affect other devices sharing the same Root + Port/switch. + + If unsure, say N. diff --git a/drivers/pci/pcie/bwctrl.c b/drivers/pci/pcie/bwctrl.c index c4c8d260bf969..0f4c654be9229 100644 --- a/drivers/pci/pcie/bwctrl.c +++ b/drivers/pci/pcie/bwctrl.c @@ -20,17 +20,26 @@ #define dev_fmt(fmt) "bwctrl: " fmt #include +#include #include #include #include +#include #include +#include #include +#include +#include +#include #include #include #include +#include #include #include +#include #include +#include #include "../pci.h" #include "portdrv.h" @@ -39,10 +48,44 @@ * struct pcie_bwctrl_data - PCIe bandwidth controller * @set_speed_mutex: Serializes link speed changes * @cdev: Thermal cooling device associated with the port + * @devfreq: On-demand Link Speed scaling devfreq device, if enabled + * @profile: devfreq profile backing @devfreq + * @ondemand_data: simple_ondemand governor tuning backing @devfreq + * @freq_table: Link Speeds usable as devfreq frequencies + * @nr_freqs: Number of valid entries in @freq_table (and matching OPPs) + * @activity_lock: Serializes @bytes_in_flight/@bytes_completed_total/@window_start_t + * @bytes_in_flight: Sum of currently-outstanding downstream request sizes + * @bytes_completed_total: Cumulative bytes completed since registration, never reset + * @window_start_t: Start of the current polling window + * @last_agg_total: Snapshot of the aggregated total as of the last poll + * @last_bytes: Bytes completed in the most recently closed window (for logging) + * @last_elapsed_us: Duration of the most recently closed window (for logging) + * @last_capacity_mbps: Link capacity during the most recently closed window (for logging) + * @last_change_t: Time of the most recent Link Speed change, for the dwell-time guard + * @held_for_optin: Set once "holding, not opted in" has been logged, until it clears */ struct pcie_bwctrl_data { struct mutex set_speed_mutex; struct thermal_cooling_device *cdev; +#ifdef CONFIG_PCIE_BW_ONDEMAND + struct devfreq *devfreq; + struct devfreq_dev_profile profile; + struct devfreq_simple_ondemand_data ondemand_data; + unsigned long freq_table[BITS_PER_TYPE(u8)]; + unsigned int nr_freqs; + + spinlock_t activity_lock; + s64 bytes_in_flight; + u64 bytes_completed_total; + ktime_t window_start_t; + u64 last_agg_total; + + u64 last_bytes; + u64 last_elapsed_us; + u32 last_capacity_mbps; + ktime_t last_change_t; + bool held_for_optin; +#endif }; /* Prevent port removal during Link Speed changes. */ @@ -120,6 +163,38 @@ static int pcie_bwctrl_change_speed(struct pci_dev *port, u16 target_speed, bool return pcie_retrain_link(port, use_lt); } +static int pcie_set_opp(struct pci_dev *pdev, struct pci_host_bridge *host, + enum pci_bus_speed speed) +{ + struct device *dev = host->dev.parent; + struct dev_pm_opp_key key = {}; + int ret, freq_mbps, width; + unsigned long freq_kbps; + struct dev_pm_opp *opp; + u16 lnksta; + + pcie_capability_read_word(pdev, PCI_EXP_LNKSTA, &lnksta); + width = FIELD_GET(PCI_EXP_LNKSTA_NLW, lnksta); + + freq_mbps = pcie_dev_speed_mbps(speed); + if (freq_mbps < 0) + return -EINVAL; + + freq_kbps = freq_mbps * KILO; + key.freq = freq_kbps; + key.level = pci_bus_speed2lnkctl2(speed); + key.bw = 0; + opp = dev_pm_opp_find_key_exact(dev, &key, true); + if (!IS_ERR(opp)) { + ret = dev_pm_opp_set_opp(dev, opp); + if (ret) + dev_err(dev, "Failed to set OPP for freq (%lu): %d\n", + freq_kbps * width, ret); + dev_pm_opp_put(opp); + } + return 0; +} + /** * pcie_set_target_speed - Set downstream Link Speed for PCIe Port * @port: PCIe Port @@ -140,9 +215,12 @@ static int pcie_bwctrl_change_speed(struct pci_dev *port, u16 target_speed, bool int pcie_set_target_speed(struct pci_dev *port, enum pci_bus_speed speed_req, bool use_lt) { + struct pci_host_bridge *host = pci_find_host_bridge(port->bus); + bool is_rootbus = pci_is_root_bus(port->bus); struct pci_bus *bus = port->subordinate; + struct pci_dev *child = NULL; + int aspm_state = 0, ret; u16 target_speed; - int ret; if (WARN_ON_ONCE(!pcie_valid_speed(speed_req))) return -EINVAL; @@ -152,6 +230,24 @@ int pcie_set_target_speed(struct pci_dev *port, enum pci_bus_speed speed_req, target_speed = pcie_bwctrl_select_speed(port, speed_req); + /* + * The host bridge driver may need to be scaled for targeted speed + * otherwise link might not come up at requested speed. + */ + if (is_rootbus && host && bus) { + /* Get function 0 of downstream device */ + list_for_each_entry(child, &bus->devices, bus_list) + if (PCI_FUNC(child->devfn) == 0) + break; + + if (child && host->disable_aspm_for_retrain) { + aspm_state = pcie_aspm_enabled(child); + // pci_disable_link_state_locked(child, PCIE_LINK_STATE_ALL); + } + if (speed_req > bus->cur_bus_speed) + pcie_set_opp(port, host, speed_req); + } + scoped_guard(rwsem_read, &pcie_bwctrl_setspeed_rwsem) { struct pcie_bwctrl_data *data = port->link_bwctrl; @@ -176,9 +272,500 @@ int pcie_set_target_speed(struct pci_dev *port, enum pci_bus_speed speed_req, !list_empty(&bus->devices)) ret = -EAGAIN; + if (bus && is_rootbus && host) { + // if (child && host->disable_aspm_for_retrain) + // pci_enable_link_state_locked(child, aspm_state); + pcie_set_opp(port, host, bus->cur_bus_speed); + } + + return ret; +} +EXPORT_SYMBOL_GPL(pcie_set_target_speed); + +#ifdef CONFIG_PCIE_BW_ONDEMAND +static enum pci_bus_speed pcie_bwctrl_cur_speed(struct pci_dev *port) +{ + struct pci_bus *bus = port->subordinate; + + return bus ? bus->cur_bus_speed : PCI_SPEED_UNKNOWN; +} + +/* + * Current negotiated link capacity in Mb/s (encoded bandwidth, i.e. link + * signaling rate reduced by 128b/130b or 8b/10b encoding overhead, times + * negotiated width) -- the real achievable throughput ceiling at the link's + * current speed, not the raw signaling rate. + */ +static u32 pcie_bwctrl_link_capacity_mbps(struct pci_dev *port) +{ + enum pci_bus_speed speed = pcie_bwctrl_cur_speed(port); + u16 lnksta; + u32 width; + int ret; + + ret = pcie_capability_read_word(port, PCI_EXP_LNKSTA, &lnksta); + if (ret != PCIBIOS_SUCCESSFUL) { + pci_err(port, "bwctrl: failed to read LNKSTA for capacity calc: %d\n", + ret); + return 0; + } + + width = FIELD_GET(PCI_EXP_LNKSTA_NLW, lnksta); + + return width * PCIE_SPEED2MBS_ENC(speed); +} + +/* + * Running totals gathered while walking a subtree: @completed_total sums + * cumulative completed bytes (see pcie_bwctrl_note_activity()), and + * @in_flight sums bytes currently outstanding (submitted, not yet + * completed) across every descendant bridge. + */ +struct pcie_bwctrl_agg { + u64 completed_total; + s64 in_flight; +}; + +/* + * Sum bytes_completed_total and bytes_in_flight across every bwctrl-tracked + * bridge found beneath the walked bus. Each endpoint's traffic is credited + * exactly once, at its immediate upstream bridge (see + * pcie_bwctrl_note_activity()), so summing across the whole subtree -- + * however deep -- picks up that credit exactly once without needing + * per-child bookkeeping at every intermediate level. + */ +static int pcie_bwctrl_agg_cb(struct pci_dev *dev, void *userdata) +{ + struct pcie_bwctrl_agg *agg = userdata; + struct pcie_bwctrl_data *cdata; + + if (!pci_is_bridge(dev)) + return 0; + + cdata = READ_ONCE(dev->link_bwctrl); + if (!cdata) + return 0; + + scoped_guard(spinlock_irqsave, &cdata->activity_lock) { + agg->completed_total += cdata->bytes_completed_total; + agg->in_flight += cdata->bytes_in_flight; + } + + return 0; +} + +/* + * Aggregate bytes completed and bytes in flight for @port itself plus + * every descendant bridge reachable from @port's subordinate bus. Runs + * from devfreq_monitor()'s workqueue context (holding devfreq->lock, a + * mutex), so it is safe to sleep here -- pci_walk_bus() takes + * down_read(&pci_bus_sem). + */ +static void pcie_bwctrl_aggregate(struct pci_dev *port, struct pcie_bwctrl_data *data, + struct pcie_bwctrl_agg *agg) +{ + scoped_guard(spinlock_irqsave, &data->activity_lock) { + agg->completed_total = data->bytes_completed_total; + agg->in_flight = data->bytes_in_flight; + } + + if (port->subordinate) + pci_walk_bus(port->subordinate, pcie_bwctrl_agg_cb, agg); +} + +static int pcie_bwctrl_devfreq_get_status(struct device *dev, + struct devfreq_dev_status *stat) +{ + struct pci_dev *port = to_pci_dev(dev); + struct pcie_bwctrl_data *data = port->link_bwctrl; + struct pcie_bwctrl_agg agg; + u64 elapsed_us, bytes, capacity_bytes, credited_in_flight; + u32 capacity_mbps; + ktime_t curr_t; + + memset(stat, 0, sizeof(*stat)); + + if (!data) + return -ENODEV; + + pcie_bwctrl_aggregate(port, data, &agg); + + scoped_guard(spinlock_irqsave, &data->activity_lock) { + curr_t = ktime_get(); + if (!data->window_start_t) { + data->window_start_t = curr_t; + data->last_agg_total = agg.completed_total; + return 0; + } + + elapsed_us = ktime_us_delta(curr_t, data->window_start_t); + bytes = agg.completed_total - data->last_agg_total; + + data->window_start_t = curr_t; + data->last_agg_total = agg.completed_total; + } + + capacity_mbps = pcie_bwctrl_link_capacity_mbps(port); + + stat->current_frequency = pcie_bwctrl_cur_speed(port); + stat->total_time = (u64)capacity_mbps * elapsed_us; + + /* + * A request that stays outstanding across an entire poll window + * (e.g. a large transfer that takes longer than polling_ms to + * complete) contributes nothing to @bytes above until it finally + * completes -- which both hides the link being busy while it's + * still in flight, and produces an oversized spike in whichever + * window the completion happens to land in. Credit each window + * with however much of the link's own capacity is still + * outstanding, capped at that window's real capacity so this can + * never claim more throughput than physically possible. + */ + capacity_bytes = stat->total_time / 8; + credited_in_flight = min_t(u64, max_t(s64, agg.in_flight, 0), capacity_bytes); + + stat->busy_time = min_t(u64, (bytes + credited_in_flight) * 8, stat->total_time); + + scoped_guard(spinlock_irqsave, &data->activity_lock) { + data->last_bytes = bytes; + data->last_elapsed_us = elapsed_us; + data->last_capacity_mbps = capacity_mbps; + } + + return 0; +} + +/* + * Minimum time a newly-reached Link Speed must be held before scaling back + * down. Sustained throughput that sits between two adjacent speeds' + * capacities (over-saturating the lower one, under-using the higher one) + * otherwise makes simple_ondemand flip-flop every poll, and each flip costs + * a real link retrain. Scaling up is never held back by this guard, since + * that would risk starving a real burst. + */ +#define PCIE_BWCTRL_DOWNSCALE_DWELL_MS 1000 + +/* + * Look for any endpoint beneath the walked bus that has not opted into + * bwctrl bandwidth scaling. Bridges are skipped -- only leaf (non-bridge) + * devices are required to opt in, since a bridge itself has no traffic of + * its own to gate on. Returning non-zero stops the walk early. + */ +static int pcie_bwctrl_optin_cb(struct pci_dev *dev, void *userdata) +{ + if (pci_is_bridge(dev)) + return 0; + + return dev->bwctrl_participate ? 0 : 1; +} + +/* + * True only if every endpoint beneath @port's subordinate bus has called + * pcie_bwctrl_register(). A port with no subordinate bus, or an empty one, + * trivially has nothing that could withhold consent. + */ +static bool pcie_bwctrl_all_opted_in(struct pci_dev *port) +{ + int not_opted_in = 0; + + if (port->subordinate) + pci_walk_bus(port->subordinate, pcie_bwctrl_optin_cb, ¬_opted_in); + + return !not_opted_in; +} + +/* + * Clamp a governor-requested scale-DOWN to at most one freq_table step + * below the current speed, so idle-driven decisions retrain one generation + * at a time rather than dropping straight to the slowest supported speed. + * Scale-up is intentionally left unclamped: simple_ondemand's up-scale + * requests DEVFREQ_MAX_FREQ once busy_time crosses upthreshold, and real + * sustained traffic should reach a speed that can actually accommodate it + * immediately, not climb through intermediate generations first. + */ +static enum pci_bus_speed pcie_bwctrl_clamp_step(struct pcie_bwctrl_data *data, + enum pci_bus_speed cur_speed, + enum pci_bus_speed speed_req) +{ + int cur_idx = -1, req_idx = -1; + + if (speed_req >= cur_speed) + return speed_req; + + for (unsigned int i = 0; i < data->nr_freqs; i++) { + if (data->freq_table[i] == cur_speed) + cur_idx = i; + if (data->freq_table[i] == speed_req) + req_idx = i; + } + + /* Have not observed the current speed in the table; do not clamp. */ + if (cur_idx < 0) + return speed_req; + + if (req_idx >= 0 && req_idx < cur_idx - 1) + return data->freq_table[cur_idx - 1]; + + return speed_req; +} + +static int pcie_bwctrl_devfreq_target(struct device *dev, unsigned long *freq, + u32 flags) +{ + struct pci_dev *port = to_pci_dev(dev); + struct pcie_bwctrl_data *data = port->link_bwctrl; + enum pci_bus_speed speed_req = (enum pci_bus_speed)*freq; + enum pci_bus_speed cur_speed = pcie_bwctrl_cur_speed(port); + int ret = 0; + + if (data) + speed_req = pcie_bwctrl_clamp_step(data, cur_speed, speed_req); + + if (data && speed_req < cur_speed) { + s64 since_change_ms; + + scoped_guard(spinlock_irqsave, &data->activity_lock) + since_change_ms = data->last_change_t ? + ktime_ms_delta(ktime_get(), data->last_change_t) : + S64_MAX; + + if (since_change_ms < PCIE_BWCTRL_DOWNSCALE_DWELL_MS) + speed_req = cur_speed; + } + + if (speed_req != cur_speed && data && !pcie_bwctrl_all_opted_in(port)) { + if (!data->held_for_optin) { + pci_dbg(port, + "bwctrl: holding at %u, not every endpoint has opted in\n", + cur_speed); + data->held_for_optin = true; + } + speed_req = cur_speed; + } else if (data) { + data->held_for_optin = false; + } + + if (speed_req != cur_speed) { + u64 last_bytes = 0, last_elapsed_us = 0; + u32 last_capacity_mbps = 0; + + if (data) { + scoped_guard(spinlock_irqsave, &data->activity_lock) { + last_bytes = data->last_bytes; + last_elapsed_us = data->last_elapsed_us; + last_capacity_mbps = data->last_capacity_mbps; + } + } + + pci_dbg(port, + "bwctrl: speed change %u -> %u (last window: %llu bytes / %llu us, capacity %u Mb/s)\n", + cur_speed, speed_req, last_bytes, last_elapsed_us, + last_capacity_mbps); + + ret = pcie_set_target_speed(port, speed_req, true); + if (ret && ret != -EAGAIN) + pci_err(port, "failed to set link speed to %u: %d\n", + speed_req, ret); + if (ret == -EAGAIN) + ret = 0; + + if (data) { + scoped_guard(spinlock_irqsave, &data->activity_lock) + data->last_change_t = ktime_get(); + } + } + + *freq = pcie_bwctrl_cur_speed(port); + return ret; } +static int pcie_bwctrl_devfreq_get_cur_freq(struct device *dev, + unsigned long *freq) +{ + *freq = pcie_bwctrl_cur_speed(to_pci_dev(dev)); + + return 0; +} + +/** + * pcie_bwctrl_note_activity - Report I/O byte transitions to bwctrl + * @pdev: Downstream PCIe device (e.g. an NVMe controller's own pci_dev) + * @bytes: Positive request size at submit, negative (matching) size at completion + * + * May be called from IRQ/atomic context (e.g. NVMe completion handlers), so + * this must not sleep: unlike pcie_set_target_speed(), it does not take + * pcie_bwctrl_setspeed_rwsem. @port->link_bwctrl is only ever cleared (never + * freed) while holding that rwsem for writing, and @data itself is + * devm-allocated against the bwctrl service device, so a lock-free read here + * is safe -- the worst case is observing a stale non-NULL @data pointer + * whose ->devfreq is already NULL, which the check below turns into a no-op. + * + * See the kdoc in for the calling convention. + */ +void pcie_bwctrl_note_activity(struct pci_dev *pdev, s64 bytes) +{ + struct pci_dev *port = pci_upstream_bridge(pdev); + struct pcie_bwctrl_data *data; + + if (!port) + return; + + data = READ_ONCE(port->link_bwctrl); + if (!data) + return; + + guard(spinlock_irqsave)(&data->activity_lock); + + if (!data->devfreq) + return; + + data->bytes_in_flight += bytes; + + if (bytes < 0) + data->bytes_completed_total += -bytes; +} +EXPORT_SYMBOL_GPL(pcie_bwctrl_note_activity); + +/** + * pcie_bwctrl_register - Opt an endpoint into bwctrl bandwidth scaling + * @pdev: Downstream PCIe device (e.g. an NVMe controller's own pci_dev) + * + * See the kdoc in for the calling convention. + */ +void pcie_bwctrl_register(struct pci_dev *pdev) +{ + pdev->bwctrl_participate = 1; +} +EXPORT_SYMBOL_GPL(pcie_bwctrl_register); + +static void pcie_bwctrl_devfreq_init(struct pci_dev *port, struct pcie_bwctrl_data *data) +{ + unsigned long bit, supported_speeds = port->supported_speeds; + unsigned int n = 0, added = 0; + int ret; + + if (hweight8(supported_speeds) <= 1) { + pci_err(port, "bwctrl devfreq: only one supported speed (0x%02lx), skipping\n", + supported_speeds); + return; + } + + /* + * supported_speeds is a PCIe Supported Link Speeds Vector: bit N + * (N >= 1) set means PCIE_SPEED_2_5GT + (N - 1) is supported. + */ + for_each_set_bit(bit, &supported_speeds, BITS_PER_TYPE(u8)) { + if (bit == 0) + continue; + data->freq_table[n++] = PCIE_SPEED_2_5GT + (bit - 1); + } + + pci_err(port, "bwctrl devfreq: supported_speeds=0x%02lx, %u freq_table entries\n", + supported_speeds, n); + + /* + * devfreq_add_device() resolves min/max scaling frequency through + * the OPP framework regardless of profile->freq_table, so an OPP + * must exist for every entry or registration fails with -EINVAL. + */ + for (added = 0; added < n; added++) { + ret = dev_pm_opp_add(&port->dev, data->freq_table[added], 0); + if (ret) { + pci_err(port, "failed to add OPP for speed %lu: %d\n", + data->freq_table[added], ret); + goto err_remove_opps; + } + pci_err(port, "bwctrl devfreq: added OPP for speed %lu\n", + data->freq_table[added]); + } + + data->ondemand_data.upthreshold = 80; + data->ondemand_data.downdifferential = 5; + + data->profile.polling_ms = 100; + data->profile.timer = DEVFREQ_TIMER_DELAYED; + data->profile.target = pcie_bwctrl_devfreq_target; + data->profile.get_dev_status = pcie_bwctrl_devfreq_get_status; + data->profile.get_cur_freq = pcie_bwctrl_devfreq_get_cur_freq; + data->profile.freq_table = data->freq_table; + data->profile.max_state = n; + data->profile.initial_freq = pcie_bwctrl_cur_speed(port); + + pci_err(port, "bwctrl devfreq: registering, initial_freq=%u\n", + data->profile.initial_freq); + + data->devfreq = devfreq_add_device(&port->dev, &data->profile, + DEVFREQ_GOV_SIMPLE_ONDEMAND, + &data->ondemand_data); + if (IS_ERR(data->devfreq)) { + pci_err(port, "failed to register bwctrl devfreq: %ld\n", + PTR_ERR(data->devfreq)); + data->devfreq = NULL; + goto err_remove_opps; + } + + data->nr_freqs = n; + pci_err(port, "bwctrl devfreq: registered successfully\n"); + + return; + +err_remove_opps: + for (unsigned int i = 0; i < added; i++) + dev_pm_opp_remove(&port->dev, data->freq_table[i]); +} + +static void pcie_bwctrl_devfreq_remove(struct pci_dev *port, struct pcie_bwctrl_data *data) +{ + struct devfreq *devfreq; + unsigned int nr_freqs; + + scoped_guard(spinlock_irqsave, &data->activity_lock) { + devfreq = data->devfreq; + data->devfreq = NULL; + nr_freqs = data->nr_freqs; + data->nr_freqs = 0; + } + + if (!devfreq) + return; + + pci_err(port, "bwctrl devfreq: removing\n"); + devfreq_remove_device(devfreq); + + for (unsigned int i = 0; i < nr_freqs; i++) + dev_pm_opp_remove(&port->dev, data->freq_table[i]); +} + +static void pcie_bwctrl_devfreq_suspend(struct pcie_bwctrl_data *data) +{ + if (data->devfreq) + devfreq_suspend_device(data->devfreq); +} + +static void pcie_bwctrl_devfreq_resume(struct pcie_bwctrl_data *data) +{ + if (data->devfreq) + devfreq_resume_device(data->devfreq); +} +#else +static void pcie_bwctrl_devfreq_init(struct pci_dev *port, struct pcie_bwctrl_data *data) +{ +} + +static void pcie_bwctrl_devfreq_remove(struct pci_dev *port, struct pcie_bwctrl_data *data) +{ +} + +static void pcie_bwctrl_devfreq_suspend(struct pcie_bwctrl_data *data) +{ +} + +static void pcie_bwctrl_devfreq_resume(struct pcie_bwctrl_data *data) +{ +} +#endif /* CONFIG_PCIE_BW_ONDEMAND */ + static void pcie_bwnotif_enable(struct pcie_device *srv) { struct pci_dev *port = srv->port; @@ -266,13 +853,17 @@ static int pcie_bwnotif_probe(struct pcie_device *srv) if (ret) return ret; +#ifdef CONFIG_PCIE_BW_ONDEMAND + spin_lock_init(&data->activity_lock); +#endif + scoped_guard(rwsem_write, &pcie_bwctrl_setspeed_rwsem) { - port->link_bwctrl = data; + WRITE_ONCE(port->link_bwctrl, data); ret = request_irq(srv->irq, pcie_bwnotif_irq, IRQF_SHARED, "PCIe bwctrl", srv); if (ret) { - port->link_bwctrl = NULL; + WRITE_ONCE(port->link_bwctrl, NULL); return ret; } @@ -286,6 +877,8 @@ static int pcie_bwnotif_probe(struct pcie_device *srv) if (IS_ERR(port->link_bwctrl->cdev)) port->link_bwctrl->cdev = NULL; + pcie_bwctrl_devfreq_init(port, data); + return 0; } @@ -293,6 +886,13 @@ static void pcie_bwnotif_remove(struct pcie_device *srv) { struct pcie_bwctrl_data *data = srv->port->link_bwctrl; + /* + * Stop devfreq polling (synchronously) before anything below tears + * down port->link_bwctrl, so no devfreq callback can ever observe a + * stale or NULL pcie_bwctrl_data. + */ + pcie_bwctrl_devfreq_remove(srv->port, data); + pcie_cooling_device_unregister(data->cdev); scoped_guard(rwsem_write, &pcie_bwctrl_setspeed_rwsem) { @@ -300,12 +900,13 @@ static void pcie_bwnotif_remove(struct pcie_device *srv) free_irq(srv->irq, srv); - srv->port->link_bwctrl = NULL; + WRITE_ONCE(srv->port->link_bwctrl, NULL); } } static int pcie_bwnotif_suspend(struct pcie_device *srv) { + pcie_bwctrl_devfreq_suspend(srv->port->link_bwctrl); pcie_bwnotif_disable(srv->port); return 0; } @@ -313,6 +914,7 @@ static int pcie_bwnotif_suspend(struct pcie_device *srv) static int pcie_bwnotif_resume(struct pcie_device *srv) { pcie_bwnotif_enable(srv); + pcie_bwctrl_devfreq_resume(srv->port->link_bwctrl); return 0; } diff --git a/include/linux/pci-bwctrl.h b/include/linux/pci-bwctrl.h index cee07127455b3..cebbc8bc9a1fa 100644 --- a/include/linux/pci-bwctrl.h +++ b/include/linux/pci-bwctrl.h @@ -25,4 +25,43 @@ static inline void pcie_cooling_device_unregister(struct thermal_cooling_device } #endif +/** + * pcie_bwctrl_register - Opt an endpoint into bwctrl bandwidth scaling + * @pdev: Downstream PCIe device (e.g. an NVMe controller's own pci_dev) + * + * Call once from the endpoint driver's probe(). Every endpoint beneath a + * given ancestor Switch/Root Port must call this before that ancestor's + * Link Speed is ever scaled up or down; a single non-participating + * endpoint (including one whose driver never calls this at all) holds + * every ancestor up to the Root Port at its current speed. Safe to call + * unconditionally; a no-op when CONFIG_PCIE_BW_ONDEMAND is disabled. + */ +#ifdef CONFIG_PCIE_BW_ONDEMAND +void pcie_bwctrl_register(struct pci_dev *pdev); +#else +static inline void pcie_bwctrl_register(struct pci_dev *pdev) +{ +} +#endif + +/** + * pcie_bwctrl_note_activity - Report I/O byte transitions to bwctrl + * @pdev: Downstream PCIe device (e.g. an NVMe controller's own pci_dev) + * @bytes: Positive request size at submit, negative (matching) size at completion + * + * Callers call this once at submit with the request's size, and once at + * completion with the negated size. bwctrl uses the resulting bytes-per-poll + * throughput, together with the devfreq simple_ondemand governor, to scale + * the upstream Root Port's Link Speed up or down. Safe to call + * unconditionally; a no-op when CONFIG_PCIE_BW_ONDEMAND is disabled or the + * port has no bwctrl instance. + */ +#ifdef CONFIG_PCIE_BW_ONDEMAND +void pcie_bwctrl_note_activity(struct pci_dev *pdev, s64 bytes); +#else +static inline void pcie_bwctrl_note_activity(struct pci_dev *pdev, s64 bytes) +{ +} +#endif + #endif diff --git a/include/linux/pci.h b/include/linux/pci.h index 09134d0a559fd..f793f9066a201 100644 --- a/include/linux/pci.h +++ b/include/linux/pci.h @@ -428,6 +428,7 @@ struct pci_dev { unsigned int clear_retrain_link:1; /* Need to clear Retrain Link bit manually */ unsigned int no_bw_notif:1; /* BW notifications may cause issues */ + unsigned int bwctrl_participate:1; /* Driver opted into bwctrl bandwidth scaling */ unsigned int d3hot_delay; /* D3hot->D0 transition time in ms */ unsigned int d3cold_delay; /* D3cold->D0 transition time in ms */ @@ -664,6 +665,7 @@ struct pci_host_bridge { unsigned int msi_domain:1; /* Bridge wants MSI domain */ unsigned int broken_l1ss_resume:1; /* Resuming from L1SS during system suspend is broken */ + unsigned int disable_aspm_for_retrain:1; /* Disable ASPM before link retain */ /* Resource alignment requirements */ resource_size_t (*align_resource)(struct pci_dev *dev,