Add support for BW scale

drivers/nvme/host/pci.c

@@ -21,6 +21,7 @@
 #include <linux/nodemask.h>
 #include <linux/once.h>
 #include <linux/pci.h>
+#include <linux/pci-bwctrl.h>
 #include <linux/suspend.h>
 #include <linux/t10-pi.h>
 #include <linux/types.h>
@@ -393,6 +394,25 @@ struct nvme_queue {
 	struct completion delete_done;
 };
 
+/*
+ * Report request byte transitions to bwctrl's on-demand PCIe Link Speed
+ * scaling. Admin-queue traffic (keep-alives, log page/identify, etc.) is
+ * excluded so link speed tracks actual data I/O, not driver housekeeping.
+ */
+static void nvme_pci_note_activity(struct nvme_queue *nvmeq, struct request *req,
+				   bool submit)
+{
+	s64 bytes = blk_rq_bytes(req);
+
+	if (!nvmeq->qid)
+		return;
+
+	if (!submit)
+		bytes = -bytes;
+
+	pcie_bwctrl_note_activity(to_pci_dev(nvmeq->dev->dev), bytes);
+}
+
 /* bits for iod->flags */
 enum nvme_iod_flags {
 	/* this command has been aborted by the timeout handler */
@@ -1457,6 +1477,7 @@ static blk_status_t nvme_queue_rq(struct blk_mq_hw_ctx *hctx,
 		return ret;
 	spin_lock(&nvmeq->sq_lock);
 	nvme_sq_copy_cmd(nvmeq, &iod->cmd);
+	nvme_pci_note_activity(nvmeq, req, true);
 	nvme_write_sq_db(nvmeq, bd->last);
 	spin_unlock(&nvmeq->sq_lock);
 	return BLK_STS_OK;
@@ -1474,6 +1495,7 @@ static void nvme_submit_cmds(struct nvme_queue *nvmeq, struct rq_list *rqlist)
 		struct nvme_iod *iod = blk_mq_rq_to_pdu(req);
 
 		nvme_sq_copy_cmd(nvmeq, &iod->cmd);
+		nvme_pci_note_activity(nvmeq, req, true);
 	}
 	nvme_write_sq_db(nvmeq, true);
 	spin_unlock(&nvmeq->sq_lock);
@@ -1518,10 +1540,13 @@ static void nvme_queue_rqs(struct rq_list *rqlist)
 
 static __always_inline void nvme_pci_unmap_rq(struct request *req)
 {
+	struct nvme_queue *nvmeq = req->mq_hctx->driver_data;
+
 	if (blk_integrity_rq(req))
 		nvme_unmap_metadata(req);
 	if (blk_rq_nr_phys_segments(req))
 		nvme_unmap_data(req);
+	nvme_pci_note_activity(nvmeq, req, false);
 }
 
 static void nvme_pci_complete_rq(struct request *req)
@@ -3753,6 +3778,8 @@ static int nvme_probe(struct pci_dev *pdev, const struct pci_device_id *id)
 	if (IS_ERR(dev))
 		return PTR_ERR(dev);
 
+	pcie_bwctrl_register(pdev);
+
 	result = nvme_add_ctrl(&dev->ctrl);
 	if (result)
 		goto out_put_ctrl;

drivers/pci/controller/dwc/pcie-qcom.c

@@ -374,6 +374,10 @@ static void qcom_pcie_set_slot_nccs(struct dw_pcie *pci)
 	val |= PCI_EXP_SLTCAP_NCCS;
 	writel(val, pci->dbi_base + offset + PCI_EXP_SLTCAP);
 
+	val = readl(pci->dbi_base + offset + PCI_EXP_LNKCAP);
+	val |= PCI_EXP_LNKCAP_LBNC;
+	writel(val, pci->dbi_base + offset + PCI_EXP_LNKCAP);
+
 	dw_pcie_dbi_ro_wr_dis(pci);
 }
 
@@ -1290,6 +1294,10 @@ static int qcom_pcie_post_init_2_9_0(struct qcom_pcie *pcie)
 	val &= ~PCI_EXP_LNKCAP_ASPMS;
 	writel(val, pci->dbi_base + offset + PCI_EXP_LNKCAP);
 
+	val = readl(pci->dbi_base + offset + PCI_EXP_LNKCAP);
+	val |= PCI_EXP_LNKCAP_LBNC;
+	writel(val, pci->dbi_base + offset + PCI_EXP_LNKCAP);
+
 	writel(PCI_EXP_DEVCTL2_COMP_TMOUT_DIS, pci->dbi_base + offset +
 			PCI_EXP_DEVCTL2);
 

drivers/pci/pcie/Kconfig

@@ -146,3 +146,19 @@ config PCIE_EDR
 	  the PCI Firmware Specification r3.2.  Enable this if you want to
 	  support hybrid DPC model which uses both firmware and OS to
 	  implement DPC.
+
+config PCIE_BW_ONDEMAND
+	bool "PCIe on-demand link speed scaling"
+	depends on PCIEPORTBUS && PM_DEVFREQ
+	default y
+	help
+	  Enable devfreq-based on-demand PCIe Link Speed scaling. Downstream
+	  device drivers report activity via pcie_bwctrl_note_activity();
+	  bwctrl uses this to scale Link Speed up or down through the
+	  simple_ondemand governor.
+
+	  Changing Link Speed requires link retraining, which briefly
+	  interrupts I/O and may affect other devices sharing the same Root
+	  Port/switch.
+
+	  If unsure, say N.