T8140 sleep: fixes from review

arch/arm64/kernel/hibernate.c

@@ -19,6 +19,7 @@
 
 #include <asm/barrier.h>
 #include <asm/cacheflush.h>
+#include <asm/cpu_ops.h>
 #include <asm/cputype.h>
 #include <asm/daifflags.h>
 #include <asm/irqflags.h>
@@ -336,7 +337,7 @@ int swsusp_arch_suspend(void)
 	unsigned long flags;
 	struct sleep_stack_data state;
 
-	if (cpus_are_stuck_in_kernel()) {
+	if (cpus_are_stuck_in_kernel() || spin_table_cpus_stuck()) {
 		pr_err("Can't hibernate: no mechanism to offline secondary CPUs.\n");
 		return -EBUSY;
 	}

arch/arm64/kernel/smp_spin_table.c

@@ -397,15 +397,13 @@ const struct cpu_operations smp_rvbar_spin_table_ops = {
 };
 
 /*
- * Parking CPUs leave the kernel only through a platform's power ops. Until one
- * registers, kexec has to take them as stuck, as on plain spin-table.
+ * An offlined parking CPU waits in this kernel's text and stack, which a kexec or a
+ * hibernation restore overwrites: it is stuck for those, power ops or not.
  */
 bool spin_table_cpus_stuck(void)
 {
 	unsigned int cpu;
 
-	if (READ_ONCE(spin_table_power))
-		return false;
 	for_each_possible_cpu(cpu)
 		if (get_cpu_ops(cpu) == &smp_rvbar_spin_table_ops)
 			return true;

drivers/dma/apple-admac.c

@@ -715,7 +715,11 @@ static int admac_resume(struct dma_chan *chan)
 	if (adchan->current_tx && adchan->current_tx->error) {
 		ret = -EIO;
 	} else if (adchan->lost && adchan->current_tx) {
-		/* The rings went with the registers: start from the top. */
+		struct admac_tx *tx = adchan->current_tx;
+
+		/* The rings went with the registers: start again from the last period reclaimed. */
+		tx->submitted_pos = tx->reclaimed_pos;
+		tx->nperiods_outstanding = 0;
 		adchan->nperiod_acks = 0;
 		admac_start_current_tx(adchan);
 	} else {
@@ -1116,17 +1120,17 @@ static int admac_runtime_resume(struct device *dev)
 }
 
 /*
- * A system sleep that reached the hardware takes the registers of a channel a
- * client still holds, and its client won't configure it again: only a new
- * hw_params does that.
+ * A firmware-driven sleep, refused or not, takes the registers of a channel a
+ * client still holds (genpd gates the domain at noirq), and its client won't
+ * configure it again: only a new hw_params does that.
  */
 static int __maybe_unused admac_resume_early(struct device *dev)
 {
 	struct admac_data *ad = dev_get_drvdata(dev);
 	unsigned long flags;
 	int i;
 
-	if (!pm_resume_via_firmware())
+	if (!pm_suspend_via_firmware())
 		return 0;
 
 	mutex_lock(&ad->cache_alloc_lock);

drivers/dma/apple-sio.c

@@ -2161,14 +2161,15 @@ static __maybe_unused int sio_resume_early(struct device *dev)
 		dev_info(dev, "sleep refused: firmware back after a reset\n");
 		return 0;
 	}
-	sio->asleep = false;
-
 	/* Endpoints and parameters stay with the firmware: nothing is sent again. */
 	ret = apple_rtkit_sleep_return(sio->rtk, sio->base + REG_CPU_CONTROL);
-	if (ret)
+	if (ret) {
+		/* Still down: the next sleep restarts it, or resets it if refused. */
 		dev_err(dev, "firmware did not come back from sleep: %d\n", ret);
-	else
-		dev_dbg(dev, "firmware back from sleep\n");
+		return 0;
+	}
+	sio->asleep = false;
+	dev_dbg(dev, "firmware back from sleep\n");
 
 	return 0;
 }

drivers/gpu/drm/apple/dcp-internal.h

@@ -977,6 +977,7 @@ int iomfb_configure_backlight_h17p(struct apple_dcp *dcp, u32 maximum,
 				   bool default_valid, u32 default_nits);
 void iomfb_queue_init(struct apple_dcp *dcp);
 void iomfb_queue_stop(struct apple_dcp *dcp);
+void iomfb_reset_transport(struct apple_dcp *dcp);
 bool iomfb_queue_drain(struct apple_dcp *dcp, unsigned long timeout);
 int iomfb_queue_settle(struct apple_dcp *dcp, unsigned long timeout);
 void iomfb_pause_presentation(struct apple_dcp *dcp, bool pause);

drivers/gpu/drm/apple/dcp.c

@@ -6775,10 +6775,13 @@ static void dcp_drain_for_sleep(struct apple_dcp *dcp)
  */
 static void dcp_hold_for_sleep(struct apple_dcp *dcp)
 {
-	/* A failed component bind leaves no coprocessor session to hold. */
+	/*
+	 * A failed component bind leaves no coprocessor session to hold. Firmware that DPMS
+	 * stopped sends nothing, and its presentation stays paused until DPMS restarts it.
+	 */
 	if (IS_ERR_OR_NULL(dcp->rtk) ||
 	    !dcp_uses_soft_dpms(dcp) || !dcp_presents_nits(dcp) ||
-	    READ_ONCE(dcp->crashed))
+	    READ_ONCE(dcp->crashed) || READ_ONCE(dcp->s2r_fw_down))
 		return;
 
 	iomfb_pause_presentation(dcp, true);
@@ -7537,6 +7540,19 @@ static int dcp_s2r_ext_domains_on(struct apple_dcp *dcp)
 	return ret;
 }
 
+/*
+ * A firmware stopped while crashed is about to start over: forget the calls and the
+ * transaction the old one left unanswered, and let the new one's messages through. A failed
+ * start marks it crashed again.
+ */
+static void dcp_s2r_forget_crash(struct apple_dcp *dcp)
+{
+	if (!READ_ONCE(dcp->crashed))
+		return;
+	iomfb_reset_transport(dcp);
+	WRITE_ONCE(dcp->crashed, false);
+}
+
 /*
  * DCPEXT back after a real sleep, the same way in every macOS wake measured: its domains,
  * its DART's streams, and a warm restart, with no secure port and no display-on call. After
@@ -7547,6 +7563,7 @@ static int dcp_s2r_ext_start(struct apple_dcp *dcp, bool cycled)
 	void __iomem *control = dcp->coproc_reg + APPLE_DCP_COPROC_CPU_CONTROL;
 	int ret;
 
+	dcp_s2r_forget_crash(dcp);
 	if (!dcp->s2r_dcp_dart)
 		return -ENODEV;
 	ret = apple_dart_gapf_ready(dcp->s2r_dcp_dart);
@@ -7689,6 +7706,7 @@ static int dcp_s2r_warm_restart(struct apple_dcp *dcp)
 {
 	int ret;
 
+	dcp_s2r_forget_crash(dcp);
 	ret = dcp_s2r_domains_on(dcp);
 	if (ret)
 		return ret;

drivers/gpu/drm/apple/iomfb.c

@@ -582,6 +582,38 @@ void iomfb_queue_init(struct apple_dcp *dcp)
 	atomic_set(&dcp->iomfb.opaque_x_state, IOMFB_OPAQUE_X_WAITING);
 }
 
+/*
+ * For firmware stopped cold and started again: the calls and the transaction the old
+ * one never answered went with it, and would hold the channels and the queue forever.
+ */
+void iomfb_reset_transport(struct apple_dcp *dcp)
+{
+	struct iomfb_transaction *transaction;
+
+	if (!iomfb_uses_queue(dcp))
+		return;
+
+	cancel_delayed_work_sync(&dcp->iomfb.timeout);
+	mutex_lock(&dcp->iomfb.lock);
+	transaction = dcp->iomfb.active;
+	dcp->iomfb.active = NULL;
+	if (transaction) {
+		if (transaction->brightness_only)
+			dcp->iomfb.backlight_queued = false;
+		transaction->release(transaction);
+	}
+	iomfb_discard_pending(dcp);
+	dcp->ch_cmd.depth = 0;
+	dcp->ch_oobcmd.depth = 0;
+	dcp->ch_cb.depth = 0;
+	dcp->ch_oobcb.depth = 0;
+	dcp->ch_async.depth = 0;
+	dcp->ch_oobasync.depth = 0;
+	dcp->present_state_h17p.pending = false;
+	dcp->present_state_h17p.accepted = false;
+	mutex_unlock(&dcp->iomfb.lock);
+}
+
 void iomfb_queue_stop(struct apple_dcp *dcp)
 {
 	if (!iomfb_uses_queue(dcp))

drivers/nvme/host/apple.c

@@ -2946,6 +2946,8 @@ static int apple_nvme_suspend(struct device *dev)
 	int ret;
 
 	if (anv->hw->needs_ioq_registers) {
+		/* A sleep right after a wake, as from a dark wake: let that reset finish first. */
+		flush_work(&anv->ctrl.reset_work);
 		anv->sleep_stop = apple_nvme_sleep_wanted(anv);
 		anv->sleep_idled = false;
 		anv->sleep_failed = false;

drivers/phy/apple/atc.c

@@ -788,6 +788,8 @@ enum atcphy_t8132_cio_tunable {
  *                 off for a PIPE or lane change and is not powered on again yet
  * @usb3_port_work: Retries the power-on of @usb3_port_off after a USB failure
  * @usb3_port_retries: Retries left for @usb3_port_work
+ * @sleep_finish_work: Ends a sleep for the controller, outside its device lock
+ * @sleep_finish_kept: Whether the controller kept its state, for @sleep_finish_work
  * @mode: Current PHY operating mode
  * @swap_lanes: True if lanes must be swapped due to cable orientation
  * @dp_link_rate: DisplayPort link rate
@@ -919,6 +921,8 @@ struct apple_atcphy {
 	bool usb3_port_off;
 	struct delayed_work usb3_port_work;
 	unsigned int usb3_port_retries;
+	struct work_struct sleep_finish_work;
+	bool sleep_finish_kept;
 	enum atcphy_mode mode;
 	int dp_link_rate;
 	bool swap_lanes;
@@ -6556,6 +6560,8 @@ static bool atcphy_has_t8103_usb3_workaround(const struct atcphy_hw *hw,
 	       of_device_is_compatible(root, "apple,t8103");
 }
 
+static void atcphy_sleep_finish_work(struct work_struct *work);
+
 static int atcphy_probe(struct platform_device *pdev)
 {
 	struct apple_atcphy *atcphy;
@@ -6578,6 +6584,9 @@ static int atcphy_probe(struct platform_device *pdev)
 	mutex_init(&atcphy->lock);
 	platform_set_drvdata(pdev, atcphy);
 	ret = devm_delayed_work_autocancel(dev, &atcphy->usb3_port_work, atcphy_usb3_port_work);
+	if (ret)
+		return ret;
+	ret = devm_work_autocancel(dev, &atcphy->sleep_finish_work, atcphy_sleep_finish_work);
 	if (ret)
 		return ret;
 
@@ -7110,12 +7119,8 @@ EXPORT_SYMBOL_GPL(apple_atcphy_sleep_result);
  * again. Only the PHY's own failures stop it for the later sleeps; a sleep refused for a
  * USB change doesn't.
  */
-void apple_atcphy_sleep_finish(struct phy *usb2_phy, bool kept)
+static void atcphy_sleep_finish(struct apple_atcphy *atcphy, bool kept)
 {
-	struct apple_atcphy *atcphy = atcphy_from_usb2_phy(usb2_phy);
-
-	if (!atcphy)
-		return;
 	scoped_guard(mutex, &atcphy->lock) {
 		if (!kept && atcphy->sleep_ret < 0 && atcphy->sleep_ret != -EBUSY)
 			atcphy->sleep_no_keep = true;
@@ -7124,8 +7129,47 @@ void apple_atcphy_sleep_finish(struct phy *usb2_phy, bool kept)
 	}
 	atcphy_sleep_ss_up(atcphy);
 }
+
+void apple_atcphy_sleep_finish(struct phy *usb2_phy, bool kept)
+{
+	struct apple_atcphy *atcphy = atcphy_from_usb2_phy(usb2_phy);
+
+	if (atcphy)
+		atcphy_sleep_finish(atcphy, kept);
+}
 EXPORT_SYMBOL_GPL(apple_atcphy_sleep_finish);
 
+static void atcphy_sleep_finish_work(struct work_struct *work)
+{
+	struct apple_atcphy *atcphy = container_of(work, struct apple_atcphy, sleep_finish_work);
+
+	atcphy_sleep_finish(atcphy, READ_ONCE(atcphy->sleep_finish_kept));
+}
+
+/*
+ * The SuperSpeed side may wait for the controller's device lock, which an unbind of the
+ * controller holds while it cancels the controller's own work: so this work is the PHY's.
+ */
+void apple_atcphy_sleep_finish_async(struct phy *usb2_phy, bool kept)
+{
+	struct apple_atcphy *atcphy = atcphy_from_usb2_phy(usb2_phy);
+
+	if (!atcphy)
+		return;
+	WRITE_ONCE(atcphy->sleep_finish_kept, kept);
+	queue_work(system_unbound_wq, &atcphy->sleep_finish_work);
+}
+EXPORT_SYMBOL_GPL(apple_atcphy_sleep_finish_async);
+
+void apple_atcphy_sleep_finish_flush(struct phy *usb2_phy)
+{
+	struct apple_atcphy *atcphy = atcphy_from_usb2_phy(usb2_phy);
+
+	if (atcphy)
+		flush_work(&atcphy->sleep_finish_work);
+}
+EXPORT_SYMBOL_GPL(apple_atcphy_sleep_finish_flush);
+
 static int atcphy_suspend(struct device *dev)
 {
 	struct apple_atcphy *atcphy = dev_get_drvdata(dev);

drivers/pmdomain/apple/pmgr-pwrstate.c

@@ -363,6 +363,12 @@ static int apple_pmgr_ps_power_off(struct generic_pm_domain *genpd)
 	/* A live coprocessor keeps this domain through system sleep. */
 	if (ps->sleep_retained && ps->system_sleep)
 		return -EBUSY;
+	/*
+	 * Its driver took it down for the sleep and raises it in its own order. Gating it
+	 * here would take its parents too, and genpd would raise them all first at the wake.
+	 */
+	if (ps->system_sleep && apple_sleep_domain_is_down(ps->dev->of_node))
+		return -EBUSY;
 
 	ret = apple_pmgr_ps_set(genpd, APPLE_PMGR_PS_PWRGATE, false);
 	if (ret)