diff --git a/Documentation/devicetree/bindings/display/apple,dcp.yaml b/Documentation/devicetree/bindings/display/apple,dcp.yaml index 431e6ffdbf1c9..41090bda6e80d 100644 --- a/Documentation/devicetree/bindings/display/apple,dcp.yaml +++ b/Documentation/devicetree/bindings/display/apple,dcp.yaml @@ -514,7 +514,13 @@ allOf: - const: 27 - const: 0 - const: 0 - else: + - if: + not: + properties: + compatible: + contains: + enum: ['apple,t6050-dcp', 'apple,t6050-dcpext', 'apple,t8142-dcp'] + then: properties: clocks: maxItems: 2 @@ -524,6 +530,26 @@ allOf: - const: clock-194 apple,clock-ids: false apple,firmware-physical-regs: false + # T8142 runs both 26.x and 27.0 DCP firmware. The 26.x paths use the first + # clock; firmware 27 asks for the provider clocks of apple,clock-ids by + # index, as firmware-0 and firmware-1. + - if: + properties: + compatible: + contains: + const: apple,t8142-dcp + then: + properties: + clocks: + minItems: 1 + maxItems: 3 + clock-names: + minItems: 1 + items: + - const: clock-154 + - const: firmware-0 + - const: firmware-1 + apple,firmware-physical-regs: false - if: not: properties: diff --git a/arch/arm64/boot/dts/apple/t8142-j813-display.dtsi b/arch/arm64/boot/dts/apple/t8142-j813-display.dtsi index d9e890af7fd44..812b558be0ee6 100644 --- a/arch/arm64/boot/dts/apple/t8142-j813-display.dtsi +++ b/arch/arm64/boot/dts/apple/t8142-j813-display.dtsi @@ -79,7 +79,14 @@ apple,bw-doorbell = <&pmgr_dcp 1 6>; /* Scan out the primary plane on IOMFB surface 2. */ apple,iomfb-surfaces = <0 0 1 0>; - clocks = <&clkref>; + /* + * clkref is the clock the 26.x firmware paths use. Firmware 27 + * (DCP_FIRMWARE_V_27_0) asks for its provider clocks by index into + * /arm-io/disp0 clock-ids, in this order: firmware-0 and firmware-1. + */ + apple,clock-ids = <585 431>; + clocks = <&clkref>, <&dcp_fw_clk585>, <&dcp_fw_clk431>; + clock-names = "clock-154", "firmware-0", "firmware-1"; /* generate phandle explicitly for use in loader */ phandle = <&dcp>; status = "disabled"; @@ -106,3 +113,28 @@ }; }; }; + +/ { + /* + * The DCP firmware's clocks 585 and 431 (/arm-io/disp0 clock-ids). The + * J813 ADT's clock-frequencies (ids 0x100..0x1ad) has no entry for + * either. Firmware 27 reads only 585, as its minimum pixel clock + * (getClockFrequency(0)); below its 138.521 MHz video clock it gives up + * on frequencies. 712 MHz is clock-frequencies entry 0x49, the one 585's + * low byte names; with it macOS 27.0's DCP firmware offered a display + * under aura. The firmware never asks for 431, which has no rate here. + */ + dcp_fw_clk585: clock-dcp-firmware-585 { + compatible = "fixed-clock"; + #clock-cells = <0>; + clock-frequency = <712000000>; + clock-output-names = "dcp_firmware_585"; + }; + + dcp_fw_clk431: clock-dcp-firmware-431 { + compatible = "fixed-clock"; + #clock-cells = <0>; + clock-frequency = <0>; + clock-output-names = "dcp_firmware_431"; + }; +}; diff --git a/arch/arm64/boot/dts/apple/t8142-j813-gpu.dtsi b/arch/arm64/boot/dts/apple/t8142-j813-gpu.dtsi index d1b8b540538bd..763125bfe4ec7 100644 --- a/arch/arm64/boot/dts/apple/t8142-j813-gpu.dtsi +++ b/arch/arm64/boot/dts/apple/t8142-j813-gpu.dtsi @@ -22,6 +22,17 @@ reg = <0 0 0 0>; }; + /* + * macOS 27.0 (26A428): both roles' __DATA and __DATA_SHARED_RO, contiguous + * (ADT gfx-asc and gfx1-asc segment-ranges), where the firmware places its crash + * logs. Fixed at the addresses iBoot has used on every 27.0 boot so far, until the + * bootloader fills it in. + */ + /* Mapped like gfx-data: the driver reads the crash logs through the linear map. */ + gpu_fw_data: fw-data { + reg = <0x100 0x02444000 0x0 0x184000>; + }; + uat_handoff: uat-handoff { reg = <0 0 0 0>; }; @@ -93,9 +104,10 @@ mboxes = <&gfx_mbox>, <&gfx1_mbox>; power-domains = <&ps_gfx>; memory-region = <&uat_ttbs>, <&uat_pagetables>, <&uat_handoff>, - <&uat_l2_pagetables>, <&gfx_data>, <&gfx1_data>; + <&uat_l2_pagetables>, <&gfx_data>, <&gfx1_data>, + <&gpu_fw_data>; memory-region-names = "ttbs", "pagetables", "handoff", - "shared-l2", "gfx-data", "gfx1-data"; + "shared-l2", "gfx-data", "gfx1-data", "fw-data"; operating-points-v2 = <&gpu_opp>; /* Holds the DCS and fabric floor while GPU work is in flight. */ apple,clpc = <&clpc>; diff --git a/drivers/gpu/drm/apple/apple_drv.c b/drivers/gpu/drm/apple/apple_drv.c index 90aee6d8950db..459d5da71fbc5 100644 --- a/drivers/gpu/drm/apple/apple_drv.c +++ b/drivers/gpu/drm/apple/apple_drv.c @@ -1135,11 +1135,23 @@ static int apple_probe_per_dcp(struct device *dev, interchange = supports_interchange && (zpos <= 1 || surf != top_surf); + /* + * The T8142 runs macOS 27.0's DCP firmware as firmware + * 27, outside the H17G profile, and blends XRGB's X + * byte as alpha all the same (GPU-rendered alacritty + * showed wrong colours on the panel). + */ + bool xrgb_is_alpha = + dcp_data->hw.iomfb_method_profile == + DCP_IOMFB_METHODS_H17G || + (dcp_data->fw_compat == DCP_FIRMWARE_V_27_0 && + of_device_is_compatible(dcp_data->dev->of_node, + "apple,t8142-dcp")); + planes[zpos] = apple_plane_init(drm, 1U << num, surf, supports_l10r, supports_xrgb2101010, - dcp_data->hw.iomfb_method_profile == - DCP_IOMFB_METHODS_H17G, + xrgb_is_alpha, plane_type, interchange, supports_premultiplied); } diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index bbabde4d64715..dce1b61f4853c 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -5013,6 +5013,18 @@ static enum dcp_firmware_version dcp_check_firmware_version(struct device *dev) char fw_str[DCP_FW_VERSION_STR_LEN]; int ret; + /* + * T8142 (J813) on macOS 27.0: the H17P interface written for its 26.x + * firmware makes the 27.0 DCP firmware panic right after the first + * hotplug ("ap_link: callback failed with: 0x5"). It runs the T6050's + * firmware-27 generation. + */ + if (of_device_is_compatible(dev->of_node, "apple,t8142-dcp")) { + ret = dcp_read_fw_version(dev, "apple,firmware-compat", compat_str); + if (ret >= 0 && !strcmp(compat_str, "27.0.0")) + return DCP_FIRMWARE_V_27_0; + } + if (hw->firmware_compat == DCP_FIRMWARE_V_27_0) { ret = dcp_read_fw_version(dev, "apple,firmware-compat", compat_str); if (ret < 0 || strcmp(compat_str, "27.0.0")) @@ -5270,6 +5282,30 @@ static int dcp_firmware_metadata_v27(struct apple_dcp *dcp) clocks = 0; if (clocks < 0 || clocks > 64) return -EINVAL; + /* + * A DCP that also boots earlier firmware (T8142) lists the clock those + * use first and names the provider clocks "firmware-". Take only + * the named ones here. + */ + if (clocks && + of_property_match_string(dev->of_node, "clock-names", "firmware-0") >= 0) { + dcp->firmware_clocks = devm_kcalloc(dev, clocks, + sizeof(*dcp->firmware_clocks), + GFP_KERNEL); + if (!dcp->firmware_clocks) + return -ENOMEM; + for (index = 0; index < clocks; index++) { + dcp->firmware_clocks[index].id = + devm_kasprintf(dev, GFP_KERNEL, "firmware-%d", index); + if (!dcp->firmware_clocks[index].id) + return -ENOMEM; + } + ret = devm_clk_bulk_get(dev, clocks, dcp->firmware_clocks); + if (ret) + return ret; + dcp->num_firmware_clocks = clocks; + return 0; + } ret = devm_clk_bulk_get_all(dev, &dcp->firmware_clocks); if (ret < 0) return ret; @@ -6147,6 +6183,13 @@ static int dcp_platform_probe_pipe(struct platform_device *pdev) if (ret) return ret; dcp->hw = *(struct apple_dcp_hw_data *)of_device_get_match_data(dev); + /* + * T8142 on firmware 27 runs as the T6050's firmware-27 instance, with + * the default method profile: with H17G every swap fails with status 4. + */ + if (fw_compat == DCP_FIRMWARE_V_27_0 && + of_device_is_compatible(dev->of_node, "apple,t8142-dcp")) + dcp->hw.iomfb_method_profile = DCP_IOMFB_METHODS_DEFAULT; /* The display such a processor finds at boot is its first sink. */ if (dcp_sink_metadata(dcp)) dcp_desktop_metadata_new_sink(dcp); diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index 4cdcbeb8b32a1..b0db980938c7c 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -3321,13 +3321,21 @@ bool iomfb_apply_backlight_h17p(struct apple_dcp *dcp, dcp->brightness.maximum, false); return true; } - if (!transaction || !dcp->brightness.maximum) + if (!transaction || !dcp->brightness.maximum) { + /* The caller drops this present ("swap failed"): say why. */ + dev_warn_ratelimited(dcp->dev, + "present dropped: transaction %d brightness maximum %u\n", + !!transaction, dcp->brightness.maximum); return false; + } for (i = 0; i < SWAP_SURFACES; i++) have_surface |= !request->surf_null[i]; ret = dcp_backlight_prepare(dcp, have_surface, &present); - if (ret && ret != -EALREADY) + if (ret && ret != -EALREADY) { + dev_warn_ratelimited(dcp->dev, + "present dropped: backlight prepare %d\n", ret); return false; + } if (!ret) { transaction->backlight = present; transaction->backlight_reserved = true; diff --git a/drivers/gpu/drm/asahi/Makefile b/drivers/gpu/drm/asahi/Makefile index 169564ce10562..1a7fed6c11403 100644 --- a/drivers/gpu/drm/asahi/Makefile +++ b/drivers/gpu/drm/asahi/Makefile @@ -1,7 +1,7 @@ # SPDX-License-Identifier: GPL-2.0 obj-$(CONFIG_DRM_ASAHI) += aurora.o -aurora-y := asahi_core.o fwid.o legacy_sched.o sysfs.o g17/sync_input.o g17/sync_output.o g17/sync_scope.o g17/trace.o g17/clpc.o g17/fans.o +aurora-y := asahi_core.o fwid.o legacy_sched.o sysfs.o g17/sync_input.o g17/sync_output.o g17/sync_scope.o g17/trace.o g17/clpc.o g17/fans.o g17/pmpdash.o CFLAGS_g17/trace.o := -I$(src)/g17 aurora-$(CONFIG_DRM_ASAHI_G15) += g15/glue.o diff --git a/drivers/gpu/drm/asahi/driver.rs b/drivers/gpu/drm/asahi/driver.rs index 2b8bdac35f43d..ba6301fa1c595 100644 --- a/drivers/gpu/drm/asahi/driver.rs +++ b/drivers/gpu/drm/asahi/driver.rs @@ -547,6 +547,17 @@ impl AsahiPlatformDriver { fn probe_m5<'bound>( pdev: &'bound platform::Device>, ) -> Result> { + // Without aura's 25G76 profile (the macOS 27.0 stub's 26A428 firmware), take the M5 + // graph's G17 path (J813 bring-up). + let has_25g76 = of::chosen().is_some_and(|chosen| { + matches!( + chosen.get_opt_property::>(c"aurora,m5-gpu-firmware-profile"), + Ok(Some(_)) + ) + }); + if !has_25g76 { + return Self::probe_g17(pdev, regs::G17Resources::new(pdev)?, &hw::t8142::CONFIG); + } // SAFETY: The M5 UAT has 42 output bits. unsafe { pdev.dma_set_mask_and_coherent(DmaMask::try_new(42)?)? }; let resources = regs::Resources::new(pdev)?; diff --git a/drivers/gpu/drm/asahi/g17/buffer.rs b/drivers/gpu/drm/asahi/g17/buffer.rs index 0747a1fe69c87..7a6e1f9837942 100644 --- a/drivers/gpu/drm/asahi/g17/buffer.rs +++ b/drivers/gpu/drm/asahi/g17/buffer.rs @@ -581,8 +581,20 @@ impl MetricsLease { } let pitch = self.allocator.entry_pitch; let offset = self.slot.checked_mul(pitch).ok_or(EOVERFLOW)?; - let client = self.backing.map_range(vm, offset..offset + pitch, BlockGeometry::M5_COMPACT, - mmu::UAT_PGSZ as u64, mmu::PROT_GPU_SHARED_RW, 0)?; + // The J813's firmware touches a render's metrics in context 0, whose user root is the + // kernel lower VM, at the client's address. That VM maps the whole backing at the compact + // window's start (InitData::bind_m5_metrics), so place the entry where its offset puts + // it there. + let fixed = super::J813_FORMS.load(Ordering::Relaxed).then(|| { + let at = BlockGeometry::M5_COMPACT.start + offset as u64; + self.backing.map_range(vm, offset..offset + pitch, at..at + pitch as u64, + mmu::UAT_PGSZ as u64, mmu::PROT_GPU_SHARED_RW, 0).ok() + }).flatten(); + let client = match fixed { + Some(client) => client, + None => self.backing.map_range(vm, offset..offset + pitch, BlockGeometry::M5_COMPACT, + mmu::UAT_PGSZ as u64, mmu::PROT_GPU_SHARED_RW, 0)?, + }; self.addresses.client = client.iova(); self.client = Some(client); Ok(()) @@ -833,6 +845,10 @@ impl Manager { pub(crate) fn firmware_target(&self) -> usize { self.blocks().saturating_add(fw::GROW_BLOCKS).min(self.tvb.geometry.max_blocks()) } + /// The most blocks this buffer's geometry can hold. + pub(crate) fn max_blocks(&self) -> usize { + self.tvb.geometry.max_blocks() + } /// Read-only diagnostic while firmware is halted. In particular this /// does not reset the high-water counter as automatic_target() does. pub(crate) fn report_recovery(&self) { diff --git a/drivers/gpu/drm/asahi/g17/buffer/target.rs b/drivers/gpu/drm/asahi/g17/buffer/target.rs index 8561053a336f2..8b87ccde7d985 100644 --- a/drivers/gpu/drm/asahi/g17/buffer/target.rs +++ b/drivers/gpu/drm/asahi/g17/buffer/target.rs @@ -277,7 +277,7 @@ impl Target { Ok(crate::g17::fw::m5::render_registers::TaExtra { gtp1: pointers[1], gtp2: pointers[2], gtp4: pointers[4], gtp5: pointers[5], gtp6: pointers[6], gtp7: pointers[7], segment_control: control[0], segment_index: control[1], - min_free_tvb_pages: control[2], scs_bracket: 0, matrix_bracket: 0, + min_free_tvb_pages: control[2], scs_bracket: 0, matrix_bracket: 0, j813: false, }) } diff --git a/drivers/gpu/drm/asahi/g17/command.rs b/drivers/gpu/drm/asahi/g17/command.rs index 796d072364fd2..54ad8992994c5 100644 --- a/drivers/gpu/drm/asahi/g17/command.rs +++ b/drivers/gpu/drm/asahi/g17/command.rs @@ -199,7 +199,34 @@ impl<'a> Parser<'a> { } } + /// A refused submission reaches userspace as EINVAL alone, which names neither the command + /// the parser stopped at nor what it holds. Name both, for the first few of a boot. pub(crate) fn next(&mut self) -> Result> { + let head = self.bytes; + let result = self.next_parsed(); + if let Err(error) = &result { + static REFUSED: core::sync::atomic::AtomicU32 = core::sync::atomic::AtomicU32::new(0); + if REFUSED.fetch_add(1, core::sync::atomic::Ordering::Relaxed) < 8 { + match Reader::new(head).read::() { + Ok(header) => pr_warn!( + "G17: refused a submission ({:?}) at its command of type {}, {} bytes, \ + barriers [{:#x}, {:#x}], with {} of the submission's bytes left\n", + error, header.cmd_type, header.size, header.vdm_barrier, + header.cdm_barrier, head.len() + ), + Err(_) => pr_warn!( + "G17: refused a submission ({:?}): its remaining {} bytes are not a \ + command header\n", + error, + head.len() + ), + } + } + } + result + } + + fn next_parsed(&mut self) -> Result> { while !self.bytes.is_empty() { let mut reader = Reader::new(self.bytes); let header: uapi::drm_asahi_cmd_header = reader.read()?; @@ -284,6 +311,13 @@ impl<'a> Parser<'a> { pub(crate) fn finish(&self) -> Result { if !self.bytes.is_empty() || self.counts == [0; 2] { + pr_warn!( + "G17: refused a submission: {} bytes past its last command, {} renders and {} \ + dispatches\n", + self.bytes.len(), + self.counts[0], + self.counts[1] + ); return Err(EINVAL); } Ok(()) @@ -494,11 +528,39 @@ impl Payload { } } + /// A refused submission reaches userspace as EINVAL alone, which names neither the command + /// nor the requirement it missed. Print the fields the requirements below are written in, + /// as userspace sent them, for the first few refusals of a boot. pub(crate) fn validate( &self, window: &UscWindow, variant: &super::fw::Variant, space: &impl AddressSpace, + ) -> Result { + let result = self.validate_checked(window, variant, space); + if let (Err(error), Self::Render(cmd, ..)) = (&result, self) { + static REFUSED: core::sync::atomic::AtomicU32 = core::sync::atomic::AtomicU32::new(0); + if REFUSED.fetch_add(1, core::sync::atomic::Ordering::Relaxed) < 4 { + pr_warn!( + "G17: refused a render ({:?}): {}x{} px, {} layers, {} samples, {} B/sample, \ + utile {}x{}, flags {:#x}, vdm {:#x}, scissor {:#x}, dbias {:#x}, oclqry {:#x}, \ + usc bg {:#x} eot {:#x} partial bg {:#x} eot {:#x}, samplers {} at {:#x}\n", + error, cmd.width_px, cmd.height_px, cmd.layers, cmd.samples, + cmd.sample_size_B, cmd.utile_width_px, cmd.utile_height_px, cmd.flags, + cmd.vdm_ctrl_stream_base, cmd.isp_scissor_base, cmd.isp_dbias_base, + cmd.isp_oclqry_base, cmd.bg.usc, cmd.eot.usc, cmd.partial_bg.usc, + cmd.partial_eot.usc, cmd.sampler_count, cmd.sampler_heap + ); + } + } + result + } + + fn validate_checked( + &self, + window: &UscWindow, + variant: &super::fw::Variant, + space: &impl AddressSpace, ) -> Result { // Everything M5 accepts beyond the G17P command set is gated on this. let m5 = variant.m5_commands; diff --git a/drivers/gpu/drm/asahi/g17/context/identity.rs b/drivers/gpu/drm/asahi/g17/context/identity.rs index 0fe045d6f780c..e8e57cf3ad185 100644 --- a/drivers/gpu/drm/asahi/g17/context/identity.rs +++ b/drivers/gpu/drm/asahi/g17/context/identity.rs @@ -36,7 +36,9 @@ pub(super) struct WorkStates { impl WorkStates { pub(super) const fn new() -> Self { Self { - next: AtomicU32::new(0), + // Start at generation 1: macOS 27.0 never uses generation 0 (its words run + // 0x17c8.. 0x17ea on the J813). + next: AtomicU32::new(0x40), live: [const { AtomicU32::new(FREE) }; 64], held: [const { AtomicU32::new(0) }; ENGINES], } diff --git a/drivers/gpu/drm/asahi/g17/freelist.rs b/drivers/gpu/drm/asahi/g17/freelist.rs index 05dbf028cbc81..0f80469d074d4 100644 --- a/drivers/gpu/drm/asahi/g17/freelist.rs +++ b/drivers/gpu/drm/asahi/g17/freelist.rs @@ -839,7 +839,8 @@ impl RenderPool { let mut bytes = [0; super::fw::m5::pool::STATE_BYTES]; super::fw::m5::pool::encode_g17s_render_state(pages.m5_layout(self.id())?, self.state.gpu_va(), &mut bytes).map_err(|_| EINVAL)?; - bytes[super::fw::m5::pool::COMPLETED..super::fw::m5::pool::COMPLETED + 8] + let completed_at = super::fw::m5::pool::completed_offset(); + bytes[completed_at..completed_at + 8] .copy_from_slice(&submitted.to_le_bytes()); let raw = self.control.pointer(0, bytes.len())?; for (index, byte) in bytes.into_iter().enumerate() { @@ -891,7 +892,7 @@ impl RenderPool { /// counter, so retain the byte accesses required by that interface. pub(crate) fn counters(&self) -> Result<[u64; 2]> { let submitted = self.state.dword(0)?.load(Ordering::Relaxed); - let offset = if self.is_m5() { super::fw::m5::pool::COMPLETED } + let offset = if self.is_m5() { super::fw::m5::pool::completed_offset() } else { FreeListControl::COMPLETED }; let completed = self.control.pointer(offset, 8)?; let mut bytes = [0; 8]; @@ -1073,7 +1074,8 @@ impl ComputePool { use super::fw::m5::pool; pool::encode_g17s_compute_state(pages.m5_layout(id)?, submitted, &mut control.bytes_mut()[..pool::STATE_BYTES]).map_err(|_| EINVAL)?; - control.bytes_mut()[pool::COMPLETED..pool::COMPLETED + 8].copy_from_slice(&completed); + let completed_at = pool::completed_offset(); + control.bytes_mut()[completed_at..completed_at + 8].copy_from_slice(&completed); Ok(()) } PageLayout::Neo { blocks, .. } => { @@ -1094,7 +1096,7 @@ impl ComputePool { Ok(u64::from_le_bytes(self.completed_bytes()?)) } fn completed_bytes(&self) -> Result<[u8; 8]> { - let offset = if self.m5 { super::fw::m5::pool::COMPLETED } else { FreeListControl::COMPLETED }; + let offset = if self.m5 { super::fw::m5::pool::completed_offset() } else { FreeListControl::COMPLETED }; let pointer = self.control.pointer(offset, 8)?; let mut bytes = [0; 8]; for (index, byte) in bytes.iter_mut().enumerate() { diff --git a/drivers/gpu/drm/asahi/g17/fw/compute.rs b/drivers/gpu/drm/asahi/g17/fw/compute.rs index 76f4f96bd6ea6..94d8f44edc7bc 100644 --- a/drivers/gpu/drm/asahi/g17/fw/compute.rs +++ b/drivers/gpu/drm/asahi/g17/fw/compute.rs @@ -200,6 +200,17 @@ impl OperandState { r0a5c9: 0x9000, size: 0x14000, }; + /// One cluster of ten cores (M5, J813), by ONE_BY_SIX's and TWO_BY_TEN's pattern: a bank of + /// ten slots of 0x1800, one tail of 0x800, the end rounded up to 16 KiB. + pub(crate) const ONE_BY_TEN: OperandState = OperandState { + r10229: 0x1e800, + r140a8: 0x1f000, + r10099: 0xf405, + r10091: 0x1e400, + r0a5c1: 0x0005, + r0a5c9: 0xf000, + size: 0x20000, + }; /// Two clusters of ten cores (M4 Pro): bank stride 0x1e400, tails 0x1000. pub(crate) const TWO_BY_TEN: OperandState = OperandState { r10229: 0x3c800, diff --git a/drivers/gpu/drm/asahi/g17/fw/m5/compute.rs b/drivers/gpu/drm/asahi/g17/fw/m5/compute.rs index 1d969ffa42e8f..a2d7205dfe9af 100644 --- a/drivers/gpu/drm/asahi/g17/fw/m5/compute.rs +++ b/drivers/gpu/drm/asahi/g17/fw/m5/compute.rs @@ -32,9 +32,17 @@ pub(crate) const COMPUTE_PIPELINED_DEPENDENCY: u8 = 2; /// The completion class, which holds a kick until the kick it names has completed. pub(crate) const COMPLETION_DEPENDENCY: u8 = 1; -/// Dependency class of an explicit dependency of a compute kick on `board`. +/// Dependency class of an explicit dependency of a compute kick on `board`: the completion class +/// where explicit waits take it ([`Board::completion_inputs`]) and on the J813, where macOS 27.0 +/// gives every dependency of every kick class 1, two bits each at kick+0x180 (0x01, 0x05, 0x15, +/// 0x55 for one to four), in its TA, 3D and compute kicks alike. Otherwise it is the pipelined +/// class. pub(crate) fn explicit_dependency(board: &Board) -> u8 { - if board.completion_inputs() { COMPLETION_DEPENDENCY } else { COMPUTE_PIPELINED_DEPENDENCY } + if board.completion_inputs() || board.j813_forms() { + COMPLETION_DEPENDENCY + } else { + COMPUTE_PIPELINED_DEPENDENCY + } } /// Dependency class of a compute kick's implicit parent (the previous kick of its queue) on @@ -53,10 +61,21 @@ pub(crate) fn parent_dependency(board: &Board, parent_retired: bool) -> u8 { } /// Whether compute kicks on `board` count their completion ordinal and hold for their parent: -/// with the G17D forms, and on boards whose explicit waits take the completion class -/// ([`Board::completion_inputs`]), as macOS encodes M5 Pro compute. +/// with the G17D forms, on boards whose explicit waits take the completion class +/// ([`Board::completion_inputs`]), as macOS encodes M5 Pro compute, and on the J813, whose +/// macOS 27.0 compute descriptors count it (+0xf70 runs 0x341c, 0x341d, ...). pub(crate) fn counted_forms(board: &Board) -> bool { - board.g17d_forms() || board.completion_inputs() + board.g17d_forms() || board.completion_inputs() || board.j813_forms() +} + +/// The descriptor's completion identity and event metadata on `board`. macOS 27.0 on the J813 +/// writes the same two values into every compute descriptor it kicks. +pub(crate) fn completion_forms(board: &Board) -> (u64, [u8; 16]) { + if board.j813_forms() { + (0x0400_0000_0000_0002, [0, 0, 0x55, 0x01, 0, 0, 0, 0, 0, 0, 0x85, 0, 0, 0, 0, 0]) + } else { + (1, [0; 16]) + } } /// The completion ordinal of a compute command on `board`. With the counted forms @@ -79,10 +98,18 @@ const ENTRY_GATE_EVENT: u32 = 254; /// The event mask of a compute kick of context `context_id` on `board`, published with its /// current-kick notification. With the G17D forms ([`Board::g17d_forms`]) the kick waits on its /// own context's pool event (64 x 2 + the context), which the firmware holds while it processes -/// that pool's dependencies, and on the entry-signal gate; a context of 64 or more is EINVAL. With -/// the other counted forms ([`counted_forms`]) it is the pool event alone. Otherwise it is the -/// fixed `[0, 0, 8, 0]` (event 131, context 3's pool event, without the gate). +/// that pool's dependencies, and on the entry-signal gate; a context of 64 or more is EINVAL. +/// On the J813 it is event 64 + the context, in word 1, without the gate, as macOS 27.0 kicks it +/// there (WindowServer's context 1 kicks with 0x2 in word 1 and nothing else). With the other +/// counted forms ([`counted_forms`]) it is the pool event alone. Otherwise it is the fixed +/// `[0, 0, 8, 0]` (event 131, context 3's pool event, without the gate). pub(crate) fn kick_event_mask(board: &Board, context_id: u32) -> Result<[u64; 4]> { + if board.j813_forms() { + if context_id >= POOL_EVENT_CONTEXTS { + return Err(EINVAL); + } + return Ok([0, 1u64 << context_id, 0, 0]); + } if !counted_forms(board) { return Ok([0, 0, 8, 0]); } @@ -116,13 +143,22 @@ pub(crate) const NORMAL_TUPLE: KickTuple = pub(crate) const QUALIFIED_TUPLE: KickTuple = KickTuple { preempt_state_offset: 0x1480, dispatch_base: 0x10c0_8860, auxiliary_control: 0 }; -/// The DRM compute tuple of `board` ([`Board::g17d_forms`]) for a kick whose parent takes class -/// `parent_class`. Where explicit waits take the completion class ([`Board::completion_inputs`]) -/// it is macOS's M5 Pro form: the J714 preemption offset with the qualified dispatch base, and the -/// auxiliary control set only when the kick's parent is pipelined. +/// The tuple macOS 27.0 kicks the J813's compute with: the qualified preemption-state offset and +/// auxiliary control with the normal dispatch base (0x1a4d0 = preempt + 0x1480, 0x1a458 = +/// 0x14c08a60, 0xa731 = 0 in every WindowServer compute descriptor). +pub(crate) const J813_TUPLE: KickTuple = + KickTuple { preempt_state_offset: 0x1480, dispatch_base: 0x14c0_8a60, auxiliary_control: 0 }; + +/// The DRM compute tuple of `board` ([`Board::g17d_forms`], [`Board::j813_forms`]) for a kick +/// whose parent takes class `parent_class`. Where explicit waits take the completion class +/// ([`Board::completion_inputs`]) it is macOS's M5 Pro form: the J714 preemption offset with the +/// qualified dispatch base, and the auxiliary control set only when the kick's parent is +/// pipelined. pub(crate) fn kick_tuple(board: &Board, parent_class: u8) -> KickTuple { if board.g17d_forms() { QUALIFIED_TUPLE + } else if board.j813_forms() { + J813_TUPLE } else if board.completion_inputs() { KickTuple { preempt_state_offset: NORMAL_TUPLE.preempt_state_offset, @@ -437,6 +473,8 @@ pub(crate) struct DescriptorArgs { pub(crate) event_metadata: [u8; 16], pub(crate) usage_fw_va: u64, pub(crate) completion_identity: u64, + /// The common 26A428 firmware's tail ([`Descriptor::tail`]). + pub(crate) common26_tail: bool, } /// The M5 descriptor has a shorter reserved region before shared-control @@ -473,22 +511,24 @@ pub(crate) struct Descriptor { status_va: [u64; 2], timestamp_va: [u64; 2], reserved_f9c: [u8; 0x10], - shared_control_va: u64, - shared_control_enabled: u8, - event_metadata: [u8; 16], - usage_fw_va: u64, - completion_identity: u64, - reserved_fd5: [u8; 0x2b], + /// From 0xfac: the shared-control pointer and its enable byte, the event metadata, the usage + /// object and the completion identity, packed in that order. The common 26A428 firmware (the + /// J813's) reads them [`COMMON26_TAIL_INSERT`] bytes later: it keeps a count at 0xfac of the + /// shared-control bytes it is yet to account for, which it rounds up to eight, adds to the + /// shared-control object's own total and clears (gfx 0x100c4..0x10110 use 0xfac, 0xfb2 and + /// 0xfcb there, never 0xfc5 or 0xfcd). + tail: [u8; DESCRIPTOR_PREFIX_BYTES - 0xfac], } +/// The bytes the common 26A428 firmware inserts before a compute descriptor's tail. +const COMMON26_TAIL_INSERT: usize = 6; + static_assert!(size_of::() == DESCRIPTOR_PREFIX_BYTES); static_assert!(core::mem::offset_of!(Descriptor, registers) == 0x40); static_assert!(core::mem::offset_of!(Descriptor, alternate) == 0x760); static_assert!(core::mem::offset_of!(Descriptor, stamp_va) == 0xf40); static_assert!(core::mem::offset_of!(Descriptor, completion_ordinal) == 0xf70); -static_assert!(core::mem::offset_of!(Descriptor, shared_control_va) == 0xfac); -static_assert!(core::mem::offset_of!(Descriptor, usage_fw_va) == 0xfc5); -static_assert!(core::mem::offset_of!(Descriptor, completion_identity) == 0xfcd); +static_assert!(core::mem::offset_of!(Descriptor, tail) == 0xfac); // SAFETY: Every field consists only of integer values and their arrays. unsafe impl Zeroable for Descriptor {} @@ -539,11 +579,14 @@ impl Descriptor { self.completion_ordinal = args.completion_ordinal; self.status_va = args.status_va; self.timestamp_va = args.timestamp_va; - self.shared_control_va = args.shared_control_va; - self.shared_control_enabled = u8::from(args.shared_control_va != 0); - self.event_metadata = args.event_metadata; - self.usage_fw_va = args.usage_fw_va; - self.completion_identity = args.completion_identity; + let mut at = if args.common26_tail { COMMON26_TAIL_INSERT } else { 0 }; + for field in [&args.shared_control_va.to_le_bytes()[..], + &[u8::from(args.shared_control_va != 0)], &args.event_metadata, + &args.usage_fw_va.to_le_bytes(), &args.completion_identity.to_le_bytes()] + { + self.tail[at..at + field.len()].copy_from_slice(field); + at += field.len(); + } Ok(()) } } diff --git a/drivers/gpu/drm/asahi/g17/fw/m5/initdata.rs b/drivers/gpu/drm/asahi/g17/fw/m5/initdata.rs index ed397b27917cb..e4de37f639579 100644 --- a/drivers/gpu/drm/asahi/g17/fw/m5/initdata.rs +++ b/drivers/gpu/drm/asahi/g17/fw/m5/initdata.rs @@ -9,6 +9,8 @@ use core::ops::Range; use core::result::Result; +use crate::g17::fw::kick::{KickTimestamp, KICK_SLOTS}; + pub(crate) const ROOT_SIZE_PRIMARY: usize = 0x0b8; pub(crate) const ROOT_SIZE_SECONDARY: usize = 0x0c8; @@ -94,6 +96,14 @@ impl RootProfile { } } + /// The GPU-visible aliases of the firmware's own tables the main config names. + pub(crate) const fn region_view_low_addrs(self) -> [u64; 2] { + match self { + Self::J873G18G26A428 => COMMON26_REGION_VIEW_LOW_ADDRS, + Self::J714G17S26A428 | Self::J700G17P | Self::J813G17G25G76 => REGION_VIEW_LOW_ADDRS, + } + } + pub(crate) const fn repeated_region_bundle_offset(self) -> usize { match self { Self::J873G18G26A428 => COMMON26_SHARED_LAYOUT.block_88, @@ -206,6 +216,9 @@ pub(crate) const BUNDLE_VIEW_OFFSETS: [usize; 5] = [0x2740, 0x3380, 0x4400, 0xbc pub(crate) const BUNDLE_VIEW_EXTENTS: [usize; 5] = [0x18c0, 0x0c80, 0x3c00, 0x3380, 0x3300]; pub(crate) const REGION_VIEW_LOW_ADDRS: [u64; 2] = [0x70_0183_8000, 0x70_0184_0000]; +/// The GPU-visible aliases of the firmware's own tables on the J813: macOS 27.0 publishes them +/// 8 GiB above where the 25G76 trace had them (the J813's own completion table, read under aura). +pub(crate) const COMMON26_REGION_VIEW_LOW_ADDRS: [u64; 2] = [0x72_0183_8000, 0x72_0184_0000]; pub(crate) const CHANNEL_TABLE_ENTRIES: usize = 17; @@ -841,9 +854,9 @@ pub(crate) fn encode_main_config_for_profile( } put_u64(out, MAIN_REGION_VIEWS, views.sentinel_high); - put_u64(out, MAIN_REGION_VIEWS + 0x08, REGION_VIEW_LOW_ADDRS[0]); + put_u64(out, MAIN_REGION_VIEWS + 0x08, profile.region_view_low_addrs()[0]); put_u64(out, MAIN_REGION_VIEWS + 0x10, views.alias_high[0]); - put_u64(out, MAIN_REGION_VIEWS + 0x18, REGION_VIEW_LOW_ADDRS[1]); + put_u64(out, MAIN_REGION_VIEWS + 0x18, profile.region_view_low_addrs()[1]); put_u64(out, MAIN_REGION_VIEWS + 0x20, views.alias_high[1]); // J714's firmware takes both primary sentinels too; they are not // exclusive to the older J700 ABI. @@ -857,8 +870,8 @@ pub(crate) fn encode_main_config_for_profile( } (InstanceRole::Secondary, _) => { - put_u64(out, MAIN_REGION_VIEWS + 0x08, REGION_VIEW_LOW_ADDRS[0]); - put_u64(out, MAIN_REGION_VIEWS + 0x18, REGION_VIEW_LOW_ADDRS[1]); + put_u64(out, MAIN_REGION_VIEWS + 0x08, profile.region_view_low_addrs()[0]); + put_u64(out, MAIN_REGION_VIEWS + 0x18, profile.region_view_low_addrs()[1]); put_u32(out, MAIN_SECONDARY_KIND, match profile { RootProfile::J700G17P => MAIN_SECONDARY_KIND_VALUE, RootProfile::J813G17G25G76 => 0xc, @@ -1538,9 +1551,13 @@ fn encode_m5_aux_perf_states(board: &Board, out: &mut [u8]) { let table = &mut out[M5_HW_AUX_PERF_STATE.. M5_HW_AUX_PERF_STATE + M5_HW_AUX_PERF_STATE_SIZE]; table.fill(0); + // Without CS/AFR domains (M5) the table stays zero, as in the J813's 25G76 HW data. + let Some(aux_perf_states) = board.aux_perf_states else { + return; + }; put_u32(table, 0, 13); put_u32(table, 4, 14); - for (state, &(frequency, core, sram)) in board.aux_perf_states.iter().enumerate() { + for (state, &(frequency, core, sram)) in aux_perf_states.iter().enumerate() { put_u32(table, 8 + state * 4, frequency); for lane in 0..2 { put_u32(table, 0x48 + state * 8 + lane * 4, core[lane]); @@ -1612,6 +1629,9 @@ const J714_AON_RECORDS: [[u32; 3]; 3] = [ [0x7230_5452, 0x00ce_0001, 0x0000_1401], ]; +// J813's row tagged RT0g (one GPU cluster), as its 25G76 AON block carries. +const J813_AON_RECORDS: [[u32; 3]; 1] = [[0x6730_5452, 0x001d_0001, 0x0000_1401]]; + // J716c's rows tagged RT0g, RT1g and RT0r. const J716C_AON_RECORDS: [[u32; 3]; 3] = [ [0x6730_5452, 0x0058_0001, 0x0000_1401], @@ -1675,7 +1695,8 @@ pub(crate) const M5_HW_DATA_SIZE: usize = ABI_ALLOCATION_SIZES_26A428_M5[ALLOC_H pub(crate) const M5_KICK_SHARED_SIZE: usize = 0x2000; pub(crate) const M5_KICK_SHARED_POINTER: usize = 0x27dc; -pub(crate) fn set_m5_kick_shared(out: &mut [u8], address: u64) -> Result<(), InitdataError> { +pub(crate) fn set_m5_kick_shared(board: &Board, out: &mut [u8], address: u64) + -> Result<(), InitdataError> { if out.len() != M5_HW_DATA_SIZE { return Err(InitdataError::BufferSize); } @@ -1686,7 +1707,7 @@ pub(crate) fn set_m5_kick_shared(out: &mut [u8], address: u64) -> Result<(), Ini if address & (UAT_PAGE_SIZE as u64 - 1) != 0 { return Err(InitdataError::AddressOutOfRange); } - put_u64(out, M5_KICK_SHARED_POINTER, address); + put_u64(out, hw_offset(board, M5_KICK_SHARED_POINTER), address); Ok(()) } @@ -1753,23 +1774,361 @@ pub(crate) const M5_HW_DATA_INSERT: usize = 0x267c; pub(crate) const M5_HW_DATA_INSERT_LEN: usize = 8; +/// The common 26A428 private cluster (J873, J813): status config 0x6d0 bytes, the secondary +/// root's second extra 0x60. +pub(crate) const COMMON26_PRIVATE_LAYOUT: PrivateClusterLayout = + private_cluster_layout(&ABI_ALLOCATION_SIZES_26A428); + +pub(crate) const COMMON26_POLICY_SIZE: usize = ABI_ALLOCATION_SIZES_26A428[ALLOC_POLICY]; + +const COMMON26_STATUS_CONFIG_SIZE: usize = ABI_ALLOCATION_SIZES_26A428[ALLOC_STATUS_CONFIG]; + +const COMMON26_STATUS_CONFIG_IN_STATUS_B: usize = + COMMON26_PRIVATE_LAYOUT.status_config - COMMON26_PRIVATE_LAYOUT.status_b; + +const COMMON26_STATUS_B_SPAN: usize = + COMMON26_PRIVATE_LAYOUT.roles[0].status_a - COMMON26_PRIVATE_LAYOUT.status_b; + +/// Status config +0x6c4 = 1 and +0x6c8 = the FW VA of status B +0xe71c. +const COMMON26_STATUS_CONFIG_LINK_WORD: usize = 0x6c4; +const COMMON26_STATUS_CONFIG_LINK_PTR: usize = 0x6c8; + +/// The policy block's FW VA of the private-cluster policy state (25G76 +0x3c98). +const COMMON26_POLICY_STATE_PTR: usize = 0x3d30; +/// The policy block's FW VA of the AON PTD (25G76 +0x6460). +const COMMON26_POLICY_AON_PTD_PTR: usize = 0x64d8; +/// The policy block's boot mach_absolute_time (25G76 +0x651c). +const COMMON26_POLICY_BOOT_TIMESTAMP: usize = 0x6c14; + +/// The 25G76 policy block (relative to the block) in the common 26A428 one, by the kext's +/// initPowerAndPerformanceData stores (the G18 backend's G18G_26A428_POLICY_BANDS). 26A428 +/// inserts 0x680 bytes between the AON PTD pointer and the boot timestamp; the 25G76 words +/// there have no place. +const COMMON26_POLICY_BANDS: &[LayoutBand] = &[ + band(0x0000, 0x009c, 0, Evidence::ProducerChecked), + band(0x00a0, 0x0200, 0x10, Evidence::ProducerChecked), + band(0x09b8, 0x295c, 0x10, Evidence::ProducerChecked), + band(0x2960, 0x29c1, 0x29, Evidence::ProducerChecked), + band(0x2a01, 0x2a05, 0x8d, Evidence::ProducerChecked), + band(0x2a08, 0x2e5c, 0x90, Evidence::ProducerChecked), + band(0x2e6c, 0x2e70, 0x9c, Evidence::ProducerChecked), + band(0x2e78, 0x2e7c, 0x74, Evidence::Inferred), + band(0x2f37, 0x2f43, -0x2b, Evidence::ProducerChecked), + band(0x2f43, 0x2f53, 0x71, Evidence::ProducerChecked), + band(0x2f58, 0x3260, 0x70, Evidence::ProducerChecked), + band(0x3c98, 0x3cac, 0x98, Evidence::ProducerChecked), + band(0x5cac, 0x5cb4, 0x98, Evidence::ProducerChecked), + band(0x63a4, 0x63f4, 0xa0, Evidence::ProducerChecked), + band(0x6458, 0x645c, 0x78, Evidence::ProducerChecked), + band(0x6460, 0x6468, 0x78, Evidence::ProducerChecked), + band(0x651c, 0x66a8, 0x6f8, Evidence::ProducerChecked), + band(0x6a18, 0x6a1c, 0x710, Evidence::ProducerChecked), +]; + +/// Policy fields the common 26A428 interface added: macOS sets them on J873 where the +/// relocated 25G76 words leave zeros. +0x64e0 is the AON row tag (RT0g, set from the board). +const COMMON26_POLICY_NEW_WORDS: [(usize, u32); 10] = [ + (0x009c, 1), (0x00a0, 1), (0x00a4, 1), (0x00a8, 1), + (0x0104, COMMON26_PERF_SCALE_VALUE), (0x0b4c, 1), + (0x2a88, 0x7148_0000), (0x2a90, 0x0064_0000), (0x30b0, 0x42c8_0000), + (0x6520, 1), +]; + +/// Status-config fields the common 26A428 interface added, as macOS sets them on J873: +/// +0x1dc/+0x1e0 after the secondary power-control inputs, +0x4e8 the two table enables. +const COMMON26_STATUS_CONFIG_NEW_WORDS: [(usize, u32); 3] = + [(0x1dc, 1), (0x1e0, 29000), (0x4e8, 0x0001_0001)]; + +/// Region-C fields the common 26A428 interface added, as macOS sets them on J873 (+0x38 is +/// the new feature bit; the M5 Pro's region C carries +0x9dc and +0xe0c at +0x9e0 and +/// +0x1c30). +const COMMON26_REGION_C_NEW_WORDS: [(usize, u32); 4] = + [(0x038, 1), (0x9b0, 0x200), (0x9dc, 50), (0xe0c, 250)]; + +/// The InitData layout a board's firmware build reads. +pub(crate) struct Graph { + /// The root, main-config and channel-table encodings. + pub(crate) profile: RootProfile, + pub(crate) private: PrivateClusterLayout, + pub(crate) shared: SharedClusterLayout, + pub(crate) policy_size: usize, + /// The five shared blocks main +0x254 points at. + pub(crate) bundle_view_offsets: [usize; 5], + pub(crate) bundle_view_extents: [usize; 5], + pub(crate) work_ring_offsets: [usize; WORK_CHANNEL_COUNT], + /// Each role's channel-state grid in the private cluster. + pub(crate) state_grids: [usize; 2], + /// Region C's power-assert tally. + pub(crate) power_assert_tally: usize, +} + +const fn state_grids(layout: &PrivateClusterLayout) -> [usize; 2] { + [ + layout.dm_ring_states[0], + layout.secondary_control_state - WORK_STATE_GRID_OFFSETS[CONTROL_CHANNEL_INDEX], + ] +} + +/// The M5 Pro's 26A428 build (J714, J716c, J775D). +pub(crate) const M5_GRAPH: Graph = Graph { + profile: RootProfile::J714G17S26A428, + private: M5_PRIVATE_LAYOUT, + shared: M5_SHARED_LAYOUT, + policy_size: M5_POLICY_SIZE, + bundle_view_offsets: M5_BUNDLE_VIEW_OFFSETS, + bundle_view_extents: M5_BUNDLE_VIEW_EXTENTS, + work_ring_offsets: M5_WORK_RING_BUNDLE_OFFSETS, + state_grids: M5_PRIVATE_STATE_GRIDS, + power_assert_tally: super::power::POWER_ASSERT_TALLY, +}; + +/// The common 26A428 build (J873's interface; the J813). +pub(crate) const COMMON26_GRAPH: Graph = Graph { + profile: RootProfile::J873G18G26A428, + private: COMMON26_PRIVATE_LAYOUT, + shared: COMMON26_SHARED_LAYOUT, + policy_size: COMMON26_POLICY_SIZE, + bundle_view_offsets: COMMON26_BUNDLE_VIEW_OFFSETS, + bundle_view_extents: COMMON26_BUNDLE_VIEW_EXTENTS, + work_ring_offsets: COMMON26_WORK_RING_BUNDLE_OFFSETS, + state_grids: state_grids(&COMMON26_PRIVATE_LAYOUT), + // The M5 Pro's tally sits in the block its region C moves from common +0xe10 to +0x1c34. + power_assert_tally: super::power::POWER_ASSERT_TALLY - 0x1c34 + 0xe10, +}; + +const _: () = { + assert!(M5_GRAPH.state_grids[0] == M5_PRIVATE_STATE_GRIDS[0]); + assert!(M5_GRAPH.state_grids[1] == state_grids(&M5_PRIVATE_LAYOUT)[1]); + assert!(COMMON26_PRIVATE_LAYOUT.total == 0x178000); + assert!(COMMON26_STATUS_CONFIG_IN_STATUS_B + COMMON26_STATUS_CONFIG_SIZE + <= COMMON26_STATUS_B_SPAN); +}; + +impl Graph { + pub(crate) fn is_common26(&self) -> bool { + self.profile == RootProfile::J873G18G26A428 + } +} + +/// Encode the common 26A428 primary status-B span: status B, status config and the gap to +/// status A. The status config carries the board's 25G76 words relocated, the fields 26A428 +/// added and its new cross-link to status B. +fn encode_common26_primary_status_b( + fwctl_state: u64, + fwctl_ring: u64, + status_b: u64, + out: &mut [u8], +) -> Result<(), InitdataError> { + if out.len() != COMMON26_STATUS_B_SPAN { + return Err(InitdataError::BufferSize); + } + required_address(fwctl_state)?; + required_address(fwctl_ring)?; + required_address(status_b)?; + let config = COMMON26_STATUS_CONFIG_IN_STATUS_B; + out.fill(0); + encode_status_block(&mut out[..STATUS_BLOCK_SIZE])?; + put_u64(out, COMMON26_STATUS_B_FWCTL_STATE, fwctl_state); + put_u64(out, COMMON26_STATUS_B_FWCTL_RING, fwctl_ring); + put_u32(out, COMMON26_STATUS_B_CONFIG_HEADER, 1); + put_u32(out, COMMON26_STATUS_B_LINK_WORD, 0); + put_u64(out, COMMON26_STATUS_B_LINK_PTR, + status_b + (config + COMMON26_STATUS_CONFIG_LINK_WORD) as u64); + let status_config = &mut out[config..config + COMMON26_STATUS_CONFIG_SIZE]; + write_relocated_words(M5_25G76_STATUS_CONFIG_WORDS, 0, COMMON26_STATUS_CONFIG_BANDS, + status_config)?; + for (offset, value) in COMMON26_STATUS_CONFIG_NEW_WORDS { + put_u32(status_config, offset, value); + } + // 26A428 moved the last of the six 25G76 words at +0x588..+0x59c to the second place: + // on J873 +0x59c..+0x5b8 read 1, 700, 12, 5000, 2000, 5000, 0, 0, where the bands carry + // the board's six words to +0x5a4. + let words: [u32; 6] = core::array::from_fn(|index| { + let at = 0x5a4 + index * 4; + u32::from_le_bytes(status_config[at..at + 4].try_into().unwrap_or([0; 4])) + }); + for (index, value) in [words[0], words[5], words[1], words[2], words[3], words[4], 0, 0] + .into_iter().enumerate() { + put_u32(status_config, 0x59c + index * 4, value); + } + // The two 16x16 tables 26A428 copies in at +0x1e8 and +0x3e8 (the 25G76 words have none): + // zero first row and column, linear elsewhere, as on J873. + for row in 0..16 { + for col in 0..16 { + let active = row != 0 && col != 0; + put_u16(status_config, 0x1e8 + 2 * (row * 16 + col), + if active { (col * 0x1000 - 1) as u16 } else { 0 }); + status_config[0x3e8 + row * 16 + col] = if active { (col * 0x10 - 1) as u8 } else { 0 }; + } + } + put_u32(status_config, COMMON26_STATUS_CONFIG_LINK_WORD, 1); + put_u64(status_config, COMMON26_STATUS_CONFIG_LINK_PTR, + status_b + COMMON26_STATUS_B_LINK_WORD as u64); + Ok(()) +} + +/// Encode the common 26A428 private cluster mapped at `private_base`: the status-B span, both +/// status A blocks, the secondary root's 0x60 object and one opening-producer entry per control +/// channel. +pub(crate) fn encode_common26_private_cluster( + private_base: u64, + fwctl_state: u64, + out: &mut [u8], +) -> Result<(), InitdataError> { + let layout = COMMON26_PRIVATE_LAYOUT; + if out.len() != layout.total { + return Err(InitdataError::BufferSize); + } + required_address(private_base)?; + let private_end = private_base + .checked_add(layout.total as u64) + .ok_or(InitdataError::AddressOutOfRange)?; + required_address(private_end - 1)?; + required_address(fwctl_state)?; + let fwctl_ring = fwctl_state + .checked_add(COMMON26_FWCTL_RING_OFFSET as u64) + .ok_or(InitdataError::AddressOutOfRange)?; + required_address(fwctl_ring)?; + if fwctl_state < private_end && private_base < fwctl_ring + CONTROL_RECORD_SIZE as u64 { + return Err(InitdataError::PairAliasedObject); + } + out.fill(0); + let status_b = layout.status_b; + let status_a = layout.roles[0].status_a; + encode_common26_primary_status_b(fwctl_state, fwctl_ring, private_base + status_b as u64, + &mut out[status_b..status_a])?; + for role in layout.roles { + encode_status_block(&mut out[role.status_a..role.status_a + STATUS_BLOCK_SIZE])?; + } + let extra = layout.secondary_extra_1; + encode_secondary_root_extra_1_for_profile(RootProfile::J873G18G26A428, + &mut out[extra..extra + STATUS_BLOCK_SIZE])?; + put_u32(out, layout.control_state + CHANNEL_STATE_PRODUCER * CHANNEL_STATE_SPACING, + CONTROL_OPENING_PRIMARY_PRODUCER); + put_u32(out, layout.secondary_control_state + CHANNEL_STATE_PRODUCER * CHANNEL_STATE_SPACING, + CONTROL_OPENING_SECONDARY_PRODUCER); + Ok(()) +} + +/// Encode the six common 26A428 blocks after the HW data (0xc18, 0x1048, 0xe10, perf/policy, +/// 0x68, 0x800): the board's 25G76 policy words relocated, the fields 26A428 added and the owned +/// pointers. Preserves the HW data and everything from the BRN table on. +pub(crate) fn encode_common26_bundle_policy( + board: &Board, + bindings: M5PolicyBindings, + out: &mut [u8], +) -> Result<(), InitdataError> { + let layout = COMMON26_SHARED_LAYOUT; + if out.len() < layout.brn_table { + return Err(InitdataError::BufferSize); + } + required_address(bindings.state)?; + required_address(bindings.aon_ptd)?; + let state_end = bindings.state.checked_add(STATUS_BLOCK_SIZE as u64) + .ok_or(InitdataError::AddressOutOfRange)?; + let aon_end = bindings.aon_ptd.checked_add(M5_AON_PTD_SIZE as u64) + .ok_or(InitdataError::AddressOutOfRange)?; + required_address(state_end - 1)?; + required_address(aon_end - 1)?; + if bindings.state < aon_end && bindings.aon_ptd < state_end { + return Err(InitdataError::PairAliasedObject); + } + out[layout.block_c18..layout.brn_table].fill(0); + let m5 = M5_25G76_SHARED_LAYOUT; + let m5_policy_end = m5.policy + ABI_ALLOCATION_SIZES_25G76[ALLOC_POLICY]; + for &(offset, value) in M5_25G76_POLICY_WORDS { + if (m5.block_1048..m5.block_e10).contains(&offset) { + put_u32(out, layout.block_1048 + (offset - m5.block_1048), value); + } else if !(m5.policy..m5_policy_end).contains(&offset) { + return Err(InitdataError::BufferSize); + } + } + let policy = &mut out[layout.policy..layout.policy + COMMON26_POLICY_SIZE]; + for &(offset, value) in M5_25G76_POLICY_WORDS { + if (m5.policy..m5_policy_end).contains(&offset) { + write_relocated_words(&[(offset, value)], m5.policy, COMMON26_POLICY_BANDS, policy)?; + } + } + // The 25G76 block's 512 x 0x7fff u16 table at +0x1c97 moves with its band. + for index in 0..512 { + for (byte_index, byte) in 0x7fffu16.to_le_bytes().into_iter().enumerate() { + let source = 0x1c97 + index * 2 + byte_index; + let target = relocate_by_bands(COMMON26_POLICY_BANDS, source) + .ok_or(InitdataError::BufferSize)?; + *policy.get_mut(target).ok_or(InitdataError::BufferSize)? = byte; + } + } + for (offset, value) in COMMON26_POLICY_NEW_WORDS { + put_u32(policy, offset, value); + } + // The AON row the policy names: the board's first record (RT0g on J813 and J873). + let record = board.aon_records.first().ok_or(InitdataError::BufferSize)?; + put_u32(policy, 0x64e0, record[0]); + put_u64(policy, COMMON26_POLICY_STATE_PTR, bindings.state); + put_u64(policy, COMMON26_POLICY_AON_PTD_PTR, bindings.aon_ptd); + put_u64(policy, COMMON26_POLICY_BOOT_TIMESTAMP, bindings.boot_timestamp); + Ok(()) +} + +/// Encode a common 26A428 region C from the board's 25G76 one: the fields from +0x38 on move up +/// by four, plus the fields 26A428 added. Unlike [`encode_common26_region_c`] it keeps the +/// board's own tunables (standby timer, minimum confidence, state counts). +pub(crate) fn encode_common26_board_region_c(out: &mut [u8]) -> Result<(), InitdataError> { + if out.len() != REGION_C_SIZE { + return Err(InitdataError::BufferSize); + } + encode_region_c_for_profile(RootProfile::J813G17G25G76, out)?; + let shifted = ABI_ALLOCATION_SIZES_25G76[ALLOC_REGION_C]; + if out[shifted..].iter().any(|byte| *byte != 0) { + return Err(InitdataError::BufferSize); + } + out.copy_within(0x38..shifted, 0x3c); + out[0x38..0x3c].fill(0); + for (offset, value) in COMMON26_REGION_C_NEW_WORDS { + put_u32(out, offset, value); + } + Ok(()) +} + +/// The board's offset of a field at `offset` of the M5 Pro's HW data: below the insert they +/// agree, past it a board without the insert has the field eight bytes lower. +const fn hw_offset(board: &Board, offset: usize) -> usize { + if !board.hw_data_insert && offset >= M5_HW_DATA_INSERT + M5_HW_DATA_INSERT_LEN { + offset - M5_HW_DATA_INSERT_LEN + } else { + offset + } +} + /// What the M5 graph's HW data and power policy depend on per board. Every board is a T6050 /// running the same firmware build; [`crate::g17::Config::m5`] names the board's. pub(crate) struct Board { + /// The chip ID the HW data names. + chip_id: u32, + /// The InitData layout the board's firmware build reads. + pub(crate) graph: &'static Graph, + /// Whether the board's HW data has the M5 Pro firmware's eight bytes at + /// [`M5_HW_DATA_INSERT`]; the common 26A428 interface (J813) has not. + hw_data_insert: bool, /// GPU clusters. pub(crate) clusters: u32, /// Cores of all clusters. pub(crate) gpu_cores: u32, /// The core type and firmware revision the HW data names. core_revision: (u32, u32), - /// Main-domain performance states. + /// Performance states after the off state, the HW data's count. + perf_state_count: u32, + /// Main-domain performance states. A board with fewer than 13 repeats its top state. pub(crate) perf_states: &'static [M5PerfState; 14], /// Voltage tables in [`Board::perf_states`]. perf_tables: usize, - /// Auxiliary-domain performance states. - aux_perf_states: &'static [M5AuxPerfState; 14], + /// Auxiliary-domain (CS/AFR) performance states, where the GPU has those domains. + aux_perf_states: Option<&'static [M5AuxPerfState; 14]>, /// The relative-frequency curve of [`Board::perf_states`]. relative_frequency: [u32; 16], + /// Each state's relative performance in percent (IEEE single), as the board's own firmware + /// data has it; without one, the frequency ratio in whole percent. + relative_power: Option<&'static [u32; 14]>, /// Rows of the board's AON table the firmware reads. aon_records: &'static [[u32; 3]], /// Register windows the firmware maps, in device-VA order. @@ -1785,12 +2144,19 @@ pub(crate) struct Board { pub(crate) calibration: Option<[u32; 3]>, /// The policy controllers' configuration, where the board has one. pub(crate) controller_defaults: Option Result<(), InitdataError>>, + + /// The flag of every register-map slot, whether or not a window uses it. Only boards whose + /// own firmware's map has been read carry one. + pub(crate) register_slot_flags: Option<&'static [u32; COMMON26_HW_REGISTER_MAP_SLOTS]>, /// Region C's idle standby timer where the board's differs from the image's 1500 us, us. pub(crate) idle_standby_us: Option, /// The active memory cache controller mask, where it differs from the inherited 0xf. mcc_mask: Option, /// The two topology words (16, and the core count), where the board sets them. topology_words: Option<(u32, u32)>, + /// The board's own 25G76 HW data words where the inherited ones are the M5 Pro's, at their + /// offsets in the board's HW data. + hw_words: &'static [(usize, u32)], /// The hotspot at which the thermal governor's fan curve starts, millidegrees C. pub(crate) fan_curve_start_mc: i32, /// Whether the host encodings are the G17D ones measured on the M5 Ultra @@ -1803,15 +2169,21 @@ pub(crate) struct Board { /// M5 Pro: two clusters of ten cores. pub(crate) const J714: Board = Board { + chip_id: 0x6050, + graph: &M5_GRAPH, + hw_data_insert: true, clusters: 2, gpu_cores: 20, core_revision: (0x18, 4), + perf_state_count: J714_GPU_NUM_PERF_STATES, perf_states: &J714_GPU_PERF_STATES, perf_tables: 2, - aux_perf_states: &J714_AUX_PERF_STATES, + aux_perf_states: Some(&J714_AUX_PERF_STATES), relative_frequency: relative_frequency(&J714_GPU_PERF_STATES, 1), + relative_power: None, aon_records: &J714_AON_RECORDS, register_windows: &J714_REGISTER_WINDOWS, + register_slot_flags: None, min_parameter_buffer: 0x0020_0000, render_pool: (0x40_0000, 1 << 32), calibration: Some([0xd2035701, 0xac3a1280, 0x0a281d81]), @@ -1819,6 +2191,7 @@ pub(crate) const J714: Board = Board { idle_standby_us: None, mcc_mask: None, topology_words: None, + hw_words: &[], fan_curve_start_mc: 84_000, g17d_forms: false, completion_inputs: true, @@ -1826,15 +2199,21 @@ pub(crate) const J714: Board = Board { /// M5 Max: four clusters of ten cores. pub(crate) const J716C: Board = Board { + chip_id: 0x6050, + graph: &M5_GRAPH, + hw_data_insert: true, clusters: 4, gpu_cores: 40, core_revision: (0x19, 4), + perf_state_count: J714_GPU_NUM_PERF_STATES, perf_states: &J716C_GPU_PERF_STATES, perf_tables: 4, - aux_perf_states: &J716C_AUX_PERF_STATES, + aux_perf_states: Some(&J716C_AUX_PERF_STATES), relative_frequency: relative_frequency(&J716C_GPU_PERF_STATES, 1), + relative_power: None, aon_records: &J716C_AON_RECORDS, register_windows: &J716C_REGISTER_WINDOWS, + register_slot_flags: None, min_parameter_buffer: 0x0030_0000, render_pool: (0x80_0000, 1 << 33), // Its own controller configuration, for a part with this calibration. @@ -1844,6 +2223,7 @@ pub(crate) const J716C: Board = Board { // The ADT GPU node's active-mcc-bit-vector: eight memory cache controllers. mcc_mask: Some(0xff), topology_words: Some((16, 40)), + hw_words: &[], fan_curve_start_mc: 84_000, g17d_forms: false, completion_inputs: false, @@ -1854,17 +2234,23 @@ const J775D_PERF_FLOOR: usize = 3; /// M5 Ultra: two dies of four clusters of ten cores (G17D). Values from the J775D ADT. pub(crate) const J775D: Board = Board { + chip_id: 0x6050, + graph: &M5_GRAPH, + hw_data_insert: true, clusters: 8, gpu_cores: 80, // Firmware revision 2 for raw revision 0x11. core_revision: (0x1a, 2), + perf_state_count: J714_GPU_NUM_PERF_STATES, perf_states: &J775D_GPU_PERF_STATES, // One voltage table per cluster. perf_tables: 8, - aux_perf_states: &J775D_AUX_PERF_STATES, + aux_perf_states: Some(&J775D_AUX_PERF_STATES), relative_frequency: relative_frequency(&J775D_GPU_PERF_STATES, J775D_PERF_FLOOR), + relative_power: None, aon_records: &J775D_AON_RECORDS, register_windows: &J775D_REGISTER_WINDOWS, + register_slot_flags: None, // The minimum parameter buffer for eight clusters. min_parameter_buffer: 0x0050_0000, // 16 MiB render blocks in a 16 GiB inventory. @@ -1876,6 +2262,7 @@ pub(crate) const J775D: Board = Board { // The ADT GPU node's active-mcc-bit-vector: eight memory cache controllers. mcc_mask: Some(0xff), topology_words: None, + hw_words: &[], // The SMC raises the fans in its automatic mode from a 97 C hotspot under a sustained // full-chip load, with the CPU and GPU still at their top states: the curve starts there, // above the SMC's own start rather than below it. @@ -1884,6 +2271,46 @@ pub(crate) const J775D: Board = Board { completion_inputs: false, }; +/// M5 (T8142, MacBook Air J813): one cluster of ten core slots, eight enabled. Values from the +/// J813 ADT (27.0) and its 25G76 HW data (core type 0x20 revision 5, twelve states). +/// Bring-up: no policy controller configuration yet. +pub(crate) const J813: Board = Board { + chip_id: 0x8142, + // The J813's 26A428 firmware compares root +0x00 with 0x0c8b_4339_83a0_04c0 (movz/movk at + // TEXT 0x21ed8 in both roles): the common 26A428 interface J873 also runs, not the M5 Pro's. + graph: &COMMON26_GRAPH, + // The common 26A428 HW data, without the M5 Pro's insert. + hw_data_insert: false, + clusters: 1, + gpu_cores: 10, + core_revision: (0x20, 5), + perf_state_count: 12, + perf_states: &J813_GPU_PERF_STATES, + perf_tables: 1, + aux_perf_states: None, + relative_frequency: relative_frequency(&J813_GPU_PERF_STATES, 1), + relative_power: Some(&J813_GPU_RELATIVE_POWER), + aon_records: &J813_AON_RECORDS, + register_windows: &J813_REGISTER_WINDOWS, + register_slot_flags: Some(&J813_HW_REGISTER_SLOT_FLAGS), + min_parameter_buffer: (super::target::min_parameter_buffer_pages(1) << 15) as u32, + // Half J714's: 2 MiB render blocks in a 2 GiB inventory. + render_pool: (0x20_0000, 1 << 31), + calibration: None, + controller_defaults: None, + idle_standby_us: None, + // One memory cache controller (/arm-io/sgx active-mcc-bit-vector; 25G76 +0xf8c = 1). + mcc_mask: Some(0x1), + topology_words: None, + // Words of macOS 27.0's own hardware data on this machine, read out of its bundle under + // aura, that the inherited 25G76 template and the generic encoding leave different. + hw_words: &[(0x0fa8, 0x8000), (0x0fb4, 2), (0x0fb8, 0), (0x1044, 0xc), (0x10a4, 0xa), + (0x10d4, 0x17), (0x1118, 0), (0x2660, 4), (0x2720, 1), (0x2724, 0)], + fan_curve_start_mc: 84_000, + g17d_forms: false, + completion_inputs: false, +}; + const _: () = { use super::target::min_parameter_buffer_pages; assert!(J714.min_parameter_buffer as u64 == min_parameter_buffer_pages(J714.clusters) << 15); @@ -1893,7 +2320,8 @@ const _: () = { /// Populate the enabled-USC count from the device's sampled core masks. /// This count covers the whole device, not one cluster. -pub(crate) fn set_m5_enabled_cores(out: &mut [u8], enabled: u32) -> Result<(), InitdataError> { +pub(crate) fn set_m5_enabled_cores(board: &Board, out: &mut [u8], enabled: u32) + -> Result<(), InitdataError> { if out.len() != M5_HW_DATA_SIZE { return Err(InitdataError::BufferSize); } @@ -1901,7 +2329,7 @@ pub(crate) fn set_m5_enabled_cores(out: &mut [u8], enabled: u32) -> Result<(), I return Err(InitdataError::InterfaceMismatch); } put_u32(out, 0x10a4, enabled); - put_u32(out, 0x268c, enabled); + put_u32(out, hw_offset(board, 0x268c), enabled); Ok(()) } @@ -1953,6 +2381,38 @@ pub(crate) const J714_GPU_PERF_STATES: [M5PerfState; 14] = two_tables([ (1578, 1000, 1000, 1000, 1000), (1620, 1025, 1040, 1025, 1040), ]); +/// One-table states (MHz, core voltage, SRAM voltage), the rest 0. +const fn one_table(states: [(u32, u32, u32); 14]) -> [M5PerfState; 14] { + let mut out = [(0, [0; M5_PERF_LANES], [0; M5_PERF_LANES]); 14]; + let mut state = 0; + while state < 14 { + let (frequency, core, sram) = states[state]; + out[state].0 = frequency; + out[state].1[0] = core; + out[state].2[0] = sram; + state += 1; + } + out +} + +// J813 ADT sgx perf-states and perf-states-sram (MHz, mV): twelve states after the off state. +// The encoder's table has fourteen; the last repeats the top state, which the HW data's count +// of twelve leaves out. +/// The J813's per-state relative performance, read out of macOS 27.0's own hardware data under +/// aura (bundle +0x19e4): 19.4 % at the lowest state up to 100 % at the top, state 13 unused. +/// It is neither the frequency ratio nor f*V or f*V^2: characterization data. +const J813_GPU_RELATIVE_POWER: [u32; 14] = [ + 0, 0x419b_36eb, 0x41d0_6202, 0x4205_907a, 0x4227_4b32, 0x4247_eae1, 0x426b_0873, + 0x428a_5bac, 0x42a4_5b3f, 0x42ab_4981, 0x42b0_3cd5, 0x42bb_20f3, 0x42c8_0000, 0, +]; + +pub(crate) const J813_GPU_PERF_STATES: [M5PerfState; 14] = one_table([ + (0, 125, 770), (338, 660, 770), (486, 675, 770), (636, 690, 770), + (796, 705, 770), (888, 735, 770), (988, 760, 770), (1084, 790, 790), + (1182, 830, 830), (1278, 865, 865), (1374, 890, 890), (1470, 945, 945), + (1578, 1010, 1010), (1578, 1010, 1010), +]); + /// Four-table states (MHz, four core-voltage tables, four SRAM-voltage tables), the rest 0. const fn four_tables(states: [(u32, [u32; 4], [u32; 4]); 14]) -> [M5PerfState; 14] { let mut out = [(0, [0; M5_PERF_LANES], [0; M5_PERF_LANES]); 14]; @@ -2075,6 +2535,51 @@ const J714_REGISTER_WINDOWS: [M5RegisterWindowSpec; 30] = { windows }; +/// J813: J714's 26A428 slots at the J813's addresses, matched through its ADT and its 25G76 +/// windows (M5_25G76_REGISTER_WINDOWS: from the ID registers up, 26A428 numbers each GPU block +/// five slots higher). Slots 4 and 11 sit where J714's do, AIC + 0x68000 and MCC reg 9 + +/// 0x8000. Bring-up: slot 20, per cluster on T6050, is left out. From the common 26A428 map +/// (J873, same GPU base): slot 17 0x36000, DPEP (29) and ADCLK (30), and slot 32, the J813's +/// 25G76 slot 27, which J714's list lacks. +const J813_GPU_BASE: u64 = 0x5_8000_0000; +/// The J813's windows, read out of macOS 27.0's own hardware-data bundle under aura (its live +/// UAT maps exactly these, at exactly the device VAs [`m5_register_windows`] computes). +const J813_REGISTER_WINDOWS: [M5RegisterWindowSpec; 22] = { + const G: u64 = J813_GPU_BASE; + [ + jw(0, &[0x3_8101_4000], 0x4000, 0, 2), + jw(3, &[0x2_2010_4000], 0x18000, 0, 2), + jw(8, &[0x3_a135_0000], 0x4000, 0, 2), + jw(9, &[0x3_803d_0000], 0x4000, 0, 2), + jw(10, &[0x3_803c_0000], 0x4000, 0, 0), + jw(12, &[0x4_2165_c000], 0x4000, 0, 2), + jw(14, &[0x3_8028_0000], 0x8000, 0, 0), + jw(15, &[0x3_8c84_0000], 0x24000, 0, 2), + // Ends right above the KSM block the firmware programs, as on 25G76. + jw(17, &[G], 0x21400, 0, 2), + jw(22, &[G + 0x100_0000], 0x8000, 0, 2), + jw(24, &[G + 0xec_0000], 0x10000, 0, 2), + jw(31, &[G + 0xd0_4000], 0x8000, 0xd0_4000, 2), + jw(32, &[G + 0xd0_d000], 0x1000, 0xd0_d000, 2), + jw(33, &[G + 0xd5_0000], 0x10000, 0xd5_0000, 2), + jw(34, &[G + 0xd1_0000], 0x4000, 0xd1_0000, 2), + jw(36, &[G + 0xd4_0000], 0x4000, 0xd4_0000, 2), + jw(37, &[G + 0xd6_0000], 0x8000, 0xd6_0000, 2), + jw(40, &[G + 0xe0_0000], 0x4000, 0xe0_0000, 2), + jw(44, &[G + 0xe0_8000, G + 0xe1_0000], 0x8000, 0, 2), + jw(46, &[G + 0xe3_3800], 0x4000, 0, 2), + jw(47, &[G + 0xe3_5000], 0x4000, 0, 2), + jw(56, &[G + 0x1_4400], 0x400, 0x1_4400, 0), + ] +}; + +/// The flag every one of the J813's slots carries, used or not, from macOS 27.0's own bundle; a +/// used slot's own flag agrees with it. +const J813_HW_REGISTER_SLOT_FLAGS: [u32; COMMON26_HW_REGISTER_MAP_SLOTS] = [ + 2, 0, 2, 2, 0, 2, 2, 2, 2, 2, 0, 0, 2, 0, 0, 2, 0, 2, 0, 0, 0, 0, 2, 0, 2, 2, 2, 2, 0, 0, 0, 2, + 2, 2, 2, 2, 2, 2, 2, 2, 2, 0, 2, 2, 2, 2, 2, 2, 2, 0, 0, 0, 2, 2, 2, 2, 0, 0, 2, 0, +]; + /// J714 with all eight AMCCs in slot 3 (J716c amcc-enable-mask 0xff, MCC /// reg[14..21] + 0x104000) and four instances, one per cluster, in slots 20 and /// 24, where J714 has two at the same stride. Slot 44 is a fixed pair on every M5 @@ -2176,6 +2681,12 @@ impl Board { self.completion_inputs } + /// Whether this is the J813, whose register programs macOS 27.0 writes in the common 26A428 + /// forms rather than the M5 Pro's (read out of its own kicks under aura). + pub(crate) const fn j813_forms(&self) -> bool { + self.chip_id == 0x8142 + } + /// The board's `index`th register window. pub(crate) fn register_window_spec(&self, index: usize) -> Option<&'static M5RegisterWindowSpec> { self.register_windows.get(index) @@ -2183,7 +2694,7 @@ impl Board { /// The board's register windows: the first [`Board::register_window_count`] entries. pub(crate) fn register_windows(&self) -> [M5RegisterWindow; M5_MAX_REGISTER_WINDOWS] { - m5_register_windows(self.register_windows) + m5_register_windows(self.register_windows, self.graph.is_common26()) } /// Register windows of the board. @@ -2192,7 +2703,7 @@ impl Board { } } -fn m5_register_windows(specs: &'static [M5RegisterWindowSpec]) +fn m5_register_windows(specs: &'static [M5RegisterWindowSpec], common26: bool) -> [M5RegisterWindow; M5_MAX_REGISTER_WINDOWS] { let mask = UAT_PAGE_SIZE as u64 - 1; @@ -2201,7 +2712,8 @@ fn m5_register_windows(specs: &'static [M5RegisterWindowSpec]) let mut next: u64 = 0; for (window, spec) in out.iter_mut().zip(specs) { let va = next + (spec.instances[0] & mask); - let span = spec.instances.len() as u64 * ((spec.size as u64 + mask) & !mask); + let size = if common26 { spec.size as u64 } else { (spec.size as u64 + mask) & !mask }; + let span = spec.instances.len() as u64 * size; *window = M5RegisterWindow { slot: spec.slot, device_va: HW_REGISTER_APERTURE.start + va, @@ -2210,7 +2722,11 @@ fn m5_register_windows(specs: &'static [M5RegisterWindowSpec]) unk_18: spec.unk_18, flag: spec.flag, }; - let end = (va + span + mask) & !mask; + // In the common 26A428 bundle the guard page follows the window's own bytes, before + // either is rounded up: a window whose size is not a whole number of pages does not push + // the next one past its page (macOS 27.0's J813 bundle, where slot 32's 0x1000 bytes + // leave the next window at the end of its own page). + let end = if common26 { va + span } else { (va + span + mask) & !mask }; next = (end + UAT_PAGE_SIZE as u64 + 0x7fff) & !0x7fff; } out @@ -2225,6 +2741,11 @@ pub(crate) fn encode_m5_register_map( return Err(InitdataError::BufferSize); } out.fill(0); + if let Some(flags) = board.register_slot_flags { + for (slot, flag) in flags.iter().enumerate() { + put_u32(out, slot * HW_REGISTER_MAP_ENTRY_SIZE + 0x20, *flag); + } + } for (index, window) in windows.iter().enumerate() { let spec = board.register_windows.get(index).ok_or(InitdataError::RegisterWindow)?; if window.slot >= COMMON26_HW_REGISTER_MAP_SLOTS @@ -2271,8 +2792,8 @@ pub(crate) fn encode_m5_hw_data( write_relocated_words(M5_25G76_HW_PLATFORM_WORDS, 0, COMMON26_HW_DATA_BANDS, g18)?; encode_m5_register_map(board, windows, &mut g18[HW_REGISTER_MAP_OFFSET..COMMON26_HW_REGISTER_MAP_END])?; - put_u32(g18, COMMON26_HW_CHIP_ID_OFFSET, 0x6050); - put_u32(g18, COMMON26_HW_PERF_STATE_COUNT, J714_GPU_NUM_PERF_STATES); + put_u32(g18, COMMON26_HW_CHIP_ID_OFFSET, board.chip_id); + put_u32(g18, COMMON26_HW_PERF_STATE_COUNT, board.perf_state_count); put_u32(g18, COMMON26_HW_FIXED_WORD_OFFSET, COMMON26_HW_FIXED_WORD_VALUE); put_u32(g18, 0x0fb4, 1); @@ -2286,11 +2807,18 @@ pub(crate) fn encode_m5_hw_data( put_u32(g18, 0x283a, 1); put_u32(g18, 0x283e, 1); let states = board.perf_states; - for (state, &(frequency, core, sram)) in states.iter().enumerate() { + // Off and the board's `perf_state_count` states: the rows above them stay zero, as in macOS + // 27.0's J813 bundle, whose twelve states leave state 13 empty where the padded table repeats + // its top state. (Every other board has 13 states, so all fourteen rows.) + let used = board.perf_state_count as usize + 1; + for (state, &(frequency, core, sram)) in states.iter().enumerate().take(used) { put_u32(g18, COMMON26_HW_PERF_FREQUENCY + state * 4, frequency); put_u32(g18, COMMON26_HW_PERF_SCALE + state * 4, COMMON26_PERF_SCALE_VALUE); - let relative = frequency * 100 / states[13].0; + let relative = match board.relative_power { + Some(table) => table[state], + None => frequency * 100 / states[13].0, + }; put_u32(g18, COMMON26_HW_PERF_RELATIVE_A + state * 4, relative); for lane in 0..COMMON26_HW_PERF_LANES { let row = state * COMMON26_HW_PERF_ROW_STRIDE + lane * 4; @@ -2301,21 +2829,34 @@ pub(crate) fn encode_m5_hw_data( } for index in 0..COMMON26_HW_PERF_ROWS { put_u32(g18, COMMON26_HW_PERF_INDEX_MAP + index * 4, index as u32); - put_u32(g18, COMMON26_HW_PERF_RELATIVE_B + index * 4, board.relative_frequency[index]); - put_u32(g18, M5_HW_AUX_RELATIVE_FREQUENCY + index * 4, board.relative_frequency[index]); + let relative = if index < used { board.relative_frequency[index] } else { 0 }; + put_u32(g18, COMMON26_HW_PERF_RELATIVE_B + index * 4, relative); + if board.aux_perf_states.is_some() { + put_u32(g18, M5_HW_AUX_RELATIVE_FREQUENCY + index * 4, relative); + } } encode_m5_aux_perf_states(board, g18); let table = COMMON26_KSM_COMPLETION_TABLE_OFFSET; g18[table..table + completion.len()].copy_from_slice(&completion); put_u64(g18, COMMON26_QOS_POINTER_OFFSET, qos_high); - let insert = M5_HW_DATA_INSERT; - let len = M5_HW_DATA_INSERT_LEN; - out.copy_within(insert..COMMON26_HW_DATA_SIZE, insert + len); - out[insert..insert + len].fill(0); + if board.hw_data_insert { + let insert = M5_HW_DATA_INSERT; + let len = M5_HW_DATA_INSERT_LEN; + out.copy_within(insert..COMMON26_HW_DATA_SIZE, insert + len); + out[insert..insert + len].fill(0); + } put_u32(out, 0x2660, 0); - put_u32(out, 0x2728, 0); - put_u32(out, 0x272c, 1); + put_u32(out, hw_offset(board, 0x2728), 0); + put_u32(out, hw_offset(board, 0x272c), 1); + // On the J813 macOS 27.0's initFirmwareData also seeds the KSM kick queues + // here: the NextTimestamp of each queue's first kick and the ring's entry + // count, taken from the driver's kick-queue setup. The firmware copies the + // timestamp to context +0xb80, where it seeds all 128 queue states. + if board.graph.is_common26() { + put_u64(out, hw_offset(board, 0x2730), KickTimestamp::FIRST.get()); + put_u32(out, hw_offset(board, 0x2738), KICK_SLOTS as u32); + } if let Some(mask) = board.mcc_mask { put_u64(out, 0x10a8, mask); } @@ -2323,8 +2864,11 @@ pub(crate) fn encode_m5_hw_data( put_u32(out, 0x100c, word); put_u32(out, 0x1048, cores); } + for &(offset, value) in board.hw_words { + put_u32(out, offset, value); + } // The inherited J813 minimum parameter-buffer size is replaced per board. - put_u32(out, 0x26b0, board.min_parameter_buffer); + put_u32(out, hw_offset(board, 0x26b0), board.min_parameter_buffer); Ok(()) } diff --git a/drivers/gpu/drm/asahi/g17/fw/m5/pool.rs b/drivers/gpu/drm/asahi/g17/fw/m5/pool.rs index b8c52bc634c5f..7a457322fb64e 100644 --- a/drivers/gpu/drm/asahi/g17/fw/m5/pool.rs +++ b/drivers/gpu/drm/asahi/g17/fw/m5/pool.rs @@ -2,7 +2,7 @@ //! M5 private-memory allocation encodings. #![allow(dead_code)] -use core::sync::atomic::{AtomicU64, AtomicUsize, Ordering}; +use core::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering}; use super::initdata::{Board, J714}; @@ -19,8 +19,27 @@ pub(crate) fn select_render_pool(board: &Board) { let (block, pool) = board.render_pool; RENDER_BLOCK_BYTES.store(block, Ordering::Relaxed); POOL_BYTES.store(pool, Ordering::Relaxed); + J813_POOLS.store(board.j813_forms(), Ordering::Relaxed); } +/// macOS 27.0 on the J813 (common 26A428) keeps its pools in the G17P form: a 0x68-byte +/// control (run cursor +0x48, state +0x4c, completed +0x54, class +0x60), a run list of +/// 0x40-byte slots, and four-word page-pool rows on a 0x20 stride. +static J813_POOLS: AtomicBool = AtomicBool::new(false); + +pub(crate) fn j813_pools() -> bool { + J813_POOLS.load(Ordering::Relaxed) +} + +/// Offset of the firmware-completed counter in this board's pool control. +pub(crate) fn completed_offset() -> usize { + if j813_pools() { J813_COMPLETED } else { COMPLETED } +} + +const J813_COMPLETED: usize = 0x54; +/// Words of one J813 page-pool row: the first four words of the M5 descriptor. +pub(crate) const J813_ROW_WORDS: usize = 4; + pub(crate) fn render_block_bytes() -> usize { RENDER_BLOCK_BYTES.load(Ordering::Relaxed) } @@ -68,7 +87,8 @@ impl PoolState { words[1] >> 32 & 0x0fff_ffff, words[2] & 0x0fff_ffff, words[5] & 0x0fff_ffff, words[6] & 0xffff, words[7] & 0x0fff_ffff, words[7] >> 32 & 0xffff]; - for (offset, value) in SAVED_FIELDS.into_iter().zip(values) { + let fields = if j813_pools() { 4 } else { SAVED_FIELDS.len() }; + for (offset, value) in SAVED_FIELDS.into_iter().zip(values).take(fields) { self.0[offset..offset + 4].copy_from_slice(&(value as u32).to_le_bytes()); } } @@ -135,7 +155,7 @@ impl PoolGeometry { } pub(crate) fn g17s_render(blocks: usize, directory_entries: u32) -> Result { - Self::with_kind(blocks, directory_entries, true) + Self::with_kind(blocks, directory_entries, !j813_pools()) } pub(crate) fn block_bytes(self) -> usize { @@ -264,9 +284,32 @@ pub(crate) fn encode_state(p: PoolLayout, submitted: u64, out: &mut [u8]) Ok(()) } +/// The J813's G17P-form pool control, as macOS 27.0 leaves it on that machine. +fn encode_j813_state(p: PoolLayout, submitted: u64, out: &mut [u8], compute: bool) + -> Result<(), PoolError> +{ + validate_layout(p)?; + if submitted == 0 || submitted & 3 != 0 { return Err(PoolError::Address); } + if out.len() < STATE_BYTES { return Err(PoolError::Buffer); } + out[..STATE_BYTES].fill(0); + // Run-list qwords in use: one 0x40-byte slot per block. + let run_qwords = if p.geometry.g17s_render_blocks { 1 } else { 8 }; + for (offset, value) in [(0, if compute { 3 } else { 1 }), (8, p.hardware_id), (0x10, 1), + (0x1c, p.geometry.directory_entries), (0x24, p.geometry.pages), + (0x2c, p.geometry.pages), (0x48, (p.geometry.blocks * run_qwords) as u32), + (0x5c, u32::from(compute)), (0x60, if compute { 1 } else { 2 })] { + out[offset..offset + 4].copy_from_slice(&value.to_le_bytes()); + } + for (offset, value) in [(0x14, p.directory), (0x30, p.freelist), (0x4c, submitted)] { + out[offset..offset + 8].copy_from_slice(&value.to_le_bytes()); + } + Ok(()) +} + pub(crate) fn encode_g17s_render_state(p: PoolLayout, submitted: u64, out: &mut [u8]) -> Result<(), PoolError> { + if j813_pools() { return encode_j813_state(p, submitted, out, false); } encode_state(p, submitted, out)?; out[0x80..0x84].copy_from_slice(&4u32.to_le_bytes()); Ok(()) @@ -275,6 +318,7 @@ pub(crate) fn encode_g17s_render_state(p: PoolLayout, submitted: u64, out: &mut pub(crate) fn encode_g17s_compute_state(p: PoolLayout, submitted: u64, out: &mut [u8]) -> Result<(), PoolError> { + if j813_pools() { return encode_j813_state(p, submitted, out, true); } if !p.geometry.g17s_render_blocks { return Err(PoolError::Geometry); } encode_state(p, submitted, out)?; out[0x7c..0x80].copy_from_slice(&1u32.to_le_bytes()); diff --git a/drivers/gpu/drm/asahi/g17/fw/m5/render.rs b/drivers/gpu/drm/asahi/g17/fw/m5/render.rs index f53956102b610..86cada2de9c07 100644 --- a/drivers/gpu/drm/asahi/g17/fw/m5/render.rs +++ b/drivers/gpu/drm/asahi/g17/fw/m5/render.rs @@ -7,7 +7,7 @@ use super::super::kick::{KickTimestamp, RegisterWrite, RegisterArrayBinding}; use super::render_tail::{self as tail, RenderStage, RenderTail}; -pub(crate) const M5_TA_COMMAND_BYTES: usize = 0x990; +pub(crate) const M5_TA_COMMAND_BYTES: usize = 0x9a0; pub(crate) const M5_3D_COMMAND_BYTES: usize = 0x2240; pub(crate) const M5_TA_SELECTOR: u32 = 0; pub(crate) const M5_3D_SELECTOR: u32 = 1; diff --git a/drivers/gpu/drm/asahi/g17/fw/m5/render_assembly.rs b/drivers/gpu/drm/asahi/g17/fw/m5/render_assembly.rs index 1966ab22f4878..7fc107313d096 100644 --- a/drivers/gpu/drm/asahi/g17/fw/m5/render_assembly.rs +++ b/drivers/gpu/drm/asahi/g17/fw/m5/render_assembly.rs @@ -70,6 +70,11 @@ pub(crate) fn encode_into(storage: &mut [u8], low: u64, ta.fill(0); fragment.fill(0); let fragment_rce = cmd::encode_m5_3d_command(fragment, fragment_low, inputs, &fragment_programs).map_err(|_| EINVAL)?; let ta_rce = cmd::encode_m5_ta_command(ta, low, inputs, &ta_program).map_err(|_| EINVAL)?; + if controls.board.graph.is_common26() { + use super::render_tail::{common26_layout, RenderStage}; + common26_layout(ta, RenderStage::Tiling).ok_or(EINVAL)?; + common26_layout(fragment, RenderStage::Fragment).ok_or(EINVAL)?; + } let (ta_storage, fragment_storage) = storage.split_at_mut(FRAGMENT_OFFSET); ta_storage[..cmd::M5_TA_COMMAND_BYTES].copy_from_slice(ta); fragment_storage[..cmd::M5_3D_COMMAND_BYTES].copy_from_slice(fragment); diff --git a/drivers/gpu/drm/asahi/g17/fw/m5/render_registers.rs b/drivers/gpu/drm/asahi/g17/fw/m5/render_registers.rs index 03cbc862b3b19..59399847c2c36 100644 --- a/drivers/gpu/drm/asahi/g17/fw/m5/render_registers.rs +++ b/drivers/gpu/drm/asahi/g17/fw/m5/render_registers.rs @@ -331,16 +331,20 @@ pub(crate) struct TaExtra { pub(crate) min_free_tvb_pages: u64, pub(crate) scs_bracket: u8, pub(crate) matrix_bracket: u8, + /// The J813's form, as macOS 27.0 writes it there: the program keeps its own 0x1748/0x1740 + /// bracket (every TA program opens with 0x1748 = 1 and closes with 0x1740 = 1) and carries + /// the common registers unchanged, without the tiler parameter sub-buffer writes. + pub(crate) j813: bool, } pub(crate) struct TaProgram { - registers: [RegisterWrite; 97], + registers: [RegisterWrite; 99], length: usize, } impl TaProgram { pub(crate) const fn empty() -> Self { - Self { registers: [RegisterWrite {reg: 0, value: 0}; 97], length: 0 } + Self { registers: [RegisterWrite {reg: 0, value: 0}; 99], length: 0 } } pub(crate) fn registers(&self) -> &[RegisterWrite] { &self.registers[..self.length] } pub(crate) fn set_lifecycle(&mut self, value: u64) { @@ -365,10 +369,19 @@ pub(crate) fn tiling_with_scratch_into(common: &[RegisterWrite], e: TaExtra, if e.matrix_bracket & 1 != 0 { out.push(0x17f1, 1)?; } let mut deflake = false; for r in common.iter().copied() { - if r.reg == 0x1748 { continue; } + if r.reg == 0x1748 { + if e.j813 { out.push(0x1748, 1)?; } + continue; + } if r.reg == 0x1740 { if e.scs_bracket & 1 != 0 { out.push(0x17d9, 1)?; } if e.matrix_bracket & 1 != 0 { out.push(0x17e9, 1)?; } + if e.j813 { out.push(0x1740, 1)?; } + continue; + } + if e.j813 { + // 0x10141 (copied to 0x1ca60) keeps the one-layer 0x200, as macOS writes it. + out.push(r.reg, r.value)?; continue; } out.push(r.reg, r.value)?; @@ -391,8 +404,8 @@ pub(crate) fn tiling_with_scratch_into(common: &[RegisterWrite], e: TaExtra, _ => {} } } - if out.length != 92 + usize::from(e.scs_bracket & 1) * 2 + usize::from(e.matrix_bracket & 1) * 2 - + usize::from(mesh_storage != 0) { + if !e.j813 && out.length != 92 + usize::from(e.scs_bracket & 1) * 2 + + usize::from(e.matrix_bracket & 1) * 2 + usize::from(mesh_storage != 0) { return Err(EINVAL); } Ok(()) @@ -400,6 +413,14 @@ pub(crate) fn tiling_with_scratch_into(common: &[RegisterWrite], e: TaExtra, pub(crate) fn fragment_main_with_scratch_into(p: &RenderPass, common: &[RegisterWrite], dispatch: FragmentDispatch, descriptor_flag: u8, scs: bool, out: &mut [RegisterWrite]) -> Result { + fragment_main_forms_into(p, common, dispatch, descriptor_flag, scs, false, out) +} + +/// As [`fragment_main_with_scratch_into`]; with `j813`, the J813's macOS 27.0 form, which writes no +/// 0x160a8 before either tile write. +pub(crate) fn fragment_main_forms_into(p: &RenderPass, common: &[RegisterWrite], + dispatch: FragmentDispatch, descriptor_flag: u8, scs: bool, j813: bool, + out: &mut [RegisterWrite]) -> Result { let mut n = 0; let mut tile = 0; @@ -410,9 +431,11 @@ pub(crate) fn fragment_main_with_scratch_into(p: &RenderPass, common: &[Register } if r.reg == 0x16068 { if tile == 2 { return Err(EINVAL); } - out[n] = RegisterWrite { reg: 0x160a8, - value: if tile == 0 { dispatch.initial(descriptor_flag) } else { 0x20 } }; - n += 1; + if !j813 { + out[n] = RegisterWrite { reg: 0x160a8, + value: if tile == 0 { dispatch.initial(descriptor_flag) } else { 0x20 } }; + n += 1; + } tile += 1; } let slot = out.get_mut(n).ok_or(EINVAL)?; @@ -526,14 +549,20 @@ impl Workspace { build_ta_registers_into(p, &c, r.free_list_slot, ta, &self.common.ta)?; let mut extra = extra; extra.scs_bracket |= u8::from(c.ta.scs.scratch.enabled()); + extra.j813 = c.board.j813_forms(); tiling_with_scratch_into(ta, extra, r.mesh_storage, &mut self.ta)?; self.ta.set_lifecycle((u64::from(inputs.ta.predecessor_id) << 32) | u64::from(inputs.ta.completion.key as u32)); c.fragment_tile_flags = (c.fragment_tile_flags & !0x7ffff) | 0x36011 | (u64::from(inputs.fragment.empty_tile_flags[3] & 1) << 18); + let j813 = c.board.j813_forms(); + if j813 { + // macOS 27.0's J813 fragment program writes 0xa741 = 0 at entry. + c.fragment_event_control = 0; + } build_fragment_registers_into(p, &c, r.free_list_slot, &mut self.base, &self.common.fragment)?; - self.fragment_count = 92 + usize::from(c.fragment.scs.scratch.enabled()) * 2; - fragment_main_with_scratch_into(p, &self.base, dispatch, inputs.fragment.empty_tile_flags[3], - c.fragment.scs.scratch.enabled(), &mut self.fragment[..self.fragment_count])?; + self.fragment_count = if j813 { 90 } else { 92 } + usize::from(c.fragment.scs.scratch.enabled()) * 2; + fragment_main_forms_into(p, &self.base, dispatch, inputs.fragment.empty_tile_flags[3], + c.fragment.scs.scratch.enabled(), j813, &mut self.fragment[..self.fragment_count])?; for reg in &mut self.fragment { if matches!(reg.reg, 0x160e0 | 0x1499 | 0xa341) { reg.value = (u64::from(inputs.fragment.predecessor_id) << 32) | u64::from(inputs.fragment.completion.key as u32); diff --git a/drivers/gpu/drm/asahi/g17/fw/m5/render_tail.rs b/drivers/gpu/drm/asahi/g17/fw/m5/render_tail.rs index b67bce2fa60ce..81fcbc93b27e2 100644 --- a/drivers/gpu/drm/asahi/g17/fw/m5/render_tail.rs +++ b/drivers/gpu/drm/asahi/g17/fw/m5/render_tail.rs @@ -4,6 +4,21 @@ #[derive(Clone, Copy)] pub(crate) enum RenderStage { Tiling, Fragment } +/// The common 26A428 interface inserts six bytes before the UMA pointer. +/// Register programs and completion stamps precede the insert and keep their +/// offsets. Move the entire trailing record, including the context token and +/// extra-address list, after encoding the original M5 layout. +pub(crate) fn common26_layout(out: &mut [u8], stage: RenderStage) -> Option<()> { + let (start, end) = match stage { + RenderStage::Tiling => (0x92e, 0x990), + RenderStage::Fragment => (0x21c8, 0x2233), + }; + out.get(..end + 6)?; + out.copy_within(start..end, start + 6); + out[start..start + 6].fill(0); + Some(()) +} + /// A context token packs the six-bit execution identity above its reuse generation. /// The token in a render record agrees with the record header's context ID. pub(crate) fn context_token(context: u32, generation: u64) -> Option { @@ -68,3 +83,45 @@ pub(crate) fn encode_extra(out: &mut [u8], stage: RenderStage, value: &ExtraAddr if let RenderStage::Fragment = stage { tail[33..37].fill(0); } Some(()) } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn common26_render_tail_offsets() { + for (stage, uma, token, extra, length) in [ + (RenderStage::Tiling, 0x934, 0x94d, 0x975, 0x9a0), + (RenderStage::Fragment, 0x21ce, 0x21e7, 0x2214, 0x2240), + ] { + let mut bytes = [0u8; 0x2240]; + let out = &mut bytes[..length]; + let pointer = 0xfffffc2000123400u64; + let identity = context_token(10, 1).unwrap(); + out[uma - 6..uma + 2].copy_from_slice(&pointer.to_le_bytes()); + out[uma - 7] = 0xa5; + encode(out, stage, &RenderTail { + descriptor_value: 0x1234, + resolved_value: Some(pointer), + trailing_value: identity, + }).unwrap(); + encode_extra(out, stage, &ExtraAddresses { + count: 1, policy: 2, addresses: [pointer; 4], + }).unwrap(); + common26_layout(out, stage).unwrap(); + assert_eq!(&out[uma..uma + 8], &pointer.to_le_bytes()); + assert_eq!(&out[token..token + 8], &identity.to_le_bytes()); + assert_eq!(&out[extra..extra + 8], &pointer.to_le_bytes()); + assert_eq!(out[extra + 32], 9); + assert_eq!(&out[uma - 6..uma], &[0; 6]); + assert_eq!(out[uma - 7], 0xa5); + } + } + + #[test] + fn common26_requires_room_for_extended_tiling_tail() { + let mut bytes = [0xa5; 0x990]; + assert!(common26_layout(&mut bytes, RenderStage::Tiling).is_none()); + assert!(bytes.iter().all(|&byte| byte == 0xa5)); + } +} diff --git a/drivers/gpu/drm/asahi/g17/initdata.rs b/drivers/gpu/drm/asahi/g17/initdata.rs index 3164097665f5c..c505bbd5b3b45 100644 --- a/drivers/gpu/drm/asahi/g17/initdata.rs +++ b/drivers/gpu/drm/asahi/g17/initdata.rs @@ -172,12 +172,16 @@ impl WorkPublication<'_> { /// Session-long storage and mappings. Aliases are released before their backing objects. pub(super) struct InitData { m5: bool, + /// The M5 firmware build's layout, where [`Self::m5`]. + graph: &'static super::fw::m5::initdata::Graph, outer_credits: Option>, // Live publication uses interior writes under the device mutex. This owner is Send but not // Sync, so safe host code cannot issue those writes concurrently through shared references. _single_publisher: core::marker::PhantomData>, aliases: KVec, objects: KVec, + /// The log buffers published to each [`Role`], for reading back what the firmware wrote. + fw_log_objects: [Option; 2], roots: usize, cluster: usize, bundle: usize, @@ -287,10 +291,12 @@ impl InitData { pub(super) fn new() -> Self { Self { m5: false, + graph: &super::fw::m5::initdata::M5_GRAPH, outer_credits: None, _single_publisher: core::marker::PhantomData, aliases: KVec::new(), objects: KVec::new(), + fw_log_objects: [None; 2], roots: 0, cluster: 0, bundle: 0, @@ -340,6 +346,20 @@ impl InitData { mmu::UAT_PGSZ as u64, mmu::PROT_FW_SHARED_RW)?; self.metrics_fw_va = alias.iova(); self.aliases.push(alias, GFP_KERNEL)?; + if super::J813_FORMS.load(Ordering::Relaxed) { + // The J813's firmware touches a render's metrics in context 0 (the kernel lower VM) + // at the client's compact-window address. Every client VM maps its metrics entry at + // the window's start plus the entry's offset (MetricsLease::map_client), so map the + // whole backing at the window's start here. + let start = super::buffer::BlockGeometry::M5_COMPACT.start; + let alias = metrics.alias_in(uat.kernel_lower_vm(), + start..start + metrics.size() as u64, mmu::UAT_PGSZ as u64, + mmu::PROT_GPU_SHARED_RW)?; + if alias.iova() != start { + return Err(EINVAL); + } + self.aliases.push(alias, GFP_KERNEL)?; + } Ok(()) } @@ -418,6 +438,17 @@ impl InitData { /// One M5 page-pool row: the pool's eight descriptor words at its 64-byte slot. fn set_m5_uma(&self, slot: u16, words: [u64; 8]) -> Result { let object = self.object(self.page_pool)?; + if super::fw::m5::pool::j813_pools() { + // The J813's rows are the descriptor's first four words, 0x20 apart. + const ROW: usize = super::fw::m5::pool::J813_ROW_WORDS * 8; + let offset = usize::from(slot) * ROW; + object.pointer(offset, ROW)?; + for (index, word) in words.into_iter().take(ROW / 8).enumerate() { + object.dword(offset + index * 8)?.store(word, Ordering::Relaxed); + } + fence(Ordering::SeqCst); + return Ok(()); + } let offset = usize::from(slot) * core::mem::size_of::<[u64; 8]>(); // Check the full row before the first store. object.pointer(offset, core::mem::size_of::<[u64; 8]>())?; @@ -1724,17 +1755,17 @@ impl InitData { } fn main_config_offset(&self, role: Role) -> usize { - if self.m5 { super::fw::m5::initdata::M5_MAIN_BUNDLE_OFFSETS[role as usize] } + if self.m5 { self.graph.shared.main_config[role as usize] } else { abi::bundle::MAIN_CONFIG[role as usize] } } fn work_ring_offset(&self, channel: usize) -> usize { - if self.m5 { super::fw::m5::initdata::M5_WORK_RING_BUNDLE_OFFSETS[channel] } + if self.m5 { self.graph.work_ring_offsets[channel] } else { abi::work_ring_offset(channel) } } fn state_offset(&self, role: Role) -> usize { - if self.m5 { return super::fw::m5::initdata::M5_PRIVATE_STATE_GRIDS[role as usize]; } + if self.m5 { return self.graph.state_grids[role as usize]; } match role { Role::Primary => abi::private::PRIMARY_STATE, Role::Secondary => abi::private::SECONDARY_STATE, @@ -1742,7 +1773,7 @@ impl InitData { } fn status_offset(&self, role: Role) -> usize { - if self.m5 { return super::fw::m5::initdata::M5_PRIVATE_STATUS_A[role as usize]; } + if self.m5 { return self.graph.private.roles[role as usize].status_a; } match role { Role::Primary => self.status_a, Role::Secondary => abi::private::SECONDARY_STATUS_A, @@ -1839,13 +1870,13 @@ impl InitData { next: u32, ) -> Result { let ring_base = if self.m5 { - super::fw::m5::initdata::M5_CONTROL_RING_BUNDLE_OFFSETS[0] + self.graph.shared.control_ring[0] } else { self.main_config_offset(Role::Primary) + abi::MainConfig::CONTROL_RING }; let offset = ring_base + (slot as usize).checked_mul(record.len()).ok_or(EINVAL)?; let ring = self.object(self.bundle)?.pointer(offset, record.len())?; let power = self .object(self.region_c)? - .pointer(if self.m5 { super::fw::m5::power::POWER_ASSERT_TALLY } + .pointer(if self.m5 { self.graph.power_assert_tally } else { abi::RegionC::POWER_ASSERT_TALLY }, 4)?; let producer = self.control_producer()?; // SAFETY: The checked pointers remain valid for the lifetime of this non-Sync owner. @@ -1896,13 +1927,13 @@ impl InitData { fence(Ordering::Acquire); Ok(self .object(self.cluster)? - .word(m::M5_PRIVATE_LAYOUT.status_b + m::M5_PROGRESS)? + .word(self.graph.private.status_b + m::M5_PROGRESS)? .load(Ordering::Relaxed)) } fn recovery_state_offset(&self) -> usize { if self.m5 { - super::fw::m5::initdata::M5_PRIVATE_LAYOUT.status_b + self.graph.private.status_b + super::fw::m5::initdata::M5_RECOVERY_STATE } else { self.status_b + abi::PrimaryStatusB::RECOVERY_STATE } } @@ -1911,7 +1942,7 @@ impl InitData { fn recovery_view_words(&self, view: usize, offset: usize) -> Result<[u32; N]> { let bundle = self.object(self.bundle)?; let extent = if self.m5 { - *super::fw::m5::initdata::M5_BUNDLE_VIEW_EXTENTS.get(view).ok_or(EINVAL)? + *self.graph.bundle_view_extents.get(view).ok_or(EINVAL)? } else { *abi::recovery::VIEW_EXTENTS.get(view).ok_or(EINVAL)? }; @@ -1920,7 +1951,7 @@ impl InitData { Some(views) => (self.object(*views.get(view).ok_or(EINVAL)?)?, 0), None if self.m5 => ( bundle, - *super::fw::m5::initdata::M5_BUNDLE_VIEW_OFFSETS.get(view).ok_or(EINVAL)?, + *self.graph.bundle_view_offsets.get(view).ok_or(EINVAL)?, ), None => (bundle, *abi::bundle::VIEWS.get(view).ok_or(EINVAL)?), }; @@ -1948,7 +1979,7 @@ impl super::recovery::Memory for InitData { } fn fault_sources(&self) -> Result { let mask_offset = if self.m5 { - super::fw::m5::initdata::M5_PRIVATE_LAYOUT.status_b + self.graph.private.status_b + super::fw::m5::initdata::M5_RECOVERY_SLOT_MASK } else { self.status_b + abi::PrimaryStatusB::RECOVERY_SLOT_MASK @@ -1996,7 +2027,7 @@ impl super::recovery::Memory for InitData { Ok(self .object(self.cluster)? .word(if self.m5 { - super::fw::m5::initdata::M5_PRIVATE_LAYOUT.status_b + self.graph.private.status_b + super::fw::m5::initdata::M5_HOST_RECOVERY } else { self.status_b + abi::PrimaryStatusB::HOST_RECOVERY @@ -2027,7 +2058,7 @@ impl super::recovery::Memory for InitData { Some(views) => (self.object(*views.get(4).ok_or(EINVAL)?)?, 0), None if self.m5 => ( self.object(self.bundle)?, - super::fw::m5::initdata::M5_BUNDLE_VIEW_OFFSETS[4], + self.graph.bundle_view_offsets[4], ), None => (self.object(self.bundle)?, abi::bundle::VIEWS[4]), }; @@ -2053,7 +2084,7 @@ impl super::recovery::Memory for InitData { } let cluster = self.object(self.cluster)?; let base = if self.m5 { - super::fw::m5::initdata::M5_PRIVATE_LAYOUT.status_b + self.graph.private.status_b + super::fw::m5::initdata::M5_RECOVERY_INFO } else { self.status_b + abi::PrimaryStatusB::RECOVERY_INFO @@ -2085,9 +2116,10 @@ impl InitData { /// M5 only: the GPU's current power draw in mW, as the firmware's power controller reports it /// (integer word of the shared power-policy block). pub(super) fn m5_gpu_power_mw(&self) -> Result { - if !self.m5 { return Err(EINVAL); } + // The power word's place is the M5 Pro build's. + if !self.m5 || self.graph.is_common26() { return Err(EINVAL); } Ok(self.object(self.bundle)? - .word(super::fw::m5::initdata::M5_SHARED_LAYOUT.policy + super::thermal::POLICY_GPU_POWER)? + .word(self.graph.shared.policy + super::thermal::POLICY_GPU_POWER)? .load(Ordering::Relaxed)) } @@ -2096,7 +2128,7 @@ impl InitData { pub(super) fn m5_firmware_clamped(&self) -> Result { if !self.m5 { return Err(EINVAL); } let block = self.object(self.bundle)?; - let base = super::fw::m5::initdata::M5_SHARED_LAYOUT.policy; + let base = self.graph.shared.policy; let mask = block.word(base + super::thermal::POLICY_LIMITER_MASK)?.load(Ordering::Relaxed); let limit = block.word(base + super::thermal::POLICY_LIMITER_LIMIT)?.load(Ordering::Relaxed); Ok(super::thermal::is_emergency_clamp(mask, limit)) diff --git a/drivers/gpu/drm/asahi/g17/initdata/m5.rs b/drivers/gpu/drm/asahi/g17/initdata/m5.rs index 5851c3ae15b14..5c3200fefce19 100644 --- a/drivers/gpu/drm/asahi/g17/initdata/m5.rs +++ b/drivers/gpu/drm/asahi/g17/initdata/m5.rs @@ -13,13 +13,25 @@ const REGION_A: u64 = BASE + 0x1b_8000; const FWCTL: u64 = BASE + 0x1c_0000; const QOS: u64 = BASE + 0x1c_8000; const AON: u64 = BASE + 0x1d_0000; +/// The always-on block the policy names, where macOS 27.0 keeps it on the J813: its policy's AON +/// pointer (bundle +0xb858, read under aura) is the kernel window's 0xc07b_8000, right above the +/// shared cluster. The 25G76 address, [`AON`], holds an empty page there. +const COMMON26_AON: u64 = BASE + 0xc07b_8000; const KICK_SHARED: u64 = BASE + 0x1d_8000; const BUNDLE: u64 = BASE + 0xc078_8000; const COMPLETION_HIGH: u64 = BASE + 0xc080_0000; const COMPLETION_LOW: u64 = 0x70_0184_8000; +/// The completion rings, where macOS 27.0 keeps them on the J813 (its own UAT, read under aura: +/// four 0x20000 backings on this stride, the log buffers right above them). +const COMMON26_COMPLETION_HIGH: u64 = BASE + 0xc07e_8000; +const COMMON26_COMPLETION_LOW: u64 = 0x72_0184_8000; const COMPLETION_STRIDE: u64 = 0x2_8000; const PB: u64 = BASE + 0x15e_0000; const UMA: u64 = BASE + 0x15e_8000; +/// The log buffer each J813 coprocessor is handed on the crashlog endpoint, by [`Role`]: the +/// addresses and the size macOS 27.0 publishes there. +const FW_LOG: [u64; 2] = [BASE + 0xc089_8000, BASE + 0xc088_8000]; +const FW_LOG_SIZE: usize = 0x8000; /// Lower-root entry 1 of the M5 layout (Mesa's driver window), linked before the firmware starts /// as the bare-metal J775D boot has always run. @@ -29,28 +41,34 @@ impl InitData { /// The kernel lower-root ranges a bare-metal M5 firmware reaches, whose top-level links must /// exist before it starts: entry 1, the descriptor tables and completion rings (entry 7) and /// the first page of the canonical arena (`canonical`, entry 8), where queues go later. - pub(in crate::g17) fn m5_lower_links(canonical: core::ops::Range) + pub(in crate::g17) fn m5_lower_links(canonical: core::ops::Range, graph: &m::Graph) -> [core::ops::Range; 8] { let page = |address: u64| address..address + mmu::UAT_PGSZ as u64; - let ring = |index: u64| page(COMPLETION_LOW + index * COMPLETION_STRIDE); - [page(BOOT_LINK), page(m::REGION_VIEW_LOW_ADDRS[0]), page(m::REGION_VIEW_LOW_ADDRS[1]), + let low = if graph.is_common26() { COMMON26_COMPLETION_LOW } else { COMPLETION_LOW }; + let ring = |index: u64| page(low + index * COMPLETION_STRIDE); + let views = graph.profile.region_view_low_addrs(); + [page(BOOT_LINK), page(views[0]), page(views[1]), ring(0), ring(1), ring(2), ring(3), page(canonical.start)] } /// Diagnostic-only read of an aligned word within the existing policy allocation. /// Callers hold the device state mutex, retaining this mapping for the entire sample. pub(crate) fn m5_policy_telemetry_word(&self, offset: usize) -> Result { - if !self.m5 || offset & 3 != 0 || offset.checked_add(4).is_none_or(|end| end > m::M5_POLICY_SIZE) { + if !self.m5 || offset & 3 != 0 + || offset.checked_add(4).is_none_or(|end| end > self.graph.policy_size) { return Err(EINVAL); } - let at = m::M5_SHARED_LAYOUT.policy.checked_add(offset).ok_or(EINVAL)?; + let at = self.graph.shared.policy.checked_add(offset).ok_or(EINVAL)?; Ok(self.object(self.bundle)?.word(at)?.load(Ordering::Relaxed)) } /// Counts `submissions` in the firmware's submission history, in the slot of this moment. pub(crate) fn note_m5_submission_history(&self, submissions: u8) -> Result { if !self.m5 { return Err(EINVAL); } + // The submission history is the M5 Pro build's status config +0x73c..; the common + // 26A428 one (0x6d0 bytes) has no such table. + if self.graph.is_common26() { return Ok(()); } let private = self.object(self.cluster)?; let base = m::M5_PRIVATE_LAYOUT.status_config; for _ in 0..64 { @@ -91,11 +109,34 @@ impl InitData { Err(EBUSY) } + /// The address and size of a coprocessor's log buffer, for the message that publishes it. + pub(in crate::g17) fn fw_log(role: Role) -> (u64, usize) { + (FW_LOG[role as usize], FW_LOG_SIZE) + } + + /// A coprocessor's log buffer, as the firmware has left it. Empty before the graph is built. + pub(in crate::g17) fn fw_log_bytes(&self, role: Role) -> &[u8] { + match self.fw_log_objects[role as usize].and_then(|index| self.object(index).ok()) { + Some(object) => object.bytes(), + None => &[], + } + } + + /// The hardware-data bundle as this driver built it, for diffing against a machine's own. + pub(in crate::g17) fn hardware_data_bytes(&self) -> &[u8] { + match self.object(self.bundle) { + Ok(object) => object.bytes(), + Err(_) => &[], + } + } + pub(crate) fn build_m5(&mut self, dev: &AsahiDrmDevice, uat: &mmu::Uat, id: &crate::regs::G17Id, board: &m::Board) -> Result { use mmu::{PROT_FW_SHARED_RW as RW, PROT_FW_SHARED_RO as RO, PROT_GPU_SHARED_RW as GPU}; self.m5 = true; + self.graph = board.graph; + let common26 = board.graph.is_common26(); // TA, fragment and compute share the compute work ring per priority; every // preparation's claim on it is counted from the first publication on. self.outer_credits = Some(super::outer_credit::Credits::new()?); @@ -105,8 +146,8 @@ impl InitData { KernelObject::new(dev, uat.kernel_vm(), Placement::At(address), size, mmu::PROT_FW_PRIV_RW, CpuMap::WriteBack) }; - let private = m::M5_PRIVATE_LAYOUT; - let shared = m::M5_SHARED_LAYOUT; + let private = board.graph.private; + let shared = board.graph.shared; self.cluster = self.add(fixed(PRIVATE, private.total, RW)?)?; self.region_c = self.add(fixed(REGION_C, mmu::UAT_PGSZ, RO)?)?; self.roots = self.add(fixed(ROOTS, 0xc000, RO)?)?; @@ -114,7 +155,16 @@ impl InitData { self.add(fixed(FWCTL, mmu::UAT_PGSZ, RW)?)?; self.qos = self.add(fixed(QOS, m::G17P_QOS_RESOURCE_SIZE, RW)?)?; self.add(fixed(KICK_SHARED, m::M5_KICK_SHARED_SIZE, mmu::PROT_GPU_FW_SHARED_RW)?)?; - let aon = self.add(fixed_controller(AON, m::M5_AON_PTD_SIZE)?)?; + // The J813's log buffers, which the firmware is told about once it runs (see + // `publish_log_buffer`); it reads and writes them through its own kernel window. + if common26 { + for (role, address) in FW_LOG.into_iter().enumerate() { + let object = self.add(fixed_controller(address, FW_LOG_SIZE)?)?; + self.fw_log_objects[role] = Some(object); + } + } + let aon_va = if common26 { COMMON26_AON } else { AON }; + let aon = self.add(fixed_controller(aon_va, m::M5_AON_PTD_SIZE)?)?; self.bundle = self.add(fixed_controller(BUNDLE, shared.total)?)?; self.pb = self.add(fixed(PB, mmu::UAT_PGSZ, RW)?)?; self.page_pool = self.add(fixed(UMA, mmu::UAT_PGSZ, RW)?)?; @@ -123,12 +173,18 @@ impl InitData { Placement::In(uat.kernel_dynamic_range(), mmu::UAT_PGSZ as u64), mmu::UAT_PGSZ, mmu::PROT_FW_SHARED_RW, CpuMap::WriteCombined)?)?; } - self.alias(self.pb, uat.kernel_lower_vm(), m::REGION_VIEW_LOW_ADDRS[0], GPU)?; - self.alias(self.page_pool, uat.kernel_lower_vm(), m::REGION_VIEW_LOW_ADDRS[1], GPU)?; + let views = board.graph.profile.region_view_low_addrs(); + self.alias(self.pb, uat.kernel_lower_vm(), views[0], GPU)?; + self.alias(self.page_pool, uat.kernel_lower_vm(), views[1], GPU)?; + let (low, high) = if common26 { + (COMMON26_COMPLETION_LOW, COMMON26_COMPLETION_HIGH) + } else { + (COMPLETION_LOW, COMPLETION_HIGH) + }; let mut completions = [m::G17PKsmCompletionAliases { low: 0, high: 0 }; m::M5_25G76_COMPLETION_QUEUE_COUNT]; for (index, pair) in completions.iter_mut().enumerate() { - pair.low = COMPLETION_LOW + index as u64 * COMPLETION_STRIDE; - pair.high = COMPLETION_HIGH + index as u64 * COMPLETION_STRIDE; + pair.low = low + index as u64 * COMPLETION_STRIDE; + pair.high = high + index as u64 * COMPLETION_STRIDE; // After initialization the host only reads completion records. // Match the standard G17 completion ring's CPU write-back mapping; // keep firmware/GPU aliases, size and lifetime unchanged. @@ -145,9 +201,18 @@ impl InitData { } m::encode_common26_aon_ptd(board, &mut self.object_mut(aon)?.bytes_mut()[..m::M5_AON_PTD_SIZE]) .map_err(|_| EINVAL)?; - m::encode_m5_region_c(self.object_mut(self.region_c)?.bytes_mut()).map_err(|_| EINVAL)?; - m::encode_m5_private_cluster(PRIVATE, FWCTL, - &mut self.object_mut(self.cluster)?.bytes_mut()[..private.total]).map_err(|_| EINVAL)?; + if common26 { + m::encode_common26_board_region_c( + &mut self.object_mut(self.region_c)?.bytes_mut()[..m::REGION_C_SIZE]) + .map_err(|_| EINVAL)?; + m::encode_common26_private_cluster(PRIVATE, FWCTL, + &mut self.object_mut(self.cluster)?.bytes_mut()[..private.total]) + .map_err(|_| EINVAL)?; + } else { + m::encode_m5_region_c(self.object_mut(self.region_c)?.bytes_mut()).map_err(|_| EINVAL)?; + m::encode_m5_private_cluster(PRIVATE, FWCTL, + &mut self.object_mut(self.cluster)?.bytes_mut()[..private.total]).map_err(|_| EINVAL)?; + } let windows = board.register_windows(); let windows = &windows[..board.register_window_count()]; for (index, window) in windows.iter().enumerate() { @@ -173,18 +238,23 @@ impl InitData { }; m::encode_m5_hw_data(board, completions, QOS, windows, &mut self.object_mut(self.bundle)?.bytes_mut()[..m::M5_HW_DATA_SIZE]).map_err(|_| EINVAL)?; - m::set_m5_kick_shared( + m::set_m5_kick_shared(board, &mut self.object_mut(self.bundle)?.bytes_mut()[..m::M5_HW_DATA_SIZE], KICK_SHARED) .map_err(|_| EINVAL)?; let masked_cores: u32 = id.core_masks.iter().map(|mask| mask.count_ones()).sum(); let enabled_cores = if masked_cores != 0 { masked_cores } else { id.num_clusters.checked_mul(id.num_cores).ok_or(EOVERFLOW)? }; - m::set_m5_enabled_cores( + m::set_m5_enabled_cores(board, &mut self.object_mut(self.bundle)?.bytes_mut()[..m::M5_HW_DATA_SIZE], enabled_cores) .map_err(|_| EINVAL)?; - m::encode_m5_bundle_policy(m::M5PolicyBindings { - state: PRIVATE + private.policy_state as u64, aon_ptd: AON, boot_timestamp: 0, - }, self.object_mut(self.bundle)?.bytes_mut()).map_err(|_| EINVAL)?; + let bindings = m::M5PolicyBindings { + state: PRIVATE + private.policy_state as u64, aon_ptd: aon_va, boot_timestamp: 0, + }; + if common26 { + m::encode_common26_bundle_policy(board, bindings, self.object_mut(self.bundle)?.bytes_mut()) + } else { + m::encode_m5_bundle_policy(bindings, self.object_mut(self.bundle)?.bytes_mut()) + }.map_err(|_| EINVAL)?; // The controller configuration applies where the part's leakage calibration matches the // one it was made for, or on a board whose configuration holds for every part. let calibrated = match board.calibration { @@ -217,9 +287,9 @@ impl InitData { [PRIVATE + private.status_config as u64, PRIVATE + private.secondary_extra_1 as u64] }, }; - let profile = m::RootProfile::J714G17S26A428; + let profile = board.graph.profile; let channels = m::derive_channel_table_for_profile(profile, role, - PRIVATE + m::M5_PRIVATE_STATE_GRIDS[index] as u64, + PRIVATE + board.graph.state_grids[index] as u64, instance.status_a, instance.main_config, BUNDLE); let offset = shared.main_config[index]; m::encode_main_config_for_profile(profile, role, &addresses, &instance, diff --git a/drivers/gpu/drm/asahi/g17/mod.rs b/drivers/gpu/drm/asahi/g17/mod.rs index 5efe538a737ef..0b3b0641181b9 100644 --- a/drivers/gpu/drm/asahi/g17/mod.rs +++ b/drivers/gpu/drm/asahi/g17/mod.rs @@ -284,12 +284,24 @@ pub(super) enum Role { Secondary = 1, } +/// RTKit's crash log endpoint, over which the host also publishes a log buffer of its own. +const EP_CRASHLOG: u8 = 0x01; /// RTKit endpoint of firmware messages: initialization, its acknowledgement and event /// notifications. const EP_FIRMWARE: u8 = 0x20; /// RTKit endpoint of host notifications to the firmware. const EP_DOORBELL: u8 = 0x21; +/// Host message on the crashlog endpoint that hands a coprocessor a log buffer of the host's +/// own, which it reaches through its kernel window: the size in 4 KiB pages and the buffer's +/// address, in the fields of a buffer request but under a type of its own. macOS 27.0 sends it +/// to both coprocessors once they run, after RTKit has echoed each one's request for the buffer +/// in its own data segment; without it the firmware asks for that buffer a second time. +const MSG_LOG_BUFFER: u64 = 0x9 << 52; +/// Bit position of [`MSG_LOG_BUFFER`]'s size, in 4 KiB pages. +const MSG_LOG_BUFFER_PAGES_SHIFT: u32 = 44; +/// Address bits of a buffer that [`MSG_LOG_BUFFER`] carries. +const MSG_LOG_BUFFER_ADDR_MASK: u64 = (1 << 44) - 1; /// Bit position of the type field of firmware messages. const MSG_TYPE_SHIFT: u32 = 48; /// Host message handing a coprocessor the address of its initialization root. @@ -298,10 +310,23 @@ const MSG_INITDATA: u64 = 0x81 << MSG_TYPE_SHIFT; const MSG_INITDATA_ADDR_MASK: u64 = (1 << 44) - 1; /// Firmware message acknowledging [`MSG_INITDATA`]. const MSG_INITDATA_ACK: u64 = 0x09; +/// Host message on the doorbell endpoint that answers [`MSG_INITDATA_ACK`]: the acknowledgement +/// type with the host bit. macOS 27.0 on the J813 sends it to both roles a few milliseconds after +/// it hands them their initialization root (aura's trace: 0x89000000000000 on endpoint 0x21). +const MSG_INITDATA_DONE: u64 = 0x89 << MSG_TYPE_SHIFT; /// Firmware message announcing new records in the event and trace rings. const MSG_EVENTS: u64 = 0x42 << MSG_TYPE_SHIFT; /// Primary processor notification for newly published device-control records. const MSG_CONTROL_NOTIFY: u64 = (0x84 << MSG_TYPE_SHIFT) | 0x11; + +/// The board is the J813 ([`fw::m5::initdata::Board::j813_forms`]), whose firmware (common +/// 26A428) takes work in the G17P forms. macOS 27.0 there submits through the doorbell alone: it +/// never writes the M5 Pro's host port (sgx+0x4090/0x4098) or the KSM queue registers +/// (sgx+0x21068..0x21080), and rings render doorbells as 0x83 | priority << 2 (0 and 8 in its +/// trace; compute is | 2). +pub(crate) static J813_FORMS: core::sync::atomic::AtomicBool = + core::sync::atomic::AtomicBool::new(false); + /// Host message, sent to both coprocessors once both are initialized, ending the power-on /// transition. The firmware then retires the opening device-control records. const MSG_POWER_ON_DONE: u64 = 0x89 << MSG_TYPE_SHIFT; @@ -619,6 +644,15 @@ impl Coprocessor { self.rtkit.send(EP_DOORBELL, msg) } + /// Hands the firmware the host's log buffer ([`MSG_LOG_BUFFER`]). + fn publish_log_buffer(&self, va: u64, size: usize) -> Result { + let msg = MSG_LOG_BUFFER + | (((size >> 12) as u64) << MSG_LOG_BUFFER_PAGES_SHIFT) + | (va & MSG_LOG_BUFFER_ADDR_MASK); + trace::message(trace::MAILBOX, msg); + self.rtkit.send(EP_CRASHLOG, msg) + } + /// Stops the CPU. The firmware keeps its state, but no longer runs. fn stop(&mut self) { if self.running { @@ -1453,6 +1487,8 @@ pub(crate) struct Gpu { _capacity: Pin>>, _reports: Pin>>, _power: Pin>>, + _firmware_log: Pin>>, + _hardware_data: Pin>>, } impl Gpu { @@ -1499,6 +1535,27 @@ impl Gpu { let m5 = cfg.profile.is_m5(); if m5 { let board = cfg.m5.ok_or(EINVAL)?; + J813_FORMS.store(board.j813_forms(), core::sync::atomic::Ordering::Relaxed); + // Hold the J813's GFX power domain for the session, as the G15 and G16 paths do + // for theirs. Without a hold genpd gates it once it finds no active consumer, and + // the GPU's cores can never come up: ps_gfx reads target 0xf with actual 0 while + // the coprocessors run out of their own always-on island and report the GPU busy. + if cfg.chip_id == 0x8142 { + extern "C" { + fn asahi_j813_hold_power(dev: *mut core::ffi::c_void) -> core::ffi::c_int; + fn asahi_j813_power_on_gfx() -> core::ffi::c_int; + } + // SAFETY: `dev` is this probe's device; runtime-PM accounting has no other + // requirement. + kernel::error::to_result(unsafe { + asahi_j813_hold_power(pdev.as_ref().as_raw().cast()) + }) + .inspect_err(|e| dev_err!(dev.as_ref(), "M5: no GFX power hold: {:?}\n", e))?; + // SAFETY: the helper only ioremaps and pokes the T8142 PMGR GPU power-state + // register. + kernel::error::to_result(unsafe { asahi_j813_power_on_gfx() }) + .inspect_err(|e| dev_err!(dev.as_ref(), "M5: GFX domain not powered: {:?}\n", e))?; + } // The M5 layouts are sized for every core slot of the board, fused-off cores // included, so the GPU is matched on clusters and slots, not on enabled cores. if id.num_clusters != board.clusters || id.num_clusters * id.num_cores != board.gpu_cores @@ -1644,6 +1701,14 @@ impl Gpu { directory.read_only_file(c"power", reports::PowerView(shared.clone())), GFP_KERNEL, )?; + let firmware_log = KBox::pin_init( + directory.read_only_file(c"firmware_log", reports::FirmwareLogView(shared.clone())), + GFP_KERNEL, + )?; + let hardware_data = KBox::pin_init( + directory.read_only_file(c"hardware_data", reports::HardwareDataView(shared.clone())), + GFP_KERNEL, + )?; let stats = Arc::new( crate::stats::StatsSnapshot { g17_reader: Some(stats::Reader(shared.clone())), @@ -1663,6 +1728,8 @@ impl Gpu { _capacity: capacity, _reports: reports, _power: power, + _firmware_log: firmware_log, + _hardware_data: hardware_data, }, GFP_KERNEL, )?; @@ -1810,8 +1877,24 @@ impl Gpu { // link the tables of every lower range the M5 graph and its queues use first. uat.prepare_lower_tables(&initdata::InitData::m5_lower_links( uat.kernel_lower_dynamic_range(), + board.graph, )) .inspect_err(at("lower links"))?; + // The J813's firmware runs part of a render in context 0, whose user root is the kernel + // lower VM, and touches the render's metrics through their compact-window address + // there (InitData::bind_m5_metrics): link that window's table too. + if J813_FORMS.load(core::sync::atomic::Ordering::Relaxed) { + let compact = buffer::BlockGeometry::M5_COMPACT.start; + uat.prepare_lower_tables(&[compact..compact + mmu::UAT_PGSZ as u64]) + .inspect_err(at("lower links"))?; + } + // The J813 in native order, from aura's trace of macOS 27.0 on that machine: the start + // control is written first (macOS does it at platform init), the coprocessors are started + // next, and the Fender gating is cleared only once both of them run -- not before them. + // The other boards prepare both after the coprocessor boot (prepare_m5_firmware). + if cfg.chip_id == 0x8142 { + regs.set_m5_start_control().inspect_err(at("M5 start control"))?; + } let primary = Coprocessor::new(pdev, dev, cfg, &uat, shared, Role::Primary) .inspect_err(|e| dev_err!(dev.as_ref(), "Primary coprocessor setup failed: {:?}\n", e))?; let secondary = Coprocessor::new(pdev, dev, cfg, &uat, shared, Role::Secondary) @@ -1881,9 +1964,24 @@ impl Gpu { let session = &mut **retained; // M5 renders take 64-byte metrics records. session.queues.metrics = metrics_ids; - session.primary.boot().inspect_err(at("primary boot"))?; - let secondary = session.secondary.as_mut().ok_or(EINVAL)?; - secondary.boot().inspect_err(at("secondary boot"))?; + // The J813's macOS 27.0 starts GFX1's coprocessor first and GFX's 330 ms later (aura's + // trace), the same way round as the InitData it publishes below. + if cfg.chip_id == 0x8142 { + session.secondary.as_mut().ok_or(EINVAL)?.boot().inspect_err(at("secondary boot"))?; + session.primary.boot().inspect_err(at("primary boot"))?; + } else { + session.primary.boot().inspect_err(at("primary boot"))?; + session.secondary.as_mut().ok_or(EINVAL)?.boot().inspect_err(at("secondary boot"))?; + } + // Both coprocessors run: clear the J813's Fender gating here, where macOS 27.0 clears it + // (it reads 0x6 and writes 0 after both ASCs are running and the first mailbox exchange). + if cfg.chip_id == 0x8142 { + session + .bridge + .regs + .enable_g17_gating() + .inspect_err(at("G17 gating after the coprocessor boot"))?; + } // The firmware built its root and kernel-window tables while booting. session.uat.invalidate_reserved_tables(); let root_entries = session @@ -1915,21 +2013,45 @@ impl Gpu { .confirm_firmware_root_entries(root_entries) .inspect_err(at("root confirmation"))?; session.uat.mirror_secondary_root().inspect_err(at("secondary root"))?; - session - .bridge - .regs - .prepare_m5_firmware() - .inspect_err(at("M5 preparation"))?; + if cfg.chip_id == 0x8142 { + // The J813's log buffers, GFX1's first and then GFX's, in macOS 27.0's order and + // before any endpoint of theirs is started. + for role in [Role::Secondary, Role::Primary] { + let (va, size) = initdata::InitData::fw_log(role); + let coprocessor = match role { + Role::Primary => &session.primary, + Role::Secondary => session.secondary.as_ref().ok_or(EINVAL)?, + }; + coprocessor.publish_log_buffer(va, size).inspect_err(at("log buffer"))?; + } + } else { + session + .bridge + .regs + .prepare_m5_firmware() + .inspect_err(at("M5 preparation"))?; + } let ack = shared.variant.init_ack; - session - .primary - .init(session.init.root_va(Role::Primary)?, ack) - .inspect_err(at("primary InitData"))?; - dev_info!(dev.as_ref(), "M5 primary initdata acknowledged\n"); - let root_va = session.init.root_va(Role::Secondary)?; + let primary_root = session.init.root_va(Role::Primary)?; + let secondary_root = session.init.root_va(Role::Secondary)?; + // The J813's macOS publishes and acknowledges GFX1's InitData first, then GFX's (its + // 25G76 trace); the M5 Pro's graph starts with the primary. + if cfg.chip_id == 0x8142 { + let secondary = session.secondary.as_mut().ok_or(EINVAL)?; + secondary.init(secondary_root, ack).inspect_err(at("secondary InitData"))?; + secondary.notify(MSG_INITDATA_DONE)?; + dev_info!(dev.as_ref(), "M5 secondary initdata acknowledged\n"); + session.primary.init(primary_root, ack).inspect_err(at("primary InitData"))?; + session.primary.notify(MSG_INITDATA_DONE)?; + dev_info!(dev.as_ref(), "M5 primary initdata acknowledged\n"); + } else { + session.primary.init(primary_root, ack).inspect_err(at("primary InitData"))?; + dev_info!(dev.as_ref(), "M5 primary initdata acknowledged\n"); + let secondary = session.secondary.as_mut().ok_or(EINVAL)?; + secondary.init(secondary_root, ack).inspect_err(at("secondary InitData"))?; + dev_info!(dev.as_ref(), "M5 secondary initdata acknowledged\n"); + } let secondary = session.secondary.as_mut().ok_or(EINVAL)?; - secondary.init(root_va, ack).inspect_err(at("secondary InitData"))?; - dev_info!(dev.as_ref(), "M5 secondary initdata acknowledged\n"); session.primary.notify(MSG_POWER_ON_DONE)?; secondary.notify(MSG_POWER_ON_DONE)?; Self::wait_for_opening(dev, &session.init, &session.primary, session.secondary.as_ref()) diff --git a/drivers/gpu/drm/asahi/g17/object.rs b/drivers/gpu/drm/asahi/g17/object.rs index 0fd65708922f7..59f82f0e65b3d 100644 --- a/drivers/gpu/drm/asahi/g17/object.rs +++ b/drivers/gpu/drm/asahi/g17/object.rs @@ -253,6 +253,13 @@ impl KernelObject { self.size } + /// Reads the contents of the object. The firmware may be writing them, so a reader sees a + /// torn snapshot: for diagnostics, never for anything the host acts on. + pub(crate) fn bytes(&self) -> &[u8] { + // SAFETY: Per the type invariant `ptr` is valid for `size` bytes. + unsafe { core::slice::from_raw_parts(self.ptr.as_ptr(), self.size) } + } + /// Returns the contents of the object. /// /// Only for building an object the firmware does not access concurrently, before its address diff --git a/drivers/gpu/drm/asahi/g17/pmpdash.c b/drivers/gpu/drm/asahi/g17/pmpdash.c new file mode 100644 index 0000000000000..479e417bf2d40 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/pmpdash.c @@ -0,0 +1,116 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* + * GFX power for the J813 (T8142): hold the power domain and keep ps_gfx at its + * top state for the firmware session. + */ + +#include +#include +#include +#include +#include +#include + +int asahi_j813_hold_power(struct device *dev); +int asahi_j813_power_on_gfx(void); + +#define SGX_PS_BASE 0x580e80000ULL +#define SGX_PS_REG 0x110 + +#define PS_TARGET GENMASK(3, 0) +#define PS_ACTUAL GENMASK(7, 4) +#define PS_WAS_PWRGATED BIT(8) +#define PS_WAS_CLKGATED BIT(9) +#define PS_DEV_DISABLE BIT(10) +#define PS_BUSY BIT(11) +#define PS_RESET BIT(12) +#define PS_AUTO_ENABLE BIT(28) +#define PS_FLAGS (PS_WAS_PWRGATED | PS_WAS_CLKGATED) + +static u32 ptd_ps_write(void __iomem *ps, u32 pstate, bool auto_enable) +{ + u32 cur = readl(ps); + u32 val = cur & ~(PS_TARGET | PS_FLAGS | PS_BUSY | PS_DEV_DISABLE | + PS_RESET | PS_AUTO_ENABLE); + + val |= FIELD_PREP(PS_TARGET, pstate); + if (auto_enable) + val |= PS_AUTO_ENABLE; + writel(val, ps); + return val; +} + +static int ptd_ps_wait(void __iomem *ps, u32 pstate, u32 *out) +{ + ktime_t t0 = ktime_get(); + u32 cur; + + for (;;) { + cur = readl(ps); + if (FIELD_GET(PS_ACTUAL, cur) == pstate) + break; + if (ktime_us_delta(ktime_get(), t0) > 20000) { + *out = cur; + return -ETIMEDOUT; + } + usleep_range(50, 100); + } + *out = cur; + return 0; +} + +/* + * hold_power: keep the GFX power domain up for the firmware session, as the G15 + * and G16 paths do for theirs (asahi_g15_hold_power). Nothing held it on the + * J813: the domain is inherited on from the boot loader, genpd gates it when it + * finds no active consumer, and ps_gfx then reads target 0xf with actual 0 while + * the coprocessors run out of their own always-on island. Returns an error when + * the device has no power domain to hold. + */ +int asahi_j813_hold_power(struct device *dev) +{ + int ret; + + pm_runtime_get_noresume(dev); + ret = pm_runtime_set_active(dev); + if (ret) + return ret; + pm_runtime_enable(dev); + return dev->pm_domain ? 0 : -EINVAL; +} + +/* + * power_on: bring the GFX domain to its top power state and leave it there for + * the firmware session. The domain is gated by the time this driver probes + * (genpd finds no active consumer, and marking the device active does not + * request a transition), so ps_gfx reads target 0xf with actual 0 while the + * coprocessors run out of their own always-on island and the GPU's cores can + * never come up. Auto power management is left off, so nothing gates the + * domain again while the session holds it; macOS re-enables it because its own + * stack keeps a demand on the domain. + */ +int asahi_j813_power_on_gfx(void) +{ + void __iomem *ps; + u32 val, cur; + int ret = 0; + + ps = ioremap(SGX_PS_BASE, 0x1000); + if (!ps) + return -ENOMEM; + cur = readl(ps + SGX_PS_REG); + if (FIELD_GET(PS_ACTUAL, cur) == 0xf && !(cur & PS_AUTO_ENABLE)) { + pr_info("asahi: GFX ps already held on (%#x)\n", cur); + iounmap(ps); + return 0; + } + val = ptd_ps_write(ps + SGX_PS_REG, 0xf, false); + if (ptd_ps_wait(ps + SGX_PS_REG, 0xf, &cur)) { + pr_warn("asahi: GFX ps did not power on (wrote %#x, now %#x)\n", val, cur); + ret = -EIO; + } else { + pr_info("asahi: GFX ps held on (wrote %#x, now %#x)\n", val, cur); + } + iounmap(ps); + return ret; +} diff --git a/drivers/gpu/drm/asahi/g17/queue/compute.rs b/drivers/gpu/drm/asahi/g17/queue/compute.rs index 57fbdffca3368..710075cd1bf9b 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute.rs @@ -2661,10 +2661,14 @@ impl Queue { // The board's core layout sizes the operand state, as on G17P. operand_region_bytes: self.graph.operand_state.bank_bytes(), operand_aux_offset: self.graph.operand_state.tail_offset(), - operand_bytes: self.graph.operand_state.size, resource_control: plan.map_or(0x54b, |p| p.resource_flags), + operand_bytes: self.graph.operand_state.size, resource_control: plan.map_or( + // macOS 27.0 on the J813 runs every compute program with one secondary and + // one imageblock unit beside the primary one (0x0104054b). + if board.j813_forms() { 0x0104_054b } else { 0x54b }, |p| p.resource_flags), dispatch_control: crate::g17::fw::m5::compute::dispatch_control(board, tuple.dispatch_base), auxiliary_control: tuple.auxiliary_control, }, + common26_tail: board.graph.is_common26(), descriptor_va: descriptor_low, // M5 Pro's descriptors carry the ordinal as the submission sequence too, as macOS's do. submit_sequence: if board.completion_inputs() { u64::from(self.ordinal) } else { 0 }, @@ -2674,10 +2678,13 @@ impl Queue { kick_timestamp: kick_timestamp.get(), completion_ordinal: crate::g17::fw::m5::compute::completion_ordinal(board, self.ordinal), status_va: args.status_va, timestamp_va: [timestamp_va, timestamp_va + 8], - shared_control_va: self.pool.control_va(), event_metadata: [0;16], - usage_fw_va: self.graph.usage.gpu_va(), completion_identity: 1, + shared_control_va: self.pool.control_va(), + event_metadata: crate::g17::fw::m5::compute::completion_forms(board).1, + usage_fw_va: self.graph.usage.gpu_va(), + completion_identity: crate::g17::fw::m5::compute::completion_forms(board).0, }}); - // Explicit dependencies take the board's explicit class; the implicit parent, last, + // Explicit dependencies take the board's class + // ([`crate::g17::fw::m5::compute::explicit_dependency`]); the implicit parent, last, // takes its own ([`crate::g17::fw::m5::compute::parent_dependency`]). let mut m5_dependency_types = [m5_board.map_or( crate::g17::fw::m5::compute::COMPUTE_PIPELINED_DEPENDENCY, @@ -2800,6 +2807,11 @@ impl Queue { })?; visible = true; self.unpublished_binding = false; + // The J813 (common 26A428) reads the descriptor from the item ring, after + // the configuration, as macOS 27.0's compute ring has it. + if crate::g17::J813_FORMS.load(Ordering::Relaxed) { + producer = self.graph.append(producer, descriptor_high, || {})?; + } // This Active and its Packet pins now own all visible records. if let Some(configuration) = self.configuration.take() { configuration.release(); } } else if first { @@ -2887,7 +2899,7 @@ impl Queue { } prepared_kick.commit(); self.submitted = self.submitted.checked_add(1).ok_or(EOVERFLOW)?; - let announce_va = if self.graph.m5 { + let announce_va = if self.graph.m5 && !crate::g17::J813_FORMS.load(Ordering::Relaxed) { self.graph.record(record_slot, ANNOUNCE, 0x40, crate::g17::fw::m5::queue::notification(qid, kick_timestamp)?)? } else { self.graph.record(record_slot, ANNOUNCE, 0x40, announce)? }; diff --git a/drivers/gpu/drm/asahi/g17/queue/mod.rs b/drivers/gpu/drm/asahi/g17/queue/mod.rs index e3be25d0ceb20..2b86aeff24f1e 100644 --- a/drivers/gpu/drm/asahi/g17/queue/mod.rs +++ b/drivers/gpu/drm/asahi/g17/queue/mod.rs @@ -433,7 +433,20 @@ impl Queue { bytes: &[u8], objects: Pin<&xarray::XArray>>, ) -> Result { - let parse_guard = self.context.vm().retain_first_job()?; + // A refused submission reaches userspace as EINVAL alone, so name the step that + // refused it: the command's own requirements are only one of several. + let refused = |step: &str, error: kernel::error::Error| { + static REFUSED: core::sync::atomic::AtomicU32 = core::sync::atomic::AtomicU32::new(0); + if REFUSED.fetch_add(1, core::sync::atomic::Ordering::Relaxed) < 8 { + pr_warn!("G17: refused a submission at {}: {:?}\n", step, error); + } + error + }; + let parse_guard = self + .context + .vm() + .retain_first_job() + .map_err(|error| refused("pinning the VM", error))?; let mut parser = Parser::new(bytes); // All historical ordinals use the same entry counters; publication // starts only after the entire ioctl has been validated and pinned. @@ -441,36 +454,47 @@ impl Queue { let mut preceding = [0; 2]; let mut pending = KVec::with_capacity(1, GFP_KERNEL)?; while let Some(command) = parser.next()? { - let mut payload = self.proof.validate( - self.context.vm(), - self.range.clone(), - &self.window, - self.variant, - &command.payload, - )?; + let mut payload = self + .proof + .validate( + self.context.vm(), + self.range.clone(), + &self.window, + self.variant, + &command.payload, + ) + .map_err(|error| refused("the command's requirements", error))?; let engine = match &payload { Validated::Render { .. } => Engine::Render, Validated::Compute { .. } => Engine::Compute, }; if !self.backend.supports_engine(engine) { return Err(ENOTSUPP); } - self.backend.prepare_command(&mut payload)?; + self.backend + .prepare_command(&mut payload) + .map_err(|error| refused("preparing the command", error))?; let mut order = Order::new( base, preceding, command.barriers, command.payload.prior(), engine, - )?; + ) + .map_err(|error| refused("its barriers and priors", error))?; if let Some((barriers, prior)) = command.payload.tiling_prefixes() { - order.narrow_tiling(&Order::new(base, preceding, barriers, prior, engine)?)?; + let tiling = Order::new(base, preceding, barriers, prior, engine) + .map_err(|error| refused("its tiling prefixes", error))?; + order + .narrow_tiling(&tiling) + .map_err(|error| refused("narrowing to its tiling prefixes", error))?; } preceding[engine.index()] += 1; let destinations = match &payload { - Validated::Render { timestamps, .. } => Destinations::resolve(objects, timestamps)?, + Validated::Render { timestamps, .. } => Destinations::resolve(objects, timestamps), Validated::Compute { timestamps, .. } => { - Destinations::resolve(objects, core::slice::from_ref(timestamps))? + Destinations::resolve(objects, core::slice::from_ref(timestamps)) } - }; + } + .map_err(|error| refused("its timestamp destinations", error))?; pending.push((payload, order, destinations), GFP_KERNEL)?; } parser.finish()?; diff --git a/drivers/gpu/drm/asahi/g17/queue/render.rs b/drivers/gpu/drm/asahi/g17/queue/render.rs index 2b7ee531560b2..27e2cb008bacf 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render.rs @@ -614,6 +614,10 @@ struct RenderBarriers { fragment_types: [u8; 5 + crate::g17::hazard::PUBLICATION_BARRIERS], } +/// The last fragment kick published on any pair: on the J813 the next tiling kick waits for it +/// (see [`Pair::publish`]). +static J813_LAST_FRAGMENT: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); + fn render_barriers( fragment_stage: bool, stage_prefixes: bool, @@ -839,7 +843,11 @@ impl Pair { crate::g17::fw::m5::queue::ConfigProfile { soft_fault_enabled: true, extended: false, - parameter_manager: (stage == 0).then(|| u32::from(self.buffer_id())), + // macOS 27.0's J813 TA configuration names the paired 3D queue at + // +0x82 (qid 1 beside pb_slot 0), not the parameter manager. + parameter_manager: (stage == 0 + && !crate::g17::J813_FORMS.load(Ordering::Relaxed)) + .then(|| u32::from(self.buffer_id())), }, )? } else { @@ -847,7 +855,10 @@ impl Pair { }); } let configs = [configs[0].ok_or(EIO)?, configs[1].ok_or(EIO)?]; - if m5 && self.registered.is_none() { + // The J813's (common 26A428) items carry configuration, descriptor and announcement in + // one publication; macOS 27.0 never publishes a configuration alone there. + let j813 = crate::g17::J813_FORMS.load(Ordering::Relaxed); + if m5 && !j813 && self.registered.is_none() { if self.installed == [true; 2] { // The cold-install configuration pair must be consumed first. let cursors = self.memory.graph.cursors()?; @@ -883,7 +894,10 @@ impl Pair { config.completion_seed_valid = 0; config.completion_seed = 0; } - publication::required(cache[stage], config) + // macOS 27.0 sends a configuration with every J813 kick, so the firmware always + // runs a kick under the context its own record names; leaving it out let a fragment + // run under another client's (or no) context under parallel load. + j813 || publication::required(cache[stage], config) }); // An idle pair's next command starts from the rings' cursors; behind a command in // flight it starts from that command's windows, and only the producers are checked. @@ -906,9 +920,28 @@ impl Pair { return Err(EINVAL); }; let own_tiling = KickDependency::new(qids[0], kicks[0]).ok_or(EINVAL)?; - let barriers = + let mut barriers = render_barriers(pass.fragment_barriers, pass.stage_prefixes, dependencies, own_tiling, self.board.map_or(2, crate::g17::fw::m5::compute::explicit_dependency)); + // On the J813 a tiling kick also waits for the last fragment kick published, on any + // pair, as macOS 27.0 orders them there: its TA kick for render n names render n - 1's + // 3D kick (every such dependency in its captured queues). Without it a TA ran beside + // another context's 3D, and that 3D fetched its own tile data (ISP parameter fetch, + // requestor 0x9a) through the TA's VM slot, or slot 0, and faulted. + if j813 { + let last = KickDependency::from_word( + J813_LAST_FRAGMENT.load(core::sync::atomic::Ordering::Acquire), + ) + .filter(|d| d.word() != 0 && d.qid() != qids[0]); + if let Some(last) = last { + if barriers.tiling_count < barriers.tiling.len() { + barriers.tiling[barriers.tiling_count] = last; + barriers.tiling_types[barriers.tiling_count] = 2; + barriers.tiling_count += 1; + barriers.tiling_types[barriers.tiling_count] = 1; + } + } + } let tiling_barriers = &barriers.tiling[..barriers.tiling_count]; // With the G17D forms a fragment kick with history also names its parent's // predecessor, in class 1, after the self-parent (classes [1, 3, 1] for a lone pass). @@ -950,8 +983,9 @@ impl Pair { }, }, mcache, - // M5 tiling kicks also retain the last event of the mask. - event_mask: if m5 && stage == 0 { + // M5 tiling kicks also retain the last event of the mask, except on the J813, whose + // macOS 27.0 TA and 3D kicks name no event at all. + event_mask: if m5 && stage == 0 && !j813 { [0, 0, 0, 1u64 << 62] } else { abi::RENDER_KICK_EVENT_MASK @@ -1181,6 +1215,7 @@ impl Pair { self.ordinal = next_ordinal; if let Some(fragment) = KickDependency::new(qids[1], kicks[1]) { packet.completion.note_render_publication(fragment); + J813_LAST_FRAGMENT.store(fragment.word(), core::sync::atomic::Ordering::Release); } self.cursors = windows.map(|window| window.target()); for stage in 0..2 { @@ -1558,6 +1593,44 @@ impl Pair { Ok(true) } + /// Evidence for a group whose watchdog expired: whether the firmware consumed its ring + /// entries (the cursors) and whether its halves ran (the pass record, which the tiler and + /// fragment processing write). It separates work the firmware never picked up from work it + /// started and never finished, which the error alone does not say. The first few groups of a + /// boot report; the rest would repeat them. + pub(crate) fn report_timeout(&self, stamps: [u32; 2], state: u32) { + static REPORTED: core::sync::atomic::AtomicU32 = core::sync::atomic::AtomicU32::new(0); + if REPORTED.fetch_add(1, core::sync::atomic::Ordering::Relaxed) >= 4 { + return; + } + let Some(front) = self.active.front() else { + return; + }; + let ordinal = front.ticket.ordinal; + let Ok(snapshot) = observation::snapshot(&self.memory, ordinal) else { + pr_warn!( + "G17: render timeout slot {} ordinal {}: its graph could not be read\n", + self.slot, + ordinal + ); + return; + }; + pr_warn!( + "G17: render timeout slot {} ordinal {} qids {:?}: stamps [{:#x}, {:#x}], recovery state {}, pass [{:#x}, {:#x}], tiler {:?}, fragment {:?}, job list {}\n", + self.slot, + ordinal, + self.qids(), + stamps[0], + stamps[1], + state, + snapshot.pass[0], + snapshot.pass[1], + snapshot.cursors[0], + snapshot.cursors[1], + if snapshot.job_list_empty { "empty" } else { "not empty" } + ); + } + fn selected(&self, masks: Option<[u64; 2]>) -> bool { masks.is_none_or(|masks| completion_noted(self.qids(), masks)) } @@ -1622,7 +1695,12 @@ impl Pair { } else { 0 }; - let required = required.max(feedback); + // The estimate is an upper bound from the tile count, and the buffer's capacity was + // fixed when it was created for this pair's first render. Grow as far as it goes, as the + // firmware's own requests do (firmware_target), instead of failing the render with + // ERANGE: a 16384x16384 no-attachment render after smaller ones lost the device + // (dEQP-GLES31.functional.fbo.no_attachments.maximums.*). + let required = required.max(feedback).min(self.memory.manager.max_blocks()); Ok((required > self.memory.manager.blocks()).then_some(required)) } diff --git a/drivers/gpu/drm/asahi/g17/queue/render/publication.rs b/drivers/gpu/drm/asahi/g17/queue/render/publication.rs index 838d3b17bab86..9ee23de892401 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render/publication.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render/publication.rs @@ -62,7 +62,11 @@ impl<'a> Lane<'a> { timestamp: KickTimestamp, lead: bool, ) -> Result { - let m5 = graph.is_m5(); + // macOS 27.0 on the J813 (common 26A428 interface) publishes the Neo form: its + // TA, 3D and compute item rings carry [configuration], descriptor, KickAnnounce (tag 0xe) + // records, never the M5 Pro's descriptor-less tag-16 notification. + let j813 = crate::g17::J813_FORMS.load(core::sync::atomic::Ordering::Relaxed); + let m5 = graph.is_m5() && !j813; // M5 items carry no descriptor: [configuration], notification. let count = u32::from(present) + u32::from(lead) + if m5 { 1 } else { 2 }; let window = Window::new(base, count, ITEM_RING_ENTRIES)?; @@ -100,10 +104,13 @@ impl<'a> Lane<'a> { if lead { push(graph.config.gpu_va() + (config_offset + memory::LEAD) as u64); } + if j813 && present { + push(graph.config.gpu_va() + config_offset as u64); + } if !m5 { push(descriptor); } - if present { + if present && !j813 { push(graph.config.gpu_va() + config_offset as u64); } push(announce_va); diff --git a/drivers/gpu/drm/asahi/g17/reports.rs b/drivers/gpu/drm/asahi/g17/reports.rs index a3bafd969981d..72771e5c4fe5f 100644 --- a/drivers/gpu/drm/asahi/g17/reports.rs +++ b/drivers/gpu/drm/asahi/g17/reports.rs @@ -132,6 +132,55 @@ impl kernel::debugfs::Writer for PowerView { } } +/// What each coprocessor has written into the log buffer the host published to it, which is +/// where an M5 firmware records what it is doing when it never reports a crash. +pub(super) struct FirmwareLogView(pub(super) kernel::sync::Arc); + +impl kernel::debugfs::Writer for FirmwareLogView { + fn write(&self, f: &mut kernel::fmt::Formatter<'_>) -> kernel::fmt::Result { + let state = self.0.state.lock(); + let Some(firmware) = state.as_deref() else { + return writeln!(f, "online=0"); + }; + for role in [Role::Primary, Role::Secondary] { + let bytes = firmware.init.fw_log_bytes(role); + let used = bytes.iter().rposition(|byte| *byte != 0).map_or(0, |last| last + 1); + writeln!(f, "role={} size={:#x} used={:#x}", role as u32, bytes.len(), used)?; + for (index, line) in bytes[..used].chunks(32).enumerate() { + write!(f, " {:#06x}:", index * 32)?; + for byte in line { + write!(f, " {:02x}", byte)?; + } + writeln!(f)?; + } + } + Ok(()) + } +} + +/// The hardware-data bundle this driver built, for comparing it word for word with the bundle a +/// machine's own firmware runs on. +pub(super) struct HardwareDataView(pub(super) kernel::sync::Arc); + +impl kernel::debugfs::Writer for HardwareDataView { + fn write(&self, f: &mut kernel::fmt::Formatter<'_>) -> kernel::fmt::Result { + let state = self.0.state.lock(); + let Some(firmware) = state.as_deref() else { + return writeln!(f, "online=0"); + }; + let bytes = firmware.init.hardware_data_bytes(); + writeln!(f, "size={:#x}", bytes.len())?; + for (index, line) in bytes.chunks(32).enumerate() { + write!(f, "{:#06x}:", index * 32)?; + for byte in line { + write!(f, " {:02x}", byte)?; + } + writeln!(f)?; + } + Ok(()) + } +} + #[kernel::macros::kunit_tests(asahi_g17_reports)] mod tests { use super::*; diff --git a/drivers/gpu/drm/asahi/g17/runtime/render.rs b/drivers/gpu/drm/asahi/g17/runtime/render.rs index cb5b4f781af62..aadf83cc792a7 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render.rs @@ -737,7 +737,7 @@ impl crate::g17::queue::render::Host for Host<'_> { let result = self.firmware.primary.notify( (0x83 << crate::g17::MSG_TYPE_SHIFT) | u64::from(priority) << 2 - | if m5 { 2 } else { 0 }, + | if m5 && !crate::g17::J813_FORMS.load(core::sync::atomic::Ordering::Relaxed) { 2 } else { 0 }, ); if result.is_err() { self.firmware.primary.state.shared.lose_device_quietly(); diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/pools.rs b/drivers/gpu/drm/asahi/g17/runtime/render/pools.rs index ba287c13456cc..25d81a1976bba 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/pools.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/pools.rs @@ -122,6 +122,17 @@ impl Pools { } fn stage(&self, init: &InitData, pool: &RenderPool) -> Result { + if pool.is_m5() && crate::g17::fw::m5::pool::j813_pools() { + // The J813's rows are the descriptor's first four words, 0x20 apart. + let words = descriptor_words::<4>(init.page_pool_descriptor_table()?, pool.id())?; + pool.repopulate()?; + let descriptor = pool.m5_descriptor()?; + for (word, value) in words.into_iter().zip(descriptor) { + word.store(value, Ordering::Relaxed); + } + fence(Ordering::SeqCst); + return Ok(()); + } if pool.is_m5() { let words = descriptor_words::<8>(init.page_pool_descriptor_table()?, pool.id())?; pool.repopulate()?; @@ -255,6 +266,16 @@ impl Pools { /// A detached pool may capture firmware counters only after its last pair /// stopped using this registration. Callers check all retained pair owners. pub(super) fn save(&self, init: &InitData, pool: &RenderPool) -> Result { + if pool.is_m5() && crate::g17::fw::m5::pool::j813_pools() { + let words = descriptor_words::<4>(init.page_pool_descriptor_table()?, pool.id())?; + fence(Ordering::Acquire); + // The row carries only the first four words; the rest are the pool's own. + let mut descriptor = pool.m5_descriptor()?; + for (value, word) in descriptor.iter_mut().zip(words) { + *value = word.load(Ordering::Relaxed); + } + return pool.store_m5_descriptor(descriptor); + } if pool.is_m5() { let words = descriptor_words::<8>(init.page_pool_descriptor_table()?, pool.id())?; fence(Ordering::Acquire); diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/settle.rs b/drivers/gpu/drm/asahi/g17/runtime/render/settle.rs index 71f0bfc052696..7b13ccba87174 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/settle.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/settle.rs @@ -90,6 +90,7 @@ impl crate::g17::Firmware { .as_mut() .and_then(|entry| entry.pair.as_mut()) { + pair.report_timeout(stamps, state); pair.expire_stamp_wait(packet, stamps, state)?; } } diff --git a/drivers/gpu/drm/asahi/hw/mod.rs b/drivers/gpu/drm/asahi/hw/mod.rs index 94f8f11e81bb4..a58fac5e12c37 100644 --- a/drivers/gpu/drm/asahi/hw/mod.rs +++ b/drivers/gpu/drm/asahi/hw/mod.rs @@ -21,6 +21,7 @@ pub(crate) mod t8103; pub(crate) mod t8112; pub(crate) mod t8140; pub(crate) mod t6050; +pub(crate) mod t8142; #[cfg(CONFIG_DRM_ASAHI_G18)] pub(crate) mod t8152; diff --git a/drivers/gpu/drm/asahi/hw/t8142.rs b/drivers/gpu/drm/asahi/hw/t8142.rs new file mode 100644 index 0000000000000..044e28f20684e --- /dev/null +++ b/drivers/gpu/drm/asahi/hw/t8142.rs @@ -0,0 +1,85 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! M5 (T8142, MacBook Air J813) on the macOS 27.0 (26A428) GPU firmware. +//! +//! Bring-up only: the M5 graph ([`crate::g17::fw::m5::initdata`]) has no J813 board yet, so +//! [`PROBE`] stops at the identity check, after `get_g17_id` has logged the ID registers and +//! core masks, before the UAT or the firmware are touched. +//! +//! J813 (2026-11-07): ID registers 0xb022000 0xb0800 0x11010a 0x40404 0x1320311 0x10030302, +//! family 0xb, variant 2, one die, one cluster of ten core slots; core mask 0x3bd (eight +//! enabled, slots 1 and 6 fused off). +use crate::g17::{Config, CoreLayout, CrashBuffers, FirmwareProfile, Identity}; +use crate::hw::{GpuGen, GpuVariant}; + + +/// Refuses the GPU after logging its identity: no variant byte is 0xff. +pub(crate) const PROBE: Config = Config { + chip_id: 0x8142, + identity: Identity { + family: 0xb, + variant: 0xff, + num_dies: 1, + gpu_variant: GpuVariant::G, + usc_generation: 3, + hal_generation: 300, + }, + uat_ias: 42, + uat_oas: 42, + // As on 25G76: each role publishes its own kernel root, the secondary 0x40000 into + // `pagetables`. + secondary_root_offset: Some(0x40000), + // Each role asks for its crash log at the physical address of its __DATA_SHARED_RO + // (0x100_0250_4000, 0x100_025c_0000 under 27.0), inside the `fw-data` region. + crash_buffers: CrashBuffers::FirmwareData, + profile: FirmwareProfile::M5PublishedRoots, + pm_metrics: None, + windows: &[], + reserved_slots: &[], + perf: None, + opps: 0, + uat_quirks: crate::mmu::UatQuirks::DEFAULT, + region_c: None, + chip: None, + // Never reached: the identity check refuses first. + m5: Some(&crate::g17::fw::m5::initdata::J714), + adopt_root_entries: true, + fabric_bridge: false, + recovery_slot_keys: None, + gpu_generation: GpuGen::G17, + firmware: crate::g17::fw::variant::Firmware::Fixed(&crate::g17::fw::variant::V26A428_M5), + core_layout: CoreLayout { + cores: None, + alternate_core_masks: None, + operand_state: crate::g17::fw::compute::OperandState::TWO_BY_TEN, + deflake: crate::g17::fw::render::Deflake::TWO_CLUSTERS, + compute_blocks: crate::g17::fw::freelist::COMPUTE_BLOCKS, + }, +}; + +/// The J813 on the M5 graph: [`PROBE`] with the GPU's identity and the J813 board. +pub(crate) const CONFIG: Config = Config { + identity: Identity { + family: 0xb, + variant: 2, + num_dies: 1, + gpu_variant: GpuVariant::G, + usc_generation: 3, + hal_generation: 300, + }, + m5: Some(&crate::g17::fw::m5::initdata::J813), + // One cluster of ten core slots; the buffers are sized for all ten. + core_layout: CoreLayout { + cores: None, + alternate_core_masks: None, + operand_state: crate::g17::fw::compute::OperandState::ONE_BY_TEN, + deflake: crate::g17::fw::render::Deflake::ONE_CLUSTER, + compute_blocks: crate::g17::fw::freelist::COMPUTE_BLOCKS, + }, + ..PROBE +}; + +// Each compute operand bank holds 6 KiB per core slot of the board. +const _: () = assert!( + CONFIG.core_layout.operand_state.bank_bytes() + == crate::g17::fw::m5::initdata::J813.gpu_cores * 0x1800 +); diff --git a/drivers/gpu/drm/asahi/regs.rs b/drivers/gpu/drm/asahi/regs.rs index 60e485f29f06b..84679b3b3609b 100644 --- a/drivers/gpu/drm/asahi/regs.rs +++ b/drivers/gpu/drm/asahi/regs.rs @@ -631,6 +631,9 @@ impl G17Resources { &self, submission: &crate::g17::fw::m5::queue::HardwareSubmission, ) -> Result { + if crate::g17::J813_FORMS.load(core::sync::atomic::Ordering::Relaxed) { + return Ok(()); + } let sgx = self.sgx.try_access().ok_or(ENODEV)?; let sgx = (&*sgx).as_view(); for &(offset, value) in submission.writes() { @@ -868,21 +871,21 @@ impl G17Resources { Ok(()) } - /// Enable dynamic power gating of idle G17 GPU cores. The firmware expects this before it is - /// initialized. - /// Prepares the GPU for the M5 firmware before its initialization: sets bit 0 of the - /// firmware start control, which reads 0x30000 or 0x30001 until then, and enables dynamic - /// core power gating. - pub(crate) fn prepare_m5_firmware(&self) -> Result { - { - let sgx = self.sgx.try_access().ok_or(ENODEV)?; - let sgx = (&*sgx).as_view(); - let value = sgx.try_read32(0xe90000)?; - if value != 0x30000 && value != 0x30001 { - return Err(EIO); - } - sgx.try_write32(value | 1, 0xe90000)?; + /// Sets bit 0 of the firmware start control, which reads 0x30000 or 0x30001 until then. + /// macOS writes it at platform init, long before the GPU's power-on and its coprocessors. + pub(crate) fn set_m5_start_control(&self) -> Result { + let sgx = self.sgx.try_access().ok_or(ENODEV)?; + let sgx = (&*sgx).as_view(); + let value = sgx.try_read32(0xe90000)?; + if value != 0x30000 && value != 0x30001 { + return Err(EIO); } + sgx.try_write32(value | 1, 0xe90000) + } + + /// Prepares the GPU for the M5 firmware: the start control, then dynamic core power gating. + pub(crate) fn prepare_m5_firmware(&self) -> Result { + self.set_m5_start_control()?; self.enable_g17_gating() } diff --git a/tools/testing/selftests/drivers/gpu/asahi-m4/test-command-stream.py b/tools/testing/selftests/drivers/gpu/asahi-m4/test-command-stream.py index b94595827a791..b0c92c63fd028 100755 --- a/tools/testing/selftests/drivers/gpu/asahi-m4/test-command-stream.py +++ b/tools/testing/selftests/drivers/gpu/asahi-m4/test-command-stream.py @@ -73,6 +73,8 @@ const E2BIG: i32 = -7; struct Error; impl Error { fn from_errno(error:i32)->i32 { error } } extern crate self as kernel; +// The parser logs why it refuses a command; check the format arguments, print nothing. +macro_rules! pr_warn { ($($arg:tt)*) => {{ let _ = format_args!($($arg)*); }}; } pub mod bindings { pub const EOPNOTSUPP:i32=95; } pub mod prelude { pub type Result = core::result::Result; pub const E2BIG:i32=-7; } mod uapi { include!("FULL_UAPI"); }