From afb88b6ac5f88302490b2945667d27000934f36f Mon Sep 17 00:00:00 2001 From: tinebp Date: Wed, 30 Sep 2026 14:29:29 -0700 Subject: [PATCH] afu: route the command processor through the bank adapter on multi-port shells The XRT shell merged the CP's device master with Vortex at AXI port 0, downstream of VX_mem_to_axi, so every CP transfer left on m_axi_mem_0 whichever bank owned the line. It went unnoticed because xrtsim backed all ports with one RAM and the HBM platforms build the merged single port. With several ports the CP now enters VX_mem_to_axi through VX_membus_from_axi as one more input port, so bank selection applies to the CP and Vortex alike; the single-port shell keeps its original structure. VX_afu_axi_limit holds port 0 to one read and one write in flight, the policy the 2:1 arbiter used to impose, which keeps cycle counts unchanged. xrtsim gives each AXI port its own memory, so a request on the wrong port misses instead of finding the data in a shared image. Co-Authored-By: Claude Fable 5.1 --- docs/designs/command_processor.md | 15 +- docs/designs/fpga_afu_shell.md | 41 +- hw/rtl/afu/common/VX_afu_axi_limit.sv | 99 +++ hw/rtl/afu/common/VX_afu_wrap.sv | 845 +++++++++++++++++--------- sim/xrtsim/xrt_sim.cpp | 28 +- 5 files changed, 708 insertions(+), 320 deletions(-) create mode 100644 hw/rtl/afu/common/VX_afu_axi_limit.sv diff --git a/docs/designs/command_processor.md b/docs/designs/command_processor.md index 77475853a1..c7f7e93962 100644 --- a/docs/designs/command_processor.md +++ b/docs/designs/command_processor.md @@ -672,13 +672,14 @@ host-side instead. They become usable once queues get independent rings. ap_ctrl stub + SCOPE; `0x1000–0x1FFF` → the CP regfile. The legacy launch FSM, DCR path, and dev-caps were **removed**; CP is the sole launch/DCR path (`vx_start = cp_gpu_if.start`). A dedicated - `m_axi_host` port carries the ring - ([`:299-320`](../../hw/rtl/afu/xrt/VX_afu_wrap.sv#L299)); `axi_dev` - shares Vortex's memory bank 0 through a 2-master `VX_mm_axi_arb` - ([`:533`](../../hw/rtl/afu/xrt/VX_afu_wrap.sv#L533)), which holds a - sticky owner per channel. The CP's narrower TID is zero-padded up to the - platform ID width, and `PLATFORM_MEMORY_OFFSET` is subtracted from its - addresses before the arbiter. + `m_axi_host` port carries the ring. With one memory port, `axi_dev` + shares it with Vortex through a 2-master `VX_mm_axi_arb`, which holds a + sticky owner per channel. With several, `axi_dev` is bridged to the + memory bus by `VX_membus_from_axi` and enters the bank adapter + (`VX_mem_to_axi`) as its own input port, upstream of bank selection, so + the CP reaches every memory bank. The CP's device addresses are + offset-relative like Vortex's own, and `PLATFORM_MEMORY_OFFSET` is added + once, at the bank port, for both masters. - **OPAE** ([`hw/rtl/afu/opae/vortex_afu.sv`](../../hw/rtl/afu/opae/vortex_afu.sv)) — `VX_cp_core` instantiated at [`:325`](../../hw/rtl/afu/opae/vortex_afu.sv#L325). MMIO uses a word-address bit-10 demux (the 0x1000 byte boundary) to the diff --git a/docs/designs/fpga_afu_shell.md b/docs/designs/fpga_afu_shell.md index eeef5aa3eb..f866d40b7a 100644 --- a/docs/designs/fpga_afu_shell.md +++ b/docs/designs/fpga_afu_shell.md @@ -23,10 +23,11 @@ DMA-command engine have been removed. |---|---| | [`vortex_afu.v`](../../hw/rtl/afu/xrt/vortex_afu.v) | Vitis RTL-kernel top — thin wrapper instantiating `VX_afu_wrap`. | | [`vortex_afu.vh`](../../hw/rtl/afu/xrt/vortex_afu.vh) | Defines/macros (`GEN_AXI_MEM`, `GEN_AXI_HOST`, the bit-12 window). | -| [`VX_afu_wrap.sv`](../../hw/rtl/afu/xrt/VX_afu_wrap.sv) | The real shell (~696 LOC): AXI-Lite bit-12 demux, `VX_cp_core`, `m_axi_host`, bank-0 `VX_axi_arb2`, the `Vortex_axi` instance. | +| [`VX_afu_wrap.sv`](../../hw/rtl/afu/common/VX_afu_wrap.sv) | The real shell (~1,200 LOC): AXI-Lite bit-12 demux, `VX_cp_core`, `m_axi_host`, the Vortex instance and the device-memory path shared with the CP. | | [`VX_afu_ctrl.sv`](../../hw/rtl/afu/xrt/VX_afu_ctrl.sv) | Slimmed AXI-Lite slave (~322 LOC): `ap_ctrl` stub at 0x00 + a SCOPE serial register pair + SCOPE watchdog. | | [`VX_afu_axil_demux.sv`](../../hw/rtl/afu/common/VX_afu_axil_demux.sv) | AXI-Lite demux splitting the control space on `addr[12]`, one outstanding transaction per direction. | | [`VX_afu_axi_drain.sv`](../../hw/rtl/afu/common/VX_afu_axi_drain.sv) | Outstanding-transaction tracker per AXI master; reports when a port owes the interconnect nothing. | +| [`VX_afu_axi_limit.sv`](../../hw/rtl/afu/common/VX_afu_axi_limit.sv) | Holds an AXI master to one read and one write in flight; applied to port 0 of the multi-port shell. | | [`VX_afu_reset_seq.sv`](../../hw/rtl/afu/common/VX_afu_reset_seq.sv) | Quiesce-before-reset sequencer for the soft reset; refuses rather than resetting a master that will not drain. | - **Control.** Host AXI-Lite `addr[12]` splits the slave: `addr[12]=0` → @@ -39,12 +40,16 @@ DMA-command engine have been removed. with its address, and routing W by a register that only updates at the AW handshake sent AW and W to different slaves and deadlocked the interface. Covered by `hw/unittest/afu_axil_demux`. -- **Memory.** Vortex banks 1..N pass straight to platform AXI; bank 0 - shares with CP `axi_dev` via `VX_axi_arb2` - ([`:506-558`](../../hw/rtl/afu/xrt/VX_afu_wrap.sv#L506)); CP `axi_host` - drives a **dedicated `m_axi_host` AXI master** for the command ring + - host DMA ([`:302-326`](../../hw/rtl/afu/xrt/VX_afu_wrap.sv#L302)). - `PLATFORM_MEMORY_OFFSET` is applied per bank. +- **Memory.** Two structures, selected by the number of memory ports. + *One port* (the merged platforms): `Vortex_axi` and the CP's `axi_dev` + share the port through a 2:1 `VX_mm_axi_arb`, and CP bursts reach the + platform as bursts. *Several ports*: Vortex's memory ports and `axi_dev` + (bridged by `VX_membus_from_axi`) are the inputs of one bank adapter, + `VX_mem_to_axi`, so the CP is merged **upstream** of bank selection and + reaches every bank. In both, port 0 carries one read and one write in + flight (`VX_afu_axi_limit` in the second case). CP `axi_host` drives a + **dedicated `m_axi_host` AXI master** for the command ring + host DMA. + `PLATFORM_MEMORY_OFFSET` is applied per bank, at the port. - **Interrupt.** The AFU `interrupt` pin is driven from `cp_interrupt` ([`:335`](../../hw/rtl/afu/xrt/VX_afu_wrap.sv#L335)). @@ -78,20 +83,22 @@ DMA-command engine have been removed. ## 3. Shared components and asymmetries Reused from the common libraries (not under `afu/`): -[`VX_axi_arb2.sv`](../../hw/rtl/libs/VX_axi_arb2.sv) (XRT bank-0 arbiter), -[`VX_mem_arb.sv`](../../hw/rtl/mem/VX_mem_arb.sv) (OPAE bank-0 arbiter), +[`VX_mm_axi_arb.sv`](../../hw/rtl/libs/VX_mm_axi_arb.sv) (XRT single-port arbiter), +[`VX_mem_to_axi.sv`](../../hw/rtl/libs/VX_mem_to_axi.sv) (XRT multi-port bank adapter), +[`VX_mem_bus_arb.sv`](../../hw/rtl/mem/VX_mem_bus_arb.sv) (OPAE port-0 arbiter), [`VX_avs_adapter.sv`](../../hw/rtl/libs/VX_avs_adapter.sv) / `VX_mem_data_adapter.sv` (OPAE Avalon), and -[`VX_cp_axi_to_membus.sv`](../../hw/rtl/cp/VX_cp_axi_to_membus.sv) (AXI→ -membus bridge, both OPAE bridges). The top-level cores are -[`Vortex.sv`](../../hw/rtl/Vortex.sv) (OPAE, membus ports) and -[`Vortex_axi.sv`](../../hw/rtl/Vortex_axi.sv) (XRT, AXI ports), each -keeping direct `start`/`busy`/`dcr_*` ports. +[`VX_membus_from_axi.sv`](../../hw/rtl/mem/VX_membus_from_axi.sv) (AXI→ +membus bridge: both OPAE bridges and the XRT multi-port device bridge). The +top-level cores are [`Vortex.sv`](../../hw/rtl/Vortex.sv) (OPAE and XRT +multi-port, membus ports) and [`Vortex_axi.sv`](../../hw/rtl/Vortex_axi.sv) +(XRT single-port, AXI ports), each keeping direct `start`/`busy`/`dcr_*` ports. **Key XRT↔OPAE asymmetries:** dedicated host-AXI master (`m_axi_host`) vs. -a CCI-P host-DMA state machine; an interrupt pin vs. none; `VX_axi_arb2` -vs. `VX_mem_arb` for bank-0 sharing; AXI-Lite `addr[12]` vs. CCI-P MMIO -word-address bit 10 for the control demux. Both expose SCOPE over a serial +a CCI-P host-DMA state machine; an interrupt pin vs. none; with several +memory ports, the CP as its own bank-adapter input vs. a `VX_mem_bus_arb` on +Vortex's port 0 (both upstream of bank selection); AXI-Lite `addr[12]` vs. +CCI-P MMIO word-address bit 10 for the control demux. Both expose SCOPE over a serial sideband. The XRT shell's reset-delay shift register is reloaded either by the platform diff --git a/hw/rtl/afu/common/VX_afu_axi_limit.sv b/hw/rtl/afu/common/VX_afu_axi_limit.sv new file mode 100644 index 0000000000..37bc3bf736 --- /dev/null +++ b/hw/rtl/afu/common/VX_afu_axi_limit.sv @@ -0,0 +1,99 @@ +// Copyright © 2019-2023 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +`include "VX_platform.vh" + +// ============================================================================ +// VX_afu_axi_limit — holds an AXI master to one read and one write in flight. +// +// A new AR is withheld until the previous read's last beat, and a new AW until +// the previous write's B. The limits only ever tighten on a handshake, so an +// offer already made to the slave is never withdrawn. +// +// AW and W of one write may fire in either order. W is let through whenever +// its AW has already gone, and AW whenever its W has, whatever the limit says +// by then: withholding half of an accepted write would hang it. +// ============================================================================ + +`TRACING_OFF +module VX_afu_axi_limit ( + input wire clk, + input wire reset, + + input wire in_awvalid, + output wire in_awready, + input wire in_wvalid, + output wire in_wready, + input wire in_wlast, + input wire in_arvalid, + output wire in_arready, + + output wire out_awvalid, + input wire out_awready, + output wire out_wvalid, + input wire out_wready, + output wire out_arvalid, + input wire out_arready, + + input wire b_fire, + input wire r_fire_last +); + reg rd_busy; + reg wr_busy; + reg w_owed; // AW sent, its W still to come + reg aw_owed; // W sent, its AW still to come + + wire aw_allow = aw_owed || ~wr_busy; + wire w_allow = w_owed || (~wr_busy && ~aw_owed); + wire ar_allow = ~rd_busy; + + assign out_awvalid = in_awvalid && aw_allow; + assign in_awready = out_awready && aw_allow; + assign out_wvalid = in_wvalid && w_allow; + assign in_wready = out_wready && w_allow; + assign out_arvalid = in_arvalid && ar_allow; + assign in_arready = out_arready && ar_allow; + + wire aw_fire = out_awvalid && out_awready; + wire w_fire_last = out_wvalid && out_wready && in_wlast; + wire ar_fire = out_arvalid && out_arready; + + always @(posedge clk) begin + if (reset) begin + rd_busy <= 1'b0; + wr_busy <= 1'b0; + w_owed <= 1'b0; + aw_owed <= 1'b0; + end else begin + if (ar_fire) begin + rd_busy <= 1'b1; + end else if (r_fire_last) begin + rd_busy <= 1'b0; + end + if (aw_fire) begin + wr_busy <= 1'b1; + end else if (b_fire) begin + wr_busy <= 1'b0; + end + if (aw_fire && ~w_fire_last) begin + aw_owed <= 1'b0; + w_owed <= ~aw_owed; + end else if (w_fire_last && ~aw_fire) begin + w_owed <= 1'b0; + aw_owed <= ~w_owed; + end + end + end + +endmodule +`TRACING_ON diff --git a/hw/rtl/afu/common/VX_afu_wrap.sv b/hw/rtl/afu/common/VX_afu_wrap.sv index ddeee45431..5f21e4842d 100644 --- a/hw/rtl/afu/common/VX_afu_wrap.sv +++ b/hw/rtl/afu/common/VX_afu_wrap.sv @@ -28,10 +28,9 @@ // register at host-offset 0x000. // // Data plane: -// * Vortex memory banks 0..N-1 ride the platform AXI4 master ports. -// * VX_cp_core has its own axi_m. Bank 0 is shared via VX_mm_axi_arb — -// the arbiter holds a sticky owner per channel until the response -// completes, so CP and Vortex can interleave without deadlock. +// * Memory banks 0..N-1 ride the platform AXI4 master ports. +// * The CP's device master shares them with Vortex: at the AXI port when +// there is one, upstream of bank selection when there are several. // // Launch / DCR: driven solely by the CP through cp_gpu_if (start + DCR). // ============================================================================ @@ -282,7 +281,7 @@ module VX_afu_wrap import VX_gpu_pkg::*; #( ); // Soft-resettable subsystem domain: every block holding state that must - // clear on a device soft reset (CP command state, bank-0 arbitration). + // clear on a device soft reset (CP command state, the shared memory path). // The AXI-Lite control path stays on `reset` alone so it can complete // the very write that triggers the sequence. wire subsys_reset = reset || vx_reset; @@ -330,7 +329,7 @@ module VX_afu_wrap import VX_gpu_pkg::*; #( // Command Processor // ======================================================================== VX_cp_gpu_if cp_gpu_if (); - // CP device-memory master (shares Vortex bank 0 via VX_mm_axi_arb). + // CP device-memory master. VX_mem_axi_if #(.ADDR_W(64), .DATA_W(C_M_AXI_MEM_DATA_WIDTH), .ID_W(`VX_CP_AXI_TID_WIDTH)) cp_axi_dev (); // CP host-memory master (command ring + host side of DMA → m_axi_host). @@ -450,104 +449,534 @@ module VX_afu_wrap import VX_gpu_pkg::*; #( assign m_axi_mem_araddr_a[i] = C_M_AXI_MEM_ADDR_WIDTH'(m_axi_mem_araddr_u[i]) + platform_memory_offsets[i]; end - // ---- Intermediate Vortex AXI signals (per-bank) — arbiter sits on bank 0 ---- - wire vx_awvalid_a [C_M_AXI_MEM_NUM_BANKS]; - wire vx_awready_a [C_M_AXI_MEM_NUM_BANKS]; - wire [M_AXI_MEM_ADDR_WIDTH-1:0] vx_awaddr_a [C_M_AXI_MEM_NUM_BANKS]; - wire [C_M_AXI_MEM_ID_WIDTH-1:0] vx_awid_a [C_M_AXI_MEM_NUM_BANKS]; - wire [7:0] vx_awlen_a [C_M_AXI_MEM_NUM_BANKS]; - - wire vx_wvalid_a [C_M_AXI_MEM_NUM_BANKS]; - wire vx_wready_a [C_M_AXI_MEM_NUM_BANKS]; - wire [C_M_AXI_MEM_DATA_WIDTH-1:0] vx_wdata_a [C_M_AXI_MEM_NUM_BANKS]; - wire [C_M_AXI_MEM_DATA_WIDTH/8-1:0] vx_wstrb_a [C_M_AXI_MEM_NUM_BANKS]; - wire vx_wlast_a [C_M_AXI_MEM_NUM_BANKS]; - - wire vx_bvalid_a [C_M_AXI_MEM_NUM_BANKS]; - wire vx_bready_a [C_M_AXI_MEM_NUM_BANKS]; - wire [C_M_AXI_MEM_ID_WIDTH-1:0] vx_bid_a [C_M_AXI_MEM_NUM_BANKS]; - wire [1:0] vx_bresp_a [C_M_AXI_MEM_NUM_BANKS]; - - wire vx_arvalid_a [C_M_AXI_MEM_NUM_BANKS]; - wire vx_arready_a [C_M_AXI_MEM_NUM_BANKS]; - wire [M_AXI_MEM_ADDR_WIDTH-1:0] vx_araddr_a [C_M_AXI_MEM_NUM_BANKS]; - wire [C_M_AXI_MEM_ID_WIDTH-1:0] vx_arid_a [C_M_AXI_MEM_NUM_BANKS]; - wire [7:0] vx_arlen_a [C_M_AXI_MEM_NUM_BANKS]; - - wire vx_rvalid_a [C_M_AXI_MEM_NUM_BANKS]; - wire vx_rready_a [C_M_AXI_MEM_NUM_BANKS]; - wire [C_M_AXI_MEM_DATA_WIDTH-1:0] vx_rdata_a [C_M_AXI_MEM_NUM_BANKS]; - wire vx_rlast_a [C_M_AXI_MEM_NUM_BANKS]; - wire [C_M_AXI_MEM_ID_WIDTH-1:0] vx_rid_a [C_M_AXI_MEM_NUM_BANKS]; - wire [1:0] vx_rresp_a [C_M_AXI_MEM_NUM_BANKS]; - + // ======================================================================== + // Device memory. + // + // With one memory port the CP shares it with Vortex through an AXI + // arbiter, and its DMA bursts reach the platform as bursts. + // + // With more than one, the two meet upstream of the bank adapter instead, + // so a single bank-select function routes every request whichever master + // issued it. Merging the CP at one bank's port would confine it to that + // bank: a line Vortex fetches from bank k would never see the image the + // host uploaded. The CP's bursts are split into lines on the way, since + // consecutive lines of a burst belong to different banks. + // ======================================================================== `SCOPE_IO_SWITCH (2); - Vortex_axi #( - .AXI_DATA_WIDTH (C_M_AXI_MEM_DATA_WIDTH), - .AXI_ADDR_WIDTH (M_AXI_MEM_ADDR_WIDTH), - .AXI_TID_WIDTH (C_M_AXI_MEM_ID_WIDTH), - .AXI_NUM_BANKS (C_M_AXI_MEM_NUM_BANKS) - ) vortex_axi ( - `SCOPE_IO_BIND (1) - - .clk (clk), - .reset (vx_reset), - - .m_axi_awvalid (vx_awvalid_a), - .m_axi_awready (vx_awready_a), - .m_axi_awaddr (vx_awaddr_a), - .m_axi_awid (vx_awid_a), - .m_axi_awlen (vx_awlen_a), - `UNUSED_PIN (m_axi_awsize), - `UNUSED_PIN (m_axi_awburst), - `UNUSED_PIN (m_axi_awlock), - `UNUSED_PIN (m_axi_awcache), - `UNUSED_PIN (m_axi_awprot), - `UNUSED_PIN (m_axi_awqos), - `UNUSED_PIN (m_axi_awregion), - - .m_axi_wvalid (vx_wvalid_a), - .m_axi_wready (vx_wready_a), - .m_axi_wdata (vx_wdata_a), - .m_axi_wstrb (vx_wstrb_a), - .m_axi_wlast (vx_wlast_a), - - .m_axi_bvalid (vx_bvalid_a), - .m_axi_bready (vx_bready_a), - .m_axi_bid (vx_bid_a), - .m_axi_bresp (vx_bresp_a), - - .m_axi_arvalid (vx_arvalid_a), - .m_axi_arready (vx_arready_a), - .m_axi_araddr (vx_araddr_a), - .m_axi_arid (vx_arid_a), - .m_axi_arlen (vx_arlen_a), - `UNUSED_PIN (m_axi_arsize), - `UNUSED_PIN (m_axi_arburst), - `UNUSED_PIN (m_axi_arlock), - `UNUSED_PIN (m_axi_arcache), - `UNUSED_PIN (m_axi_arprot), - `UNUSED_PIN (m_axi_arqos), - `UNUSED_PIN (m_axi_arregion), - - .m_axi_rvalid (vx_rvalid_a), - .m_axi_rready (vx_rready_a), - .m_axi_rdata (vx_rdata_a), - .m_axi_rlast (vx_rlast_a), - .m_axi_rid (vx_rid_a), - .m_axi_rresp (vx_rresp_a), - - .dcr_req_valid (dcr_req_valid), - .dcr_req_rw (dcr_req_rw), - .dcr_req_addr (dcr_req_addr), - .dcr_req_data (dcr_req_data), - .dcr_rsp_valid (dcr_rsp_valid), - .dcr_rsp_data (dcr_rsp_data), - - .start (vx_start), - .busy (vx_busy) - ); + // Address requests ahead of the request gate. + wire pre_awvalid_a [C_M_AXI_MEM_NUM_BANKS]; + wire pre_awready_a [C_M_AXI_MEM_NUM_BANKS]; + wire pre_arvalid_a [C_M_AXI_MEM_NUM_BANKS]; + wire pre_arready_a [C_M_AXI_MEM_NUM_BANKS]; + + if (C_M_AXI_MEM_NUM_BANKS == 1) begin : g_single_port + + // ---- Vortex's AXI port, ahead of the arbiter ---- + wire vx_awvalid_a [C_M_AXI_MEM_NUM_BANKS]; + wire vx_awready_a [C_M_AXI_MEM_NUM_BANKS]; + wire [M_AXI_MEM_ADDR_WIDTH-1:0] vx_awaddr_a [C_M_AXI_MEM_NUM_BANKS]; + wire [C_M_AXI_MEM_ID_WIDTH-1:0] vx_awid_a [C_M_AXI_MEM_NUM_BANKS]; + wire [7:0] vx_awlen_a [C_M_AXI_MEM_NUM_BANKS]; + + wire vx_wvalid_a [C_M_AXI_MEM_NUM_BANKS]; + wire vx_wready_a [C_M_AXI_MEM_NUM_BANKS]; + wire [C_M_AXI_MEM_DATA_WIDTH-1:0] vx_wdata_a [C_M_AXI_MEM_NUM_BANKS]; + wire [C_M_AXI_MEM_DATA_WIDTH/8-1:0] vx_wstrb_a [C_M_AXI_MEM_NUM_BANKS]; + wire vx_wlast_a [C_M_AXI_MEM_NUM_BANKS]; + + wire vx_bvalid_a [C_M_AXI_MEM_NUM_BANKS]; + wire vx_bready_a [C_M_AXI_MEM_NUM_BANKS]; + wire [C_M_AXI_MEM_ID_WIDTH-1:0] vx_bid_a [C_M_AXI_MEM_NUM_BANKS]; + wire [1:0] vx_bresp_a [C_M_AXI_MEM_NUM_BANKS]; + + wire vx_arvalid_a [C_M_AXI_MEM_NUM_BANKS]; + wire vx_arready_a [C_M_AXI_MEM_NUM_BANKS]; + wire [M_AXI_MEM_ADDR_WIDTH-1:0] vx_araddr_a [C_M_AXI_MEM_NUM_BANKS]; + wire [C_M_AXI_MEM_ID_WIDTH-1:0] vx_arid_a [C_M_AXI_MEM_NUM_BANKS]; + wire [7:0] vx_arlen_a [C_M_AXI_MEM_NUM_BANKS]; + + wire vx_rvalid_a [C_M_AXI_MEM_NUM_BANKS]; + wire vx_rready_a [C_M_AXI_MEM_NUM_BANKS]; + wire [C_M_AXI_MEM_DATA_WIDTH-1:0] vx_rdata_a [C_M_AXI_MEM_NUM_BANKS]; + wire vx_rlast_a [C_M_AXI_MEM_NUM_BANKS]; + wire [C_M_AXI_MEM_ID_WIDTH-1:0] vx_rid_a [C_M_AXI_MEM_NUM_BANKS]; + wire [1:0] vx_rresp_a [C_M_AXI_MEM_NUM_BANKS]; + + Vortex_axi #( + .AXI_DATA_WIDTH (C_M_AXI_MEM_DATA_WIDTH), + .AXI_ADDR_WIDTH (M_AXI_MEM_ADDR_WIDTH), + .AXI_TID_WIDTH (C_M_AXI_MEM_ID_WIDTH), + .AXI_NUM_BANKS (C_M_AXI_MEM_NUM_BANKS) + ) vortex_axi ( + `SCOPE_IO_BIND (1) + + .clk (clk), + .reset (vx_reset), + + .m_axi_awvalid (vx_awvalid_a), + .m_axi_awready (vx_awready_a), + .m_axi_awaddr (vx_awaddr_a), + .m_axi_awid (vx_awid_a), + .m_axi_awlen (vx_awlen_a), + `UNUSED_PIN (m_axi_awsize), + `UNUSED_PIN (m_axi_awburst), + `UNUSED_PIN (m_axi_awlock), + `UNUSED_PIN (m_axi_awcache), + `UNUSED_PIN (m_axi_awprot), + `UNUSED_PIN (m_axi_awqos), + `UNUSED_PIN (m_axi_awregion), + + .m_axi_wvalid (vx_wvalid_a), + .m_axi_wready (vx_wready_a), + .m_axi_wdata (vx_wdata_a), + .m_axi_wstrb (vx_wstrb_a), + .m_axi_wlast (vx_wlast_a), + + .m_axi_bvalid (vx_bvalid_a), + .m_axi_bready (vx_bready_a), + .m_axi_bid (vx_bid_a), + .m_axi_bresp (vx_bresp_a), + + .m_axi_arvalid (vx_arvalid_a), + .m_axi_arready (vx_arready_a), + .m_axi_araddr (vx_araddr_a), + .m_axi_arid (vx_arid_a), + .m_axi_arlen (vx_arlen_a), + `UNUSED_PIN (m_axi_arsize), + `UNUSED_PIN (m_axi_arburst), + `UNUSED_PIN (m_axi_arlock), + `UNUSED_PIN (m_axi_arcache), + `UNUSED_PIN (m_axi_arprot), + `UNUSED_PIN (m_axi_arqos), + `UNUSED_PIN (m_axi_arregion), + + .m_axi_rvalid (vx_rvalid_a), + .m_axi_rready (vx_rready_a), + .m_axi_rdata (vx_rdata_a), + .m_axi_rlast (vx_rlast_a), + .m_axi_rid (vx_rid_a), + .m_axi_rresp (vx_rresp_a), + + .dcr_req_valid (dcr_req_valid), + .dcr_req_rw (dcr_req_rw), + .dcr_req_addr (dcr_req_addr), + .dcr_req_data (dcr_req_data), + .dcr_rsp_valid (dcr_rsp_valid), + .dcr_rsp_data (dcr_rsp_data), + + .start (vx_start), + .busy (vx_busy) + ); + + // ---- 2:1 arbiter merges Vortex + CP axi_dev ---- + // Pad CP's narrower ID into the platform ID width so the arbiter sees + // identical signal widths from both sources. + wire [C_M_AXI_MEM_ID_WIDTH-1:0] cp_awid_padded = + {{(C_M_AXI_MEM_ID_WIDTH - `VX_CP_AXI_TID_WIDTH){1'b0}}, cp_axi_dev.awid}; + wire [C_M_AXI_MEM_ID_WIDTH-1:0] cp_arid_padded = + {{(C_M_AXI_MEM_ID_WIDTH - `VX_CP_AXI_TID_WIDTH){1'b0}}, cp_axi_dev.arid}; + + // The CP's device addresses come from the same host-side allocator as the + // pointers handed to the cores -- Device::global_mem_, based at + // VX_MEM_USER_BASE_ADDR -- so they are offset-relative already, exactly + // like vx_awaddr_a[0]. Feed them to the arbiter unchanged and let + // PLATFORM_MEMORY_OFFSET be applied once, at the bank port, for both + // masters. Subtracting it here would cancel that re-offset and leave every + // CP DMA pointed outside the platform's memory aperture. + wire [M_AXI_MEM_ADDR_WIDTH-1:0] cp_awaddr_dev = + M_AXI_MEM_ADDR_WIDTH'(cp_axi_dev.awaddr); + wire [M_AXI_MEM_ADDR_WIDTH-1:0] cp_araddr_dev = + M_AXI_MEM_ADDR_WIDTH'(cp_axi_dev.araddr); + + // Packed 2-master AXI arbiter: index 0 = Vortex bank-0 (priority via + // ARBITER="P"), index 1 = CP device master. Input channels are packed + // {cp, vx}; the arbiter's slave-side outputs land in local packed wires + // and are split back to the two masters below. + localparam BANK0_STRB_W = C_M_AXI_MEM_DATA_WIDTH/8; + + wire [1:0] b0_awready; + wire [1:0] b0_wready; + wire [1:0] b0_bvalid; + wire [1:0][C_M_AXI_MEM_ID_WIDTH-1:0] b0_bid; + wire [1:0][1:0] b0_bresp; + wire [1:0] b0_arready; + wire [1:0] b0_rvalid; + wire [1:0][C_M_AXI_MEM_DATA_WIDTH-1:0] b0_rdata; + wire [1:0] b0_rlast; + wire [1:0][C_M_AXI_MEM_ID_WIDTH-1:0] b0_rid; + wire [1:0][1:0] b0_rresp; + + VX_mm_axi_arb #( + .NUM_INPUTS (2), + .ADDR_WIDTH (M_AXI_MEM_ADDR_WIDTH), + .DATA_WIDTH (C_M_AXI_MEM_DATA_WIDTH), + .ID_WIDTH (C_M_AXI_MEM_ID_WIDTH), + .ARBITER ("P"), // index 0 (Vortex bank-0) > index 1 (CP) + .STRB_WIDTH (BANK0_STRB_W) + ) bank0_arb ( + .clk (clk), + .reset (subsys_reset), + + .s_awvalid ({cp_axi_dev.awvalid, vx_awvalid_a[0]}), + .s_awready (b0_awready), + .s_awaddr ({cp_awaddr_dev, vx_awaddr_a[0]}), + .s_awid ({cp_awid_padded, vx_awid_a[0]}), + .s_awlen ({cp_axi_dev.awlen, vx_awlen_a[0]}), + + .s_wvalid ({cp_axi_dev.wvalid, vx_wvalid_a[0]}), + .s_wready (b0_wready), + .s_wdata ({cp_axi_dev.wdata, vx_wdata_a[0]}), + .s_wstrb ({cp_axi_dev.wstrb, vx_wstrb_a[0]}), + .s_wlast ({cp_axi_dev.wlast, vx_wlast_a[0]}), + + .s_bvalid (b0_bvalid), + .s_bready ({cp_axi_dev.bready, vx_bready_a[0]}), + .s_bid (b0_bid), + .s_bresp (b0_bresp), + + .s_arvalid ({cp_axi_dev.arvalid, vx_arvalid_a[0]}), + .s_arready (b0_arready), + .s_araddr ({cp_araddr_dev, vx_araddr_a[0]}), + .s_arid ({cp_arid_padded, vx_arid_a[0]}), + .s_arlen ({cp_axi_dev.arlen, vx_arlen_a[0]}), + + .s_rvalid (b0_rvalid), + .s_rready ({cp_axi_dev.rready, vx_rready_a[0]}), + .s_rdata (b0_rdata), + .s_rlast (b0_rlast), + .s_rid (b0_rid), + .s_rresp (b0_rresp), + + .m_awvalid (pre_awvalid_a[0]), .m_awready (pre_awready_a[0]), + .m_awaddr (m_axi_mem_awaddr_u[0]), .m_awid (m_axi_mem_awid_a[0]), + .m_awlen (m_axi_mem_awlen_a[0]), + .m_wvalid (m_axi_mem_wvalid_a[0]), .m_wready (m_axi_mem_wready_a[0]), + .m_wdata (m_axi_mem_wdata_a[0]), .m_wstrb (m_axi_mem_wstrb_a[0]), + .m_wlast (m_axi_mem_wlast_a[0]), + .m_bvalid (m_axi_mem_bvalid_a[0]), .m_bready (m_axi_mem_bready_a[0]), + .m_bid (m_axi_mem_bid_a[0]), .m_bresp (m_axi_mem_bresp_a[0]), + .m_arvalid (pre_arvalid_a[0]), .m_arready (pre_arready_a[0]), + .m_araddr (m_axi_mem_araddr_u[0]), .m_arid (m_axi_mem_arid_a[0]), + .m_arlen (m_axi_mem_arlen_a[0]), + .m_rvalid (m_axi_mem_rvalid_a[0]), .m_rready (m_axi_mem_rready_a[0]), + .m_rdata (m_axi_mem_rdata_a[0]), .m_rlast (m_axi_mem_rlast_a[0]), + .m_rid (m_axi_mem_rid_a[0]), .m_rresp (m_axi_mem_rresp_a[0]) + ); + + // ---- Split the arbiter's packed slave-side outputs to the two masters ---- + // index 0 = Vortex bank-0, index 1 = CP device master. + // Declared before their first assignment below (the implicit-net footgun: + // a pre-declaration use makes Vivado mint 1-bit nets and keep both). + wire [C_M_AXI_MEM_ID_WIDTH-1:0] cp_axi_dev_bid_full; + wire [C_M_AXI_MEM_ID_WIDTH-1:0] cp_axi_dev_rid_full; + assign vx_awready_a[0] = b0_awready[0]; + assign cp_axi_dev.awready = b0_awready[1]; + assign vx_wready_a[0] = b0_wready[0]; + assign cp_axi_dev.wready = b0_wready[1]; + assign vx_bvalid_a[0] = b0_bvalid[0]; + assign cp_axi_dev.bvalid = b0_bvalid[1]; + assign vx_bid_a[0] = b0_bid[0]; + assign cp_axi_dev_bid_full = b0_bid[1]; + assign vx_bresp_a[0] = b0_bresp[0]; + assign cp_axi_dev.bresp = b0_bresp[1]; + assign vx_arready_a[0] = b0_arready[0]; + assign cp_axi_dev.arready = b0_arready[1]; + assign vx_rvalid_a[0] = b0_rvalid[0]; + assign cp_axi_dev.rvalid = b0_rvalid[1]; + assign vx_rdata_a[0] = b0_rdata[0]; + assign cp_axi_dev.rdata = b0_rdata[1]; + assign vx_rlast_a[0] = b0_rlast[0]; + assign cp_axi_dev.rlast = b0_rlast[1]; + assign vx_rid_a[0] = b0_rid[0]; + assign cp_axi_dev_rid_full = b0_rid[1]; + assign vx_rresp_a[0] = b0_rresp[0]; + assign cp_axi_dev.rresp = b0_rresp[1]; + + // Truncate the arbiter's wider ID back to CP's narrower native ID width. + assign cp_axi_dev.bid = cp_axi_dev_bid_full[`VX_CP_AXI_TID_WIDTH-1:0]; + assign cp_axi_dev.rid = cp_axi_dev_rid_full[`VX_CP_AXI_TID_WIDTH-1:0]; + `UNUSED_VAR (cp_axi_dev_bid_full) + `UNUSED_VAR (cp_axi_dev_rid_full) + + // The optional AXI4 sideband signals (size/burst) are unused by the + // reduced VX_mm_axi_arb view — pin them sink-side so lint stays clean. + `UNUSED_VAR (cp_axi_dev.awsize) + `UNUSED_VAR (cp_axi_dev.awburst) + `UNUSED_VAR (cp_axi_dev.arsize) + `UNUSED_VAR (cp_axi_dev.arburst) + + end else begin : g_multi_port + + localparam DST_LDATAW = `CLOG2(C_M_AXI_MEM_DATA_WIDTH); + localparam SRC_LDATAW = `CLOG2(VX_MEM_DATA_WIDTH); + localparam SUB_LDATAW = DST_LDATAW - SRC_LDATAW; + localparam VX_MEM_TAG_A_WIDTH = VX_MEM_TAG_WIDTH + `MAX(SUB_LDATAW, 0); + localparam VX_MEM_ADDR_A_WIDTH = VX_MEM_ADDR_WIDTH - SUB_LDATAW; + + localparam FAB_NUM_PORTS = VX_MEM_PORTS + 1; + localparam FAB_CP_PORT = VX_MEM_PORTS; + localparam FAB_TAG_WIDTH = `MAX(VX_MEM_TAG_A_WIDTH, `VX_CP_AXI_TID_WIDTH); + localparam FAB_DATA_SIZE = C_M_AXI_MEM_DATA_WIDTH / 8; + localparam CP_LINE_ADDRW = 64 - `CLOG2(FAB_DATA_SIZE); + + wire vx_mem_req_valid [VX_MEM_PORTS]; + wire vx_mem_req_rw [VX_MEM_PORTS]; + wire [VX_MEM_BYTEEN_WIDTH-1:0] vx_mem_req_byteen [VX_MEM_PORTS]; + wire [VX_MEM_ADDR_WIDTH-1:0] vx_mem_req_addr [VX_MEM_PORTS]; + wire [VX_MEM_DATA_WIDTH-1:0] vx_mem_req_data [VX_MEM_PORTS]; + wire [VX_MEM_TAG_WIDTH-1:0] vx_mem_req_tag [VX_MEM_PORTS]; + wire vx_mem_req_ready [VX_MEM_PORTS]; + + wire vx_mem_rsp_valid [VX_MEM_PORTS]; + wire [VX_MEM_DATA_WIDTH-1:0] vx_mem_rsp_data [VX_MEM_PORTS]; + wire [VX_MEM_TAG_WIDTH-1:0] vx_mem_rsp_tag [VX_MEM_PORTS]; + wire vx_mem_rsp_ready [VX_MEM_PORTS]; + + Vortex vortex ( + `SCOPE_IO_BIND (1) + + .clk (clk), + .reset (vx_reset), + + .mem_req_valid (vx_mem_req_valid), + .mem_req_rw (vx_mem_req_rw), + .mem_req_byteen (vx_mem_req_byteen), + .mem_req_addr (vx_mem_req_addr), + .mem_req_data (vx_mem_req_data), + .mem_req_tag (vx_mem_req_tag), + .mem_req_ready (vx_mem_req_ready), + + .mem_rsp_valid (vx_mem_rsp_valid), + .mem_rsp_data (vx_mem_rsp_data), + .mem_rsp_tag (vx_mem_rsp_tag), + .mem_rsp_ready (vx_mem_rsp_ready), + + .dcr_req_valid (dcr_req_valid), + .dcr_req_rw (dcr_req_rw), + .dcr_req_addr (dcr_req_addr), + .dcr_req_data (dcr_req_data), + + .dcr_rsp_valid (dcr_rsp_valid), + .dcr_rsp_data (dcr_rsp_data), + + .start (vx_start), + .busy (vx_busy) + ); + + wire fab_req_valid [FAB_NUM_PORTS]; + wire fab_req_rw [FAB_NUM_PORTS]; + wire [FAB_DATA_SIZE-1:0] fab_req_byteen [FAB_NUM_PORTS]; + wire [VX_MEM_ADDR_A_WIDTH-1:0] fab_req_addr [FAB_NUM_PORTS]; + wire [C_M_AXI_MEM_DATA_WIDTH-1:0] fab_req_data [FAB_NUM_PORTS]; + wire [FAB_TAG_WIDTH-1:0] fab_req_tag [FAB_NUM_PORTS]; + wire fab_req_ready [FAB_NUM_PORTS]; + + wire fab_rsp_valid [FAB_NUM_PORTS]; + wire [C_M_AXI_MEM_DATA_WIDTH-1:0] fab_rsp_data [FAB_NUM_PORTS]; + wire [FAB_TAG_WIDTH-1:0] fab_rsp_tag [FAB_NUM_PORTS]; + wire fab_rsp_ready [FAB_NUM_PORTS]; + + for (genvar i = 0; i < VX_MEM_PORTS; ++i) begin : g_vx_mem_adapter + VX_mem_data_adapter #( + .SRC_DATA_WIDTH (VX_MEM_DATA_WIDTH), + .DST_DATA_WIDTH (C_M_AXI_MEM_DATA_WIDTH), + .SRC_ADDR_WIDTH (VX_MEM_ADDR_WIDTH), + .DST_ADDR_WIDTH (VX_MEM_ADDR_A_WIDTH), + .SRC_TAG_WIDTH (VX_MEM_TAG_WIDTH), + .DST_TAG_WIDTH (FAB_TAG_WIDTH), + .REQ_OUT_BUF (0), + .RSP_OUT_BUF (0) + ) mem_data_adapter ( + .clk (clk), + .reset (vx_reset), + + .mem_req_valid_in (vx_mem_req_valid[i]), + .mem_req_addr_in (vx_mem_req_addr[i]), + .mem_req_rw_in (vx_mem_req_rw[i]), + .mem_req_byteen_in (vx_mem_req_byteen[i]), + .mem_req_data_in (vx_mem_req_data[i]), + .mem_req_tag_in (vx_mem_req_tag[i]), + .mem_req_ready_in (vx_mem_req_ready[i]), + + .mem_rsp_valid_in (vx_mem_rsp_valid[i]), + .mem_rsp_data_in (vx_mem_rsp_data[i]), + .mem_rsp_tag_in (vx_mem_rsp_tag[i]), + .mem_rsp_ready_in (vx_mem_rsp_ready[i]), + + .mem_req_valid_out (fab_req_valid[i]), + .mem_req_addr_out (fab_req_addr[i]), + .mem_req_rw_out (fab_req_rw[i]), + .mem_req_byteen_out (fab_req_byteen[i]), + .mem_req_data_out (fab_req_data[i]), + .mem_req_tag_out (fab_req_tag[i]), + .mem_req_ready_out (fab_req_ready[i]), + + .mem_rsp_valid_out (fab_rsp_valid[i]), + .mem_rsp_data_out (fab_rsp_data[i]), + .mem_rsp_tag_out (fab_rsp_tag[i]), + .mem_rsp_ready_out (fab_rsp_ready[i]) + ); + end + + // The CP's device addresses come from the same host-side allocator as the + // pointers handed to the cores -- Device::global_mem_, based at + // VX_MEM_USER_BASE_ADDR -- so they are offset-relative already, exactly + // like Vortex's own. They enter the fabric unchanged and + // PLATFORM_MEMORY_OFFSET is applied once, at the bank port, for both + // masters. + wire [CP_LINE_ADDRW-1:0] cp_mem_req_addr; + wire [`VX_CP_AXI_TID_WIDTH-1:0] cp_mem_req_tag; + + VX_membus_from_axi #( + .ADDR_W (64), + .DATA_W (C_M_AXI_MEM_DATA_WIDTH), + .ID_W (`VX_CP_AXI_TID_WIDTH) + ) cp_dev_bridge ( + .clk (clk), + .reset (subsys_reset), + .axi_s (cp_axi_dev), + .mem_req_valid (fab_req_valid[FAB_CP_PORT]), + .mem_req_rw (fab_req_rw[FAB_CP_PORT]), + .mem_req_addr (cp_mem_req_addr), + .mem_req_data (fab_req_data[FAB_CP_PORT]), + .mem_req_byteen (fab_req_byteen[FAB_CP_PORT]), + .mem_req_tag (cp_mem_req_tag), + .mem_req_ready (fab_req_ready[FAB_CP_PORT]), + .mem_rsp_valid (fab_rsp_valid[FAB_CP_PORT]), + .mem_rsp_data (fab_rsp_data[FAB_CP_PORT]), + .mem_rsp_tag (fab_rsp_tag[FAB_CP_PORT][`VX_CP_AXI_TID_WIDTH-1:0]), + .mem_rsp_ready (fab_rsp_ready[FAB_CP_PORT]) + ); + + assign fab_req_addr[FAB_CP_PORT] = cp_mem_req_addr[VX_MEM_ADDR_A_WIDTH-1:0]; + assign fab_req_tag[FAB_CP_PORT] = FAB_TAG_WIDTH'(cp_mem_req_tag); + `UNUSED_VAR (cp_mem_req_addr[CP_LINE_ADDRW-1:VX_MEM_ADDR_A_WIDTH]) + `UNUSED_VAR (fab_rsp_tag[FAB_CP_PORT]) + + // The optional AXI4 sideband signals (size/burst) are unused by the + // bridge's reduced view — pin them sink-side so lint stays clean. + `UNUSED_VAR (cp_axi_dev.awsize) + `UNUSED_VAR (cp_axi_dev.awburst) + `UNUSED_VAR (cp_axi_dev.arsize) + `UNUSED_VAR (cp_axi_dev.arburst) + + + wire ad_awvalid_a [C_M_AXI_MEM_NUM_BANKS]; + wire ad_awready_a [C_M_AXI_MEM_NUM_BANKS]; + wire ad_arvalid_a [C_M_AXI_MEM_NUM_BANKS]; + wire ad_arready_a [C_M_AXI_MEM_NUM_BANKS]; + wire ad_wvalid_a [C_M_AXI_MEM_NUM_BANKS]; + wire ad_wready_a [C_M_AXI_MEM_NUM_BANKS]; + + VX_mem_to_axi #( + .DATA_WIDTH (C_M_AXI_MEM_DATA_WIDTH), + .ADDR_WIDTH_IN (VX_MEM_ADDR_A_WIDTH), + .ADDR_WIDTH_OUT (M_AXI_MEM_ADDR_WIDTH), + .TAG_WIDTH_IN (FAB_TAG_WIDTH), + .TAG_WIDTH_OUT (C_M_AXI_MEM_ID_WIDTH), + .NUM_PORTS_IN (FAB_NUM_PORTS), + .NUM_BANKS_OUT (C_M_AXI_MEM_NUM_BANKS), + .INTERLEAVE (`VX_CFG_PLATFORM_MEMORY_INTERLEAVE), + .REQ_OUT_BUF ((VX_MEM_PORTS > 1) ? 2 : 0), + .RSP_OUT_BUF ((VX_MEM_PORTS > 1 || C_M_AXI_MEM_NUM_BANKS > 1) ? 2 : 0) + ) bank_adapter ( + .clk (clk), + .reset (subsys_reset), + + .mem_req_valid (fab_req_valid), + .mem_req_rw (fab_req_rw), + .mem_req_byteen (fab_req_byteen), + .mem_req_addr (fab_req_addr), + .mem_req_data (fab_req_data), + .mem_req_tag (fab_req_tag), + .mem_req_ready (fab_req_ready), + + .mem_rsp_valid (fab_rsp_valid), + .mem_rsp_data (fab_rsp_data), + .mem_rsp_tag (fab_rsp_tag), + .mem_rsp_ready (fab_rsp_ready), + + .m_axi_awvalid (ad_awvalid_a), + .m_axi_awready (ad_awready_a), + .m_axi_awaddr (m_axi_mem_awaddr_u), + .m_axi_awid (m_axi_mem_awid_a), + .m_axi_awlen (m_axi_mem_awlen_a), + `UNUSED_PIN (m_axi_awsize), + `UNUSED_PIN (m_axi_awburst), + `UNUSED_PIN (m_axi_awlock), + `UNUSED_PIN (m_axi_awcache), + `UNUSED_PIN (m_axi_awprot), + `UNUSED_PIN (m_axi_awqos), + `UNUSED_PIN (m_axi_awregion), + + .m_axi_wvalid (ad_wvalid_a), + .m_axi_wready (ad_wready_a), + .m_axi_wdata (m_axi_mem_wdata_a), + .m_axi_wstrb (m_axi_mem_wstrb_a), + .m_axi_wlast (m_axi_mem_wlast_a), + + .m_axi_bvalid (m_axi_mem_bvalid_a), + .m_axi_bready (m_axi_mem_bready_a), + .m_axi_bid (m_axi_mem_bid_a), + .m_axi_bresp (m_axi_mem_bresp_a), + + .m_axi_arvalid (ad_arvalid_a), + .m_axi_arready (ad_arready_a), + .m_axi_araddr (m_axi_mem_araddr_u), + .m_axi_arid (m_axi_mem_arid_a), + .m_axi_arlen (m_axi_mem_arlen_a), + `UNUSED_PIN (m_axi_arsize), + `UNUSED_PIN (m_axi_arburst), + `UNUSED_PIN (m_axi_arlock), + `UNUSED_PIN (m_axi_arcache), + `UNUSED_PIN (m_axi_arprot), + `UNUSED_PIN (m_axi_arqos), + `UNUSED_PIN (m_axi_arregion), + + .m_axi_rvalid (m_axi_mem_rvalid_a), + .m_axi_rready (m_axi_mem_rready_a), + .m_axi_rdata (m_axi_mem_rdata_a), + .m_axi_rlast (m_axi_mem_rlast_a), + .m_axi_rid (m_axi_mem_rid_a), + .m_axi_rresp (m_axi_mem_rresp_a) + ); + + // Bank 0 carries one read and one write at a time, the other banks + // as many as the adapter tracks. Reads in flight cost cycles on a + // saturated channel and save them across several, so moving this + // limit changes cycle counts in both directions. + VX_afu_axi_limit bank0_limit ( + .clk (clk), + .reset (subsys_reset), + .in_awvalid (ad_awvalid_a[0]), + .in_awready (ad_awready_a[0]), + .in_wvalid (ad_wvalid_a[0]), + .in_wready (ad_wready_a[0]), + .in_wlast (m_axi_mem_wlast_a[0]), + .in_arvalid (ad_arvalid_a[0]), + .in_arready (ad_arready_a[0]), + .out_awvalid (pre_awvalid_a[0]), + .out_awready (pre_awready_a[0]), + .out_wvalid (m_axi_mem_wvalid_a[0]), + .out_wready (m_axi_mem_wready_a[0]), + .out_arvalid (pre_arvalid_a[0]), + .out_arready (pre_arready_a[0]), + .b_fire (m_axi_mem_bvalid_a[0] && m_axi_mem_bready_a[0]), + .r_fire_last (m_axi_mem_rvalid_a[0] && m_axi_mem_rready_a[0] + && m_axi_mem_rlast_a[0]) + ); + + for (genvar i = 1; i < C_M_AXI_MEM_NUM_BANKS; ++i) begin : g_bank_direct + assign pre_awvalid_a[i] = ad_awvalid_a[i]; + assign ad_awready_a[i] = pre_awready_a[i]; + assign m_axi_mem_wvalid_a[i] = ad_wvalid_a[i]; + assign ad_wready_a[i] = m_axi_mem_wready_a[i]; + assign pre_arvalid_a[i] = ad_arvalid_a[i]; + assign ad_arready_a[i] = pre_arready_a[i]; + end + + end // ======================================================================== // Request gate + drain tracking. @@ -558,11 +987,6 @@ module VX_afu_wrap import VX_gpu_pkg::*; #( // never gated: a burst whose address the interconnect has already accepted // must be allowed to finish. // ======================================================================== - wire pre_awvalid_a [C_M_AXI_MEM_NUM_BANKS]; - wire pre_awready_a [C_M_AXI_MEM_NUM_BANKS]; - wire pre_arvalid_a [C_M_AXI_MEM_NUM_BANKS]; - wire pre_arready_a [C_M_AXI_MEM_NUM_BANKS]; - wire mem_idle_a [C_M_AXI_MEM_NUM_BANKS]; for (genvar i = 0; i < C_M_AXI_MEM_NUM_BANKS; ++i) begin : g_axi_gate @@ -634,176 +1058,6 @@ module VX_afu_wrap import VX_gpu_pkg::*; #( end assign masters_idle = mem_idle_all && host_idle; - // ---- Banks 1..N-1: direct passthrough ---- - for (genvar i = 1; i < C_M_AXI_MEM_NUM_BANKS; ++i) begin : g_bank_passthrough - assign pre_awvalid_a[i] = vx_awvalid_a[i]; - assign m_axi_mem_awaddr_u[i] = vx_awaddr_a[i]; - assign m_axi_mem_awid_a[i] = vx_awid_a[i]; - assign m_axi_mem_awlen_a[i] = vx_awlen_a[i]; - assign vx_awready_a[i] = pre_awready_a[i]; - - assign m_axi_mem_wvalid_a[i] = vx_wvalid_a[i]; - assign m_axi_mem_wdata_a[i] = vx_wdata_a[i]; - assign m_axi_mem_wstrb_a[i] = vx_wstrb_a[i]; - assign m_axi_mem_wlast_a[i] = vx_wlast_a[i]; - assign vx_wready_a[i] = m_axi_mem_wready_a[i]; - - assign vx_bvalid_a[i] = m_axi_mem_bvalid_a[i]; - assign vx_bid_a[i] = m_axi_mem_bid_a[i]; - assign vx_bresp_a[i] = m_axi_mem_bresp_a[i]; - assign m_axi_mem_bready_a[i] = vx_bready_a[i]; - - assign pre_arvalid_a[i] = vx_arvalid_a[i]; - assign m_axi_mem_araddr_u[i] = vx_araddr_a[i]; - assign m_axi_mem_arid_a[i] = vx_arid_a[i]; - assign m_axi_mem_arlen_a[i] = vx_arlen_a[i]; - assign vx_arready_a[i] = pre_arready_a[i]; - - assign vx_rvalid_a[i] = m_axi_mem_rvalid_a[i]; - assign vx_rdata_a[i] = m_axi_mem_rdata_a[i]; - assign vx_rlast_a[i] = m_axi_mem_rlast_a[i]; - assign vx_rid_a[i] = m_axi_mem_rid_a[i]; - assign vx_rresp_a[i] = m_axi_mem_rresp_a[i]; - assign m_axi_mem_rready_a[i] = vx_rready_a[i]; - end - - // ---- Bank 0: 2:1 arbiter merges Vortex bank-0 + CP axi_m ---- - // Pad CP's narrower ID into the platform ID width so the arbiter sees - // identical signal widths from both sources. - wire [C_M_AXI_MEM_ID_WIDTH-1:0] cp_awid_padded = - {{(C_M_AXI_MEM_ID_WIDTH - `VX_CP_AXI_TID_WIDTH){1'b0}}, cp_axi_dev.awid}; - wire [C_M_AXI_MEM_ID_WIDTH-1:0] cp_arid_padded = - {{(C_M_AXI_MEM_ID_WIDTH - `VX_CP_AXI_TID_WIDTH){1'b0}}, cp_axi_dev.arid}; - - // The CP's device addresses come from the same host-side allocator as the - // pointers handed to the cores -- Device::global_mem_, based at - // VX_MEM_USER_BASE_ADDR -- so they are offset-relative already, exactly - // like vx_awaddr_a[0]. Feed them to the arbiter unchanged and let - // PLATFORM_MEMORY_OFFSET be applied once, at the bank port, for both - // masters. Subtracting it here would cancel that re-offset and leave every - // CP DMA pointed outside the platform's memory aperture. - wire [M_AXI_MEM_ADDR_WIDTH-1:0] cp_awaddr_dev = - M_AXI_MEM_ADDR_WIDTH'(cp_axi_dev.awaddr); - wire [M_AXI_MEM_ADDR_WIDTH-1:0] cp_araddr_dev = - M_AXI_MEM_ADDR_WIDTH'(cp_axi_dev.araddr); - - // Packed 2-master AXI arbiter: index 0 = Vortex bank-0 (priority via - // ARBITER="P"), index 1 = CP device master. Input channels are packed - // {cp, vx}; the arbiter's slave-side outputs land in local packed wires - // and are split back to the two masters below. - localparam BANK0_STRB_W = C_M_AXI_MEM_DATA_WIDTH/8; - - wire [1:0] b0_awready; - wire [1:0] b0_wready; - wire [1:0] b0_bvalid; - wire [1:0][C_M_AXI_MEM_ID_WIDTH-1:0] b0_bid; - wire [1:0][1:0] b0_bresp; - wire [1:0] b0_arready; - wire [1:0] b0_rvalid; - wire [1:0][C_M_AXI_MEM_DATA_WIDTH-1:0] b0_rdata; - wire [1:0] b0_rlast; - wire [1:0][C_M_AXI_MEM_ID_WIDTH-1:0] b0_rid; - wire [1:0][1:0] b0_rresp; - - VX_mm_axi_arb #( - .NUM_INPUTS (2), - .ADDR_WIDTH (M_AXI_MEM_ADDR_WIDTH), - .DATA_WIDTH (C_M_AXI_MEM_DATA_WIDTH), - .ID_WIDTH (C_M_AXI_MEM_ID_WIDTH), - .ARBITER ("P"), // index 0 (Vortex bank-0) > index 1 (CP) - .STRB_WIDTH (BANK0_STRB_W) - ) bank0_arb ( - .clk (clk), - .reset (subsys_reset), - - .s_awvalid ({cp_axi_dev.awvalid, vx_awvalid_a[0]}), - .s_awready (b0_awready), - .s_awaddr ({cp_awaddr_dev, vx_awaddr_a[0]}), - .s_awid ({cp_awid_padded, vx_awid_a[0]}), - .s_awlen ({cp_axi_dev.awlen, vx_awlen_a[0]}), - - .s_wvalid ({cp_axi_dev.wvalid, vx_wvalid_a[0]}), - .s_wready (b0_wready), - .s_wdata ({cp_axi_dev.wdata, vx_wdata_a[0]}), - .s_wstrb ({cp_axi_dev.wstrb, vx_wstrb_a[0]}), - .s_wlast ({cp_axi_dev.wlast, vx_wlast_a[0]}), - - .s_bvalid (b0_bvalid), - .s_bready ({cp_axi_dev.bready, vx_bready_a[0]}), - .s_bid (b0_bid), - .s_bresp (b0_bresp), - - .s_arvalid ({cp_axi_dev.arvalid, vx_arvalid_a[0]}), - .s_arready (b0_arready), - .s_araddr ({cp_araddr_dev, vx_araddr_a[0]}), - .s_arid ({cp_arid_padded, vx_arid_a[0]}), - .s_arlen ({cp_axi_dev.arlen, vx_arlen_a[0]}), - - .s_rvalid (b0_rvalid), - .s_rready ({cp_axi_dev.rready, vx_rready_a[0]}), - .s_rdata (b0_rdata), - .s_rlast (b0_rlast), - .s_rid (b0_rid), - .s_rresp (b0_rresp), - - .m_awvalid (pre_awvalid_a[0]), .m_awready (pre_awready_a[0]), - .m_awaddr (m_axi_mem_awaddr_u[0]), .m_awid (m_axi_mem_awid_a[0]), - .m_awlen (m_axi_mem_awlen_a[0]), - .m_wvalid (m_axi_mem_wvalid_a[0]), .m_wready (m_axi_mem_wready_a[0]), - .m_wdata (m_axi_mem_wdata_a[0]), .m_wstrb (m_axi_mem_wstrb_a[0]), - .m_wlast (m_axi_mem_wlast_a[0]), - .m_bvalid (m_axi_mem_bvalid_a[0]), .m_bready (m_axi_mem_bready_a[0]), - .m_bid (m_axi_mem_bid_a[0]), .m_bresp (m_axi_mem_bresp_a[0]), - .m_arvalid (pre_arvalid_a[0]), .m_arready (pre_arready_a[0]), - .m_araddr (m_axi_mem_araddr_u[0]), .m_arid (m_axi_mem_arid_a[0]), - .m_arlen (m_axi_mem_arlen_a[0]), - .m_rvalid (m_axi_mem_rvalid_a[0]), .m_rready (m_axi_mem_rready_a[0]), - .m_rdata (m_axi_mem_rdata_a[0]), .m_rlast (m_axi_mem_rlast_a[0]), - .m_rid (m_axi_mem_rid_a[0]), .m_rresp (m_axi_mem_rresp_a[0]) - ); - - // ---- Split the arbiter's packed slave-side outputs to the two masters ---- - // index 0 = Vortex bank-0, index 1 = CP device master. - // Declared before their first assignment below (the implicit-net footgun: - // a pre-declaration use makes Vivado mint 1-bit nets and keep both). - wire [C_M_AXI_MEM_ID_WIDTH-1:0] cp_axi_dev_bid_full; - wire [C_M_AXI_MEM_ID_WIDTH-1:0] cp_axi_dev_rid_full; - assign vx_awready_a[0] = b0_awready[0]; - assign cp_axi_dev.awready = b0_awready[1]; - assign vx_wready_a[0] = b0_wready[0]; - assign cp_axi_dev.wready = b0_wready[1]; - assign vx_bvalid_a[0] = b0_bvalid[0]; - assign cp_axi_dev.bvalid = b0_bvalid[1]; - assign vx_bid_a[0] = b0_bid[0]; - assign cp_axi_dev_bid_full = b0_bid[1]; - assign vx_bresp_a[0] = b0_bresp[0]; - assign cp_axi_dev.bresp = b0_bresp[1]; - assign vx_arready_a[0] = b0_arready[0]; - assign cp_axi_dev.arready = b0_arready[1]; - assign vx_rvalid_a[0] = b0_rvalid[0]; - assign cp_axi_dev.rvalid = b0_rvalid[1]; - assign vx_rdata_a[0] = b0_rdata[0]; - assign cp_axi_dev.rdata = b0_rdata[1]; - assign vx_rlast_a[0] = b0_rlast[0]; - assign cp_axi_dev.rlast = b0_rlast[1]; - assign vx_rid_a[0] = b0_rid[0]; - assign cp_axi_dev_rid_full = b0_rid[1]; - assign vx_rresp_a[0] = b0_rresp[0]; - assign cp_axi_dev.rresp = b0_rresp[1]; - - // Truncate the arbiter's wider ID back to CP's narrower native ID width. - assign cp_axi_dev.bid = cp_axi_dev_bid_full[`VX_CP_AXI_TID_WIDTH-1:0]; - assign cp_axi_dev.rid = cp_axi_dev_rid_full[`VX_CP_AXI_TID_WIDTH-1:0]; - `UNUSED_VAR (cp_axi_dev_bid_full) - `UNUSED_VAR (cp_axi_dev_rid_full) - - // The optional AXI4 sideband signals (size/burst) are unused by the - // reduced VX_mm_axi_arb view — pin them sink-side so lint stays clean. - `UNUSED_VAR (cp_axi_dev.awsize) - `UNUSED_VAR (cp_axi_dev.awburst) - `UNUSED_VAR (cp_axi_dev.arsize) - `UNUSED_VAR (cp_axi_dev.arburst) - // We only use addr[12:0] of the AXI-Lite address space; bits 15:13 are // always 0 from the kernel.xml-advertised slave size but Verilator // still flags them — pin to UNUSED. @@ -882,27 +1136,48 @@ module VX_afu_wrap import VX_gpu_pkg::*; #( // disable assertions until full reset reg [`CLOG2(`VX_CFG_RESET_DELAY+1)-1:0] assert_delay_ctr; reg assert_enabled; - initial begin - $assertoff(0, vortex_axi); - end + wire assert_turn_off = (reset || vx_reset) && assert_enabled; + wire assert_turn_on = ~(reset || vx_reset) && ~assert_enabled + && (assert_delay_ctr == (`VX_CFG_RESET_DELAY-1)); always @(posedge clk) begin if (reset || vx_reset) begin assert_delay_ctr <= '0; assert_enabled <= 0; - if (assert_enabled) begin - $assertoff(0, vortex_axi); - end end else begin if (~assert_enabled) begin - if (assert_delay_ctr == (`VX_CFG_RESET_DELAY-1)) begin + if (assert_turn_on) begin assert_enabled <= 1; - $asserton(0, vortex_axi); end else begin assert_delay_ctr <= assert_delay_ctr + 1; end end end end + if (C_M_AXI_MEM_NUM_BANKS == 1) begin : g_assert_single_port + initial begin + $assertoff(0, g_single_port.vortex_axi); + end + always @(posedge clk) begin + if (assert_turn_off) begin + $assertoff(0, g_single_port.vortex_axi); + end + if (assert_turn_on) begin + $asserton(0, g_single_port.vortex_axi); + end + end + end else begin : g_assert_multi_port + initial begin + $assertoff(0, g_multi_port.vortex); + end + always @(posedge clk) begin + if (assert_turn_off) begin + $assertoff(0, g_multi_port.vortex); + end + if (assert_turn_on) begin + $asserton(0, g_multi_port.vortex); + end + end + end `endif `endif diff --git a/sim/xrtsim/xrt_sim.cpp b/sim/xrtsim/xrt_sim.cpp index df10bdb62b..fcab150cfe 100644 --- a/sim/xrtsim/xrt_sim.cpp +++ b/sim/xrtsim/xrt_sim.cpp @@ -141,7 +141,7 @@ class xrt_sim::Impl { public: Impl() : device_(nullptr) - , ram_(nullptr) + , ram_{} , host_ram_(nullptr) , host_alloc_(nullptr) , dram_sim_(VX_CFG_PLATFORM_MEMORY_NUM_BANKS, VX_CFG_PLATFORM_MEMORY_DATA_SIZE, MEM_CLOCK_RATIO) @@ -167,8 +167,8 @@ class xrt_sim::Impl { for (int b = 0; b < VX_CFG_PLATFORM_MEMORY_NUM_BANKS; ++b) { delete mem_alloc_[b]; } - if (ram_) { - delete ram_; + for (auto ram : ram_) { + delete ram; } if (host_alloc_) { delete host_alloc_; @@ -223,8 +223,12 @@ class xrt_sim::Impl { // calculate memory bank size mem_bank_size_ = (1ull << VX_CFG_PLATFORM_MEMORY_ADDR_WIDTH) / VX_CFG_PLATFORM_MEMORY_NUM_BANKS; - // allocate RAM - ram_ = new RAM(0, RAM_PAGE_SIZE); + // One memory per AXI port, as on the platform, where each kernel master + // is wired to its own memory resource. A request issued on the wrong port + // then misses its data instead of finding it in a shared image. + for (int b = 0; b < VX_CFG_PLATFORM_MEMORY_NUM_BANKS; ++b) { + ram_[b] = new RAM(0, RAM_PAGE_SIZE); + } // initialize AXI memory interfaces MP_M_AXI_MEM(VX_CFG_PLATFORM_MEMORY_NUM_BANKS); @@ -311,7 +315,7 @@ class xrt_sim::Impl { if (bank_id >= VX_CFG_PLATFORM_MEMORY_NUM_BANKS) return -1; uint64_t base_addr = bank_id * mem_bank_size_ + addr; - ram_->write(data, base_addr, size); + ram_[bank_id]->write(data, base_addr, size); return 0; } @@ -321,7 +325,7 @@ class xrt_sim::Impl { if (bank_id >= VX_CFG_PLATFORM_MEMORY_NUM_BANKS) return -1; uint64_t base_addr = bank_id * mem_bank_size_ + addr; - ram_->read(data, base_addr, size); + ram_[bank_id]->read(data, base_addr, size); return 0; } @@ -331,7 +335,9 @@ class xrt_sim::Impl { return -1; uint64_t dest_base_addr = bank_id_dest * mem_bank_size_ + dest_addr; uint64_t src_base_addr = bank_id_src * mem_bank_size_ + src_addr; - ram_->copy(dest_base_addr, src_base_addr, size); + std::vector data(size); + ram_[bank_id_src]->read(data.data(), src_base_addr, size); + ram_[bank_id_dest]->write(data.data(), dest_base_addr, size); return 0; } @@ -777,7 +783,7 @@ class xrt_sim::Impl { for (auto& beat : mem_rsp->w_beats) { for (int i = 0; i < VX_CFG_PLATFORM_MEMORY_DATA_SIZE; ++i) { if ((beat.byteen >> i) & 0x1) { - (*ram_)[beat.addr + i] = beat.data[i]; + (*ram_[b])[beat.addr + i] = beat.data[i]; } } } @@ -798,7 +804,7 @@ class xrt_sim::Impl { auto mem_req = new mem_req_t(); mem_req->tag = *m_axi_mem_[b].arid; mem_req->addr = base + uint64_t(beat) * VX_CFG_PLATFORM_MEMORY_DATA_SIZE; - ram_->read(mem_req->data.data(), mem_req->addr, VX_CFG_PLATFORM_MEMORY_DATA_SIZE); + ram_[b]->read(mem_req->data.data(), mem_req->addr, VX_CFG_PLATFORM_MEMORY_DATA_SIZE); mem_req->write = false; mem_req->ready = false; mem_req->last = (beat == len); @@ -983,7 +989,7 @@ class xrt_sim::Impl { } m_axi_mem_t; Vvortex_afu_shim* device_; - RAM* ram_; + RAM* ram_[VX_CFG_PLATFORM_MEMORY_NUM_BANKS]; RAM* host_ram_; MemoryAllocator* host_alloc_; DramSim dram_sim_;