diff --git a/VX_types.toml b/VX_types.toml index 495afbf7cc..dc0f368b2b 100644 --- a/VX_types.toml +++ b/VX_types.toml @@ -429,7 +429,7 @@ VX_RT_HIT_BARY_U = 11 VX_RT_HIT_BARY_V = 12 VX_RT_HIT_PRIMITIVE_ID = 13 VX_RT_HIT_INSTANCE_ID = 14 -VX_RT_HIT_GEOMETRY_INDEX = 15 +VX_RT_HIT_GEOMETRY_INDEX = 15 # VX_RT_HIT_GEOMETRY_MASK bits: gl_GeometryIndexEXT; bit 31: VX_RT_HIT_BACK_FACING VX_RT_HIT_INSTANCE_CUSTOM = 16 VX_RT_OBJECT_RAY_ORIGIN = 17 # object_ray.origin.{x,y,z} (17..19) VX_RT_OBJECT_RAY_DIRECTION = 20 # object_ray.direction.{x,y,z} (20..22) @@ -477,6 +477,14 @@ VX_RT_FLAG_SKIP_AABBS = 0x200 VX_RT_FLAG_ENABLE_CHS = 0x400 VX_RT_FLAG_ENABLE_MISS = 0x800 +[rtu_hit_bits] +# The HIT_GEOMETRY_INDEX word: the leaf's geometry index in the low 28 bits (the +# Vulkan BVH layout keeps flags above them, which the RTU drops), and +# BACK_FACING set when a triangle hit lies on its back face after the instance's +# FLIP_FACING, i.e. gl_HitKindEXT is BACK_FACING. +VX_RT_HIT_GEOMETRY_MASK = 0x0fffffff +VX_RT_HIT_BACK_FACING = 0x80000000 + [rtu_cb_actions] # vx_rt_cb_ret action codes VX_RT_CB_ACCEPT = 1 diff --git a/ci/testcases/unittest.yaml b/ci/testcases/unittest.yaml index 5a3893e535..89c8ba08e8 100644 --- a/ci/testcases/unittest.yaml +++ b/ci/testcases/unittest.yaml @@ -31,3 +31,10 @@ tests: via: script run: "make -C hw/unittest run-tcu-dsp" touches: [hw/rtl/tcu, hw/unittest/tcu_fedp] + # Texel blend (VX_tex_lerp, both forms the sampler uses) against the C model + # the SW sampler and SimX share, and the exact Vulkan f/256 blend, over every + # input. + - id: hw-tex-lerp + via: script + run: "make -C hw/unittest/tex_lerp run" + touches: [hw/rtl/tex, hw/unittest/tex_lerp, sw/common/vx_gfx_abi.h, sw/common/gfx_frag_tex.h] diff --git a/docs/designs/ray_tracing_architecture.md b/docs/designs/ray_tracing_architecture.md index e1590aae68..cc100fc1eb 100644 --- a/docs/designs/ray_tracing_architecture.md +++ b/docs/designs/ray_tracing_architecture.md @@ -299,9 +299,15 @@ Two work products leave the scheduler: returned `hitAttribute`). Robustness details worth naming: a short-stack of depth `RTU_STACK_DEPTH` bounds -per-context node stack RAM; on overflow the walker sets an `ovf` flag and, at -pop-time, **re-descends** the subtree pruned by the tightened `best_t` (bounded by -`RTU_RESTART_CAP = 8` restarts) — a full traversal on a finite stack. A 16-entry +per-context node stack RAM, and an overflow never loses a hit. The walk visits +nodes in rank-path order (each level's child rank in the t-sorted list, an +instance's index in its leaf); a child that does not fit is dropped and the walk +records it, then only descends until it would pop, and instead **restarts** from +the root along the dropped child's rank path, held in a small per-context path +RAM. Everything before that child has been visited, a tightened `best_t` only +culls a suffix of a node's sorted children (so ranks stay valid), and each +restart starts strictly further along, so the walk is exact and terminates on +any tree up to 63 levels deep. A deep tree costs restarts, not hits. A 16-entry box collector insertion-sorts a node's child hits t-ascending so descent is nearest-first. The insertion slot is decoded from the **admit thermometer**: the collected list is sorted and its count mask is a prefix, so the "entries at or @@ -321,20 +327,23 @@ subnormals flushed either way), and `VX_CFG_FMA_LATENCY` / whatever depth results: - **`VX_rtu_box_pe`** — pipelined ray/AABB slab test, one child box per cycle, - emitting `{hit, t_near}`. Dequantizes the node's int8 child corners - (`origin + q·2^exp`), does the slab test with `VX_fma_unit` + `VX_fncp_unit`, - and subtracts the ray origin *before* multiplying by `inv_d` so axis-aligned - rays (`inv_d = ±inf`) stay NaN-free. Also handles raw/procedural boxes. -- **`VX_rtu_tri_pe`** — pipelined Möller–Trumbore triangle test, one triangle per - cycle, emitting `{hit, t, u, v, back_facing}`; reuses `VX_fma_unit`, - `VX_fdiv_unit` (1/det), `VX_fncp_unit`, and `VX_rtu_fdot3`/`fcross3`. The - dot/cross helpers pipeline their 24×24 mantissa products into DSP multipliers - (`LATENCY_IMUL` deep) fed the **raw** mantissas: a flushed (subnormal/zero) - term is discarded downstream in the `VX_rtu_fmac3` accumulator by its zero - product-exponent, so no subnormal-flush select sits in front of the multiplier - inputs and the DSPs launch straight from the source flops. -- **`VX_rtu_xform`** — TLAS world→object transform, `obj = Rᵀ·(ro−t)` — FMA-only - (an orthonormal TLAS rotation needs no determinant or divide). Always built: + emitting `{hit, t_near}`. Mirrors SimX `box_rel` + `ray_box` bit for bit: + corners relative to the ray, `q·2^exp + (origin − ro)` (the product exact), + slabs `rel·inv_d`, culled against the ray interval `[t_min, t_max]` with + `t_max` the committed hit. `inv_d` is `FLT_MAX` for a zero direction + component, so no slab is ever `0·inf`. Also handles raw/procedural boxes + (`origin = +0`). +- **`VX_rtu_tri_pe`** — pipelined watertight triangle test (Woop, Benthin, Wald, + JCGT 2013), all F32: shear, edge functions as rounded products and a rounded + difference (so a shared edge evaluates to exactly negated weights in its two + triangles), `det = (w0 + w1) + w2`, `T` as an FMA chain, one `1/det` scaling + `t`, `u`, `v`. Accepts `t_min < t < t_max` (Vulkan's open triangle interval), + with `t_max` the committed hit, one triangle per cycle, emitting + `{hit, t, u, v, back_facing}`; bit-exact against SimX `ray_triangle`. +- **`VX_rtu_xform`** — TLAS world→object transform. The instance record holds + the world→object matrix, so no inverse is taken; each object-ray component is + three dependent FMAs (`fma(z, m2, fma(y, m1, fma(x, m0, t)))`, the direction + seeded with `x·m0`), 18 FMA units, `3·FMA` deep. Always built: the CW-BVH walker descends `LEAF_INST` natively; only the flat walker's (`WIDTH = 0`) instancing loop is gated by `VX_CFG_RTU_TLAS_ENABLE`. - **`VX_rtu_recip`** — F32 reciprocal for `inv_d`, either a portable LUT+Newton diff --git a/docs/designs/texture_sampler_architecture.md b/docs/designs/texture_sampler_architecture.md index 46ebcc6f86..ee24885894 100644 --- a/docs/designs/texture_sampler_architecture.md +++ b/docs/designs/texture_sampler_architecture.md @@ -256,15 +256,15 @@ stays a plain 4-byte-word interface. colour here instead (§3.1). 2. **U lerps**: per lane and level, 8 `VX_tex_lerp` instances (4 channels × {low, high} texel pairs), each a 3-cycle fixed-point datapath computing - `(s + (s >> 8)) >> 8` with `s = a·(255−f) + b·f + 0x80` — the exact - divide-by-255 rounding, not a plain shift. + `(a·(256−f) + b·f + 0x80) >> 8` — the weight is `f/256`, the 8-bit + subtexel fraction Vulkan's texel filtering defines + (`subTexelPrecisionBits = 8`), rounded to nearest. 3. **V lerp**: 4 more lerps per level blend the two U results, another 3 cycles. 4. **Level lerp**: 4 final lerps blend the two levels' texels by the - request's lod fraction — a fraction of **256** that truncates, the form - the software sampler blends levels in, where a tap weight is a fraction - of 255 (§2.3). A single-level sample carries weight 0 and passes level 0 - through. + request's lod fraction — also a fraction of 256, but truncated rather + than rounded, the form the software sampler blends levels in (§2.3). A + single-level sample carries weight 0 and passes level 0 through. The whole sampler is ~10 cycles fixed latency, one request per cycle throughput, per-channel 8-bit arithmetic — no floating-point anywhere (the diff --git a/hw/rtl/fpu/VX_fma_unit.sv b/hw/rtl/fpu/VX_fma_unit.sv index 9337a46f66..6195be219c 100644 --- a/hw/rtl/fpu/VX_fma_unit.sv +++ b/hw/rtl/fpu/VX_fma_unit.sv @@ -71,7 +71,7 @@ module VX_fma_unit import VX_gpu_pkg::*, VX_fpu_pkg::*; #( if (USE_VENDOR_IP) begin : g_vendor // xil_fma / acl_fmadd compute a*b+c, so the FMA-core opcodes are remapped: - // MUL : a*b + 0 + // MUL : a*b + -0 (-0 is the additive identity: a -0 product stays -0) // ADD/SUB : a*1.0 (+/-) b // MADD/NMADD : (+/-)a*b (+/-) c // The vendor IP rounds round-to-nearest-even only (frm is ignored). @@ -89,7 +89,7 @@ module VX_fma_unit import VX_gpu_pkg::*, VX_fpu_pkg::*; #( if (is_neg) begin // MUL a32 = dataa[31:0]; b32 = datab[31:0]; - c32 = '0; + c32 = 32'h80000000; end else begin // ADD/SUB a32 = dataa[31:0]; b32 = 32'h3f800000; // 1.0f diff --git a/hw/rtl/rtu/VX_rtu_box_pe.sv b/hw/rtl/rtu/VX_rtu_box_pe.sv index 456b05e50d..446e42a1e6 100644 --- a/hw/rtl/rtu/VX_rtu_box_pe.sv +++ b/hw/rtl/rtu/VX_rtu_box_pe.sv @@ -12,22 +12,20 @@ // limitations under the License. // VX_rtu_box_pe — pipelined ray-vs-AABB slab intersector for one child box. -// Streams one box per cycle; emits {hit, t_near} after a fixed latency. +// Streams one box per cycle; emits {hit, t_near} after a fixed latency. Mirrors +// SimX rtu::box_rel + rtu::ray_box op for op: // -// dequant mn[a] = origin[a] + qmin[a] * 2^exp[a] (qmax symmetric) -// slab t0[a] = (mn[a] - ro[a]) * inv_d[a] (t1 from mx) -// lo[a] = min(t0,t1) hi[a] = max(t0,t1) -// reduce t_near = max(t_min, lo[x], lo[y], lo[z]) -// t_far = min(t_max, hi[x], hi[y], hi[z]) -// hit = (t_near <= t_far) +// base c[a] = origin[a] - ro[a] (raw box: +0 - ro[a]) +// corner d[a] = q[a]*2^exp[a] + c[a] (product exact; raw: corner + c) +// slab t0[a] = dmn[a] * inv_d[a] (t1 from dmx) +// reduce lo = max(t_min, min(t0,t1)[*]) hi = min(t_max, max(t0,t1)[*]) +// (fmin/fmax drop a NaN operand) +// hit = lo <= hi, t_near = lo // -// The slab subtracts the ray origin before multiplying by inv_d (rather than -// the algebraically-equal mn*inv_d - ro*inv_d) so axis-aligned rays — where -// inv_d is +/-inf — stay numerically correct: (mn-ro) is finite, so (mn-ro)*inf -// is a signed infinity that the min/max reduction treats as a non-constraining -// slab, instead of inf-inf = NaN. The uint8->fp32 and 2^exp dequant terms are -// combinational; the FP add/mul use VX_fma_unit (a*b±c) and the min/max/compare -// use VX_fncp_unit, register-balanced to the configured latencies. +// The box is culled against the ray interval [t_min, t_max] (t_max is the +// committed hit). inv_d is the ray-setup reciprocal (VX_rtu_recip), FLT_MAX for +// a zero direction component, so no slab is ever 0 * inf. The FP units flush +// subnormals. `include "VX_define.vh" @@ -48,8 +46,7 @@ module VX_rtu_box_pe import VX_gpu_pkg::*, VX_fpu_pkg::*, VX_rtu_pkg::*; #( input wire [2:0][7:0] qmin, input wire [2:0][7:0] qmax, // raw (unquantized) AABB path — procedural-leaf boxes carry float min/max - // directly instead of node-relative quantized corners. raw=0 is bit- - // identical to the quantized path (BVH internal-node box tests). + // directly instead of node-relative quantized corners. input wire raw, input wire [2:0][31:0] raw_min, input wire [2:0][31:0] raw_max, @@ -67,162 +64,167 @@ module VX_rtu_box_pe import VX_gpu_pkg::*, VX_fpu_pkg::*, VX_rtu_pkg::*; #( output wire hit, output wire [31:0] t_near ); - // VX_fncp_unit result latency is 1 (one input pipe reg, OUT_REG=0); its - // LATENCY param only sizes the internal mask pipe, not the result path, so - // size it to 2 to avoid a degenerate [-1:0] mask-pipe slice while the result - // still lands after one cycle. - localparam FNCP_LAT = 1; // result latency for alignment - localparam FNCP_SIZE = 2; // mask-pipe sizing param - localparam LAT_ORIGIN = LATENCY_FMA; // origin - ro - localparam LAT_DEQUANT = LATENCY_FMA; // q*scale + (origin - ro) - localparam LAT_SLAB = LATENCY_FMA; // (mn - ro)*inv_d - localparam LAT_MINMAX = FNCP_LAT; // lo/hi per axis - localparam LAT_REDUCE = 2 * FNCP_LAT; // 4-input min/max tree - localparam LAT_CMP = FNCP_LAT; // t_near <= t_far - localparam LATENCY = LAT_ORIGIN + LAT_DEQUANT + LAT_SLAB + LAT_MINMAX + LAT_REDUCE + LAT_CMP; + localparam F = LATENCY_FMA; + localparam LAT_FMA = 3 * F; // origin - ro, + corner, * inv_d + localparam LATENCY = LAT_FMA + 4; // + per-axis, 2 reduce, verdict localparam [INST_FMT_BITS-1:0] FMT_ADD = 2'b00; // F32, a*b + c localparam [INST_FMT_BITS-1:0] FMT_SUB = 2'b10; // F32, a*b - c + localparam [31:0] F32_NEG0 = 32'h80000000; // x + -0 == x, signs kept + localparam [31:0] F32_PINF = 32'h7F800000; + localparam [31:0] F32_NINF = 32'hFF800000; - // ── combinational uint8 -> fp32 ─────────────────────────────────── - function automatic logic [31:0] u8_to_f32(input logic [7:0] n); + // ── helpers ─────────────────────────────────────────────────────── + // q * 2^e for an 8-bit integer q and an int8 e, as the F32 product rounds: + // exact, +inf past the range, 0 below it (subnormals flush). + function automatic logic [31:0] q_scale(input logic [7:0] q, input logic [7:0] e); logic [2:0] msb; - logic [6:0] shifted; - logic [22:0] man; - if (n == 8'd0) begin - u8_to_f32 = 32'd0; + logic [6:0] frac; + logic signed [9:0] be; + if (q == 8'd0) begin + q_scale = 32'd0; end else begin msb = 3'd0; for (integer b = 0; b < 8; ++b) begin - if (n[b]) begin + if (q[b]) begin msb = b[2:0]; end end - // normalize so the leading 1 sits at bit 7, then the 7 bits - // below it become the top of the fp32 mantissa. - shifted = 7'(n << (3'd7 - msb)); - man = {shifted, 16'd0}; - u8_to_f32 = {1'b0, (8'd127 + 8'(msb)), man}; + frac = 7'(q << (3'd7 - msb)); + be = 10'sd127 + 10'(msb) + 10'($signed(e)); + if (be >= 10'sd255) begin + q_scale = F32_PINF; + end else if (be <= 10'sd0) begin + q_scale = 32'd0; + end else begin + q_scale = {1'b0, be[7:0], frac, 16'd0}; + end end endfunction - // ── combinational 2^exp as fp32 (well-conditioned exponents) ────── - function automatic logic [31:0] pow2_f32(input logic [7:0] e); - logic [7:0] biased; - biased = 8'(9'sd127 + {e[7], e}); // sign-extend int8 exponent - pow2_f32 = {1'b0, biased, 23'd0}; + function automatic logic f32_is_nan(input logic [30:0] a); + f32_is_nan = (a[30:23] == 8'hff) && (a[22:0] != 23'd0); endfunction - // ── stage 0: prep per-axis float operands ───────────────────────── - wire [2:0][31:0] qmin_f, qmax_f, scale; - for (genvar a = 0; a < 3; ++a) begin : g_prep - assign qmin_f[a] = u8_to_f32(qmin[a]); - assign qmax_f[a] = u8_to_f32(qmax[a]); - assign scale[a] = pow2_f32(exp[a]); - end + // monotone integer key of a non-NaN F32 (+0 and -0 share one key) + function automatic logic [31:0] f32_key(input logic [31:0] a); + f32_key = (a[30:0] == 31'd0) ? 32'h80000000 : (a[31] ? ~a : {1'b1, a[30:0]}); + endfunction + + // IEEE a <= b; false on NaN + function automatic logic f32_le(input logic [31:0] a, input logic [31:0] b); + f32_le = !f32_is_nan(a[30:0]) && !f32_is_nan(b[30:0]) && (f32_key(a) <= f32_key(b)); + endfunction + + // fmin / fmax: a NaN operand yields the other one; two NaNs yield `none` + function automatic logic [31:0] f32_min(input logic [31:0] a, input logic [31:0] b, + input logic [31:0] none); + if (f32_is_nan(a[30:0])) begin + f32_min = f32_is_nan(b[30:0]) ? none : b; + end else if (f32_is_nan(b[30:0])) begin + f32_min = a; + end else begin + f32_min = (f32_key(b) < f32_key(a)) ? b : a; + end + endfunction + + function automatic logic [31:0] f32_max(input logic [31:0] a, input logic [31:0] b, + input logic [31:0] none); + if (f32_is_nan(a[30:0])) begin + f32_max = f32_is_nan(b[30:0]) ? none : b; + end else if (f32_is_nan(b[30:0])) begin + f32_max = a; + end else begin + f32_max = (f32_key(b) > f32_key(a)) ? b : a; + end + endfunction - // ── stage 1: origin - ro (per axis) ─────────────────────────────── - wire [2:0][31:0] oro; - for (genvar a = 0; a < 3; ++a) begin : g_origin + // ── stage 1: the box base relative to the ray origin ───────────── + wire [2:0][31:0] mn_a, mx_a, base_a, c_b; + for (genvar a = 0; a < 3; ++a) begin : g_base + assign mn_a[a] = raw ? raw_min[a] : q_scale(qmin[a], exp[a]); + assign mx_a[a] = raw ? raw_max[a] : q_scale(qmax[a], exp[a]); + assign base_a[a] = raw ? 32'd0 : origin[a]; VX_fma_unit #( - .USE_DSP (`VX_CFG_RTU_USE_DSP), // vendor xil_fma on Vivado (soft in sim), like the FPU - .LATENCY (LAT_ORIGIN), + .USE_DSP (`VX_CFG_RTU_USE_DSP), + .LATENCY (F), .SUBNORM_ENABLE (0), - .EXCEPT_ENABLE (0) - ) fma_oro ( + .EXCEPT_ENABLE (1) + ) fsub_c ( .clk (clk), .reset (reset), .enable (enable), - .mask (valid_in), - .op_type (INST_FPU_MADD), + .mask (1'b1), + .op_type (INST_FPU_ADD), .fmt (FMT_SUB), .frm (INST_FRM_RNE), - .dataa (origin[a]), - .datab (32'h3F800000 /*1.0*/), - .datac (ro[a]), - .result (oro[a]), + .dataa (base_a[a]), + .datab (ro[a]), + .datac ('0), + .result (c_b[a]), `UNUSED_PIN (fflags) ); end - // quantized corners delayed to align with origin-ro - wire [2:0][31:0] qmin_f_q, qmax_f_q, scale_q; + wire [2:0][31:0] mn_b, mx_b; VX_shift_register #( - .DATAW (3*32*3), - .DEPTH (LAT_ORIGIN) - ) sr_q ( + .DATAW (6*32), + .DEPTH (F) + ) sr_corner ( .clk (clk), .reset (reset), .enable (enable), - .data_in ({qmin_f, qmax_f, scale}), - .data_out ({qmin_f_q, qmax_f_q, scale_q}) + .data_in ({mn_a, mx_a}), + .data_out ({mn_b, mx_b}) ); - // raw-path operands delayed to align with the dequant-FMA inputs. - wire raw_d; - wire [2:0][31:0] raw_min_d, raw_max_d, ro_d; - VX_shift_register #( - .DATAW (1 + 3*32*3), - .DEPTH (LAT_ORIGIN) - ) sr_raw ( - .clk (clk), - .reset (reset), - .enable (enable), - .data_in ({raw, raw_min, raw_max, ro}), - .data_out ({raw_d, raw_min_d, raw_max_d, ro_d}) - ); - - // ── stage 2: corners relative to the ray origin (mn-ro, mx-ro). Quantized: - // q*scale + (origin-ro). Raw procedural box: (min*1.0 - ro) directly, - // reusing the same FMAs (FMT_SUB). ── - localparam [31:0] FP_ONE = 32'h3F800000; + // ── stage 2: corners relative to the ray origin, corner + c ────── wire [2:0][31:0] dmn, dmx; - for (genvar a = 0; a < 3; ++a) begin : g_dequant + for (genvar a = 0; a < 3; ++a) begin : g_rel VX_fma_unit #( - .USE_DSP (`VX_CFG_RTU_USE_DSP), // vendor xil_fma on Vivado (soft in sim), like the FPU - .LATENCY (LAT_DEQUANT), + .USE_DSP (`VX_CFG_RTU_USE_DSP), + .LATENCY (F), .SUBNORM_ENABLE (0), - .EXCEPT_ENABLE (0) - ) fma_mn ( + .EXCEPT_ENABLE (1) + ) fadd_dmn ( .clk (clk), .reset (reset), .enable (enable), .mask (1'b1), - .op_type (INST_FPU_MADD), - .fmt (raw_d ? FMT_SUB : FMT_ADD), + .op_type (INST_FPU_ADD), + .fmt (FMT_ADD), .frm (INST_FRM_RNE), - .dataa (raw_d ? raw_min_d[a] : qmin_f_q[a]), - .datab (raw_d ? FP_ONE : scale_q[a]), - .datac (raw_d ? ro_d[a] : oro[a]), + .dataa (mn_b[a]), + .datab (c_b[a]), + .datac ('0), .result (dmn[a]), `UNUSED_PIN (fflags) ); VX_fma_unit #( - .USE_DSP (`VX_CFG_RTU_USE_DSP), // vendor xil_fma on Vivado (soft in sim), like the FPU - .LATENCY (LAT_DEQUANT), + .USE_DSP (`VX_CFG_RTU_USE_DSP), + .LATENCY (F), .SUBNORM_ENABLE (0), - .EXCEPT_ENABLE (0) - ) fma_mx ( + .EXCEPT_ENABLE (1) + ) fadd_dmx ( .clk (clk), .reset (reset), .enable (enable), .mask (1'b1), - .op_type (INST_FPU_MADD), - .fmt (raw_d ? FMT_SUB : FMT_ADD), + .op_type (INST_FPU_ADD), + .fmt (FMT_ADD), .frm (INST_FRM_RNE), - .dataa (raw_d ? raw_max_d[a] : qmax_f_q[a]), - .datab (raw_d ? FP_ONE : scale_q[a]), - .datac (raw_d ? ro_d[a] : oro[a]), + .dataa (mx_b[a]), + .datab (c_b[a]), + .datac ('0), .result (dmx[a]), `UNUSED_PIN (fflags) ); end - // inv_d delayed to align with the origin-relative corners wire [2:0][31:0] inv_d_q; VX_shift_register #( .DATAW (3*32), - .DEPTH (LAT_ORIGIN + LAT_DEQUANT) + .DEPTH (2 * F) ) sr_invd ( .clk (clk), .reset (reset), @@ -232,13 +234,14 @@ module VX_rtu_box_pe import VX_gpu_pkg::*, VX_fpu_pkg::*, VX_rtu_pkg::*; #( ); // ── stage 3: slab entry/exit per axis = (corner - ro) * inv_d ───── + // a*b + -0 is the exact product, its zero sign included wire [2:0][31:0] t0, t1; for (genvar a = 0; a < 3; ++a) begin : g_slab VX_fma_unit #( - .USE_DSP (`VX_CFG_RTU_USE_DSP), // vendor xil_fma on Vivado (soft in sim), like the FPU - .LATENCY (LAT_SLAB), + .USE_DSP (`VX_CFG_RTU_USE_DSP), + .LATENCY (F), .SUBNORM_ENABLE (0), - .EXCEPT_ENABLE (0) + .EXCEPT_ENABLE (1) ) fma_t0 ( .clk (clk), .reset (reset), @@ -249,15 +252,15 @@ module VX_rtu_box_pe import VX_gpu_pkg::*, VX_fpu_pkg::*, VX_rtu_pkg::*; #( .frm (INST_FRM_RNE), .dataa (dmn[a]), .datab (inv_d_q[a]), - .datac (32'h0), + .datac (F32_NEG0), .result (t0[a]), `UNUSED_PIN (fflags) ); VX_fma_unit #( - .USE_DSP (`VX_CFG_RTU_USE_DSP), // vendor xil_fma on Vivado (soft in sim), like the FPU - .LATENCY (LAT_SLAB), + .USE_DSP (`VX_CFG_RTU_USE_DSP), + .LATENCY (F), .SUBNORM_ENABLE (0), - .EXCEPT_ENABLE (0) + .EXCEPT_ENABLE (1) ) fma_t1 ( .clk (clk), .reset (reset), @@ -268,59 +271,17 @@ module VX_rtu_box_pe import VX_gpu_pkg::*, VX_fpu_pkg::*, VX_rtu_pkg::*; #( .frm (INST_FRM_RNE), .dataa (dmx[a]), .datab (inv_d_q[a]), - .datac (32'h0), + .datac (F32_NEG0), .result (t1[a]), `UNUSED_PIN (fflags) ); end - // ── stage 4: per-axis lo/hi ─────────────────────────────────────── - // VX_fncp_unit returns an XLEN-wide result (it also serves the - // integer-returning compare and class ops); the traversal math is fp32, so - // every min/max result is taken from the low word. - wire [2:0][`VX_CFG_XLEN-1:0] lo_res, hi_res; - `UNUSED_VAR ({lo_res, hi_res}) - wire [2:0][31:0] lo, hi; - for (genvar a = 0; a < 3; ++a) begin : g_minmax - VX_fncp_unit #( - .LATENCY (FNCP_SIZE) - ) fncp_lo ( - .clk (clk), - .reset (reset), - .enable (enable), - .mask (1'b1), - .op_type (INST_FPU_MISC), - .fmt ('0), - .frm (3'd6 /*FMIN*/), - .dataa (t0[a]), - .datab (t1[a]), - .result (lo_res[a]), - `UNUSED_PIN (fflags) - ); - VX_fncp_unit #( - .LATENCY (FNCP_SIZE) - ) fncp_hi ( - .clk (clk), - .reset (reset), - .enable (enable), - .mask (1'b1), - .op_type (INST_FPU_MISC), - .fmt ('0), - .frm (3'd7 /*FMAX*/), - .dataa (t0[a]), - .datab (t1[a]), - .result (hi_res[a]), - `UNUSED_PIN (fflags) - ); - assign lo[a] = lo_res[a][31:0]; - assign hi[a] = hi_res[a][31:0]; - end - - // t_min/t_max delayed to align with lo/hi + // t_min/t_max delayed to the verdict stage wire [31:0] tmin_r, tmax_r; VX_shift_register #( .DATAW (64), - .DEPTH (LAT_ORIGIN + LAT_DEQUANT + LAT_SLAB + LAT_MINMAX) + .DEPTH (LAT_FMA + 1) ) sr_t ( .clk (clk), .reset (reset), @@ -329,140 +290,42 @@ module VX_rtu_box_pe import VX_gpu_pkg::*, VX_fpu_pkg::*, VX_rtu_pkg::*; #( .data_out ({tmin_r, tmax_r}) ); - // ── stage 5: reduce — t_near = max(tmin, lo[*]), t_far = min(tmax, hi[*]) ── - wire [`VX_CFG_XLEN-1:0] near_a_res, near_b_res, far_a_res, far_b_res; - `UNUSED_VAR ({near_a_res, near_b_res, far_a_res, far_b_res}) - wire [31:0] near_a = near_a_res[31:0]; // first reduce level - wire [31:0] near_b = near_b_res[31:0]; - wire [31:0] far_a = far_a_res[31:0]; - wire [31:0] far_b = far_b_res[31:0]; - VX_fncp_unit #( - .LATENCY (FNCP_SIZE) - ) r_near_a ( - .clk (clk), - .reset (reset), - .enable (enable), - .mask (1'b1), - .op_type (INST_FPU_MISC), - .fmt ('0), - .frm (3'd7), - .dataa (lo[0]), - .datab (lo[1]), - .result (near_a_res), - `UNUSED_PIN (fflags) - ); - VX_fncp_unit #( - .LATENCY (FNCP_SIZE) - ) r_near_b ( - .clk (clk), - .reset (reset), - .enable (enable), - .mask (1'b1), - .op_type (INST_FPU_MISC), - .fmt ('0), - .frm (3'd7), - .dataa (lo[2]), - .datab (tmin_r), - .result (near_b_res), - `UNUSED_PIN (fflags) - ); - VX_fncp_unit #( - .LATENCY (FNCP_SIZE) - ) r_far_a ( - .clk (clk), - .reset (reset), - .enable (enable), - .mask (1'b1), - .op_type (INST_FPU_MISC), - .fmt ('0), - .frm (3'd6), - .dataa (hi[0]), - .datab (hi[1]), - .result (far_a_res), - `UNUSED_PIN (fflags) - ); - VX_fncp_unit #( - .LATENCY (FNCP_SIZE) - ) r_far_b ( - .clk (clk), - .reset (reset), - .enable (enable), - .mask (1'b1), - .op_type (INST_FPU_MISC), - .fmt ('0), - .frm (3'd6), - .dataa (hi[2]), - .datab (tmax_r), - .result (far_b_res), - `UNUSED_PIN (fflags) - ); - - wire [`VX_CFG_XLEN-1:0] t_near_res, t_far_res; - `UNUSED_VAR ({t_near_res, t_far_res}) - wire [31:0] t_near_w = t_near_res[31:0]; // second reduce level - wire [31:0] t_far_w = t_far_res[31:0]; - VX_fncp_unit #( - .LATENCY (FNCP_SIZE) - ) r_near ( - .clk (clk), - .reset (reset), - .enable (enable), - .mask (1'b1), - .op_type (INST_FPU_MISC), - .fmt ('0), - .frm (3'd7), - .dataa (near_a), - .datab (near_b), - .result (t_near_res), - `UNUSED_PIN (fflags) - ); - VX_fncp_unit #( - .LATENCY (FNCP_SIZE) - ) r_far ( - .clk (clk), - .reset (reset), - .enable (enable), - .mask (1'b1), - .op_type (INST_FPU_MISC), - .fmt ('0), - .frm (3'd6), - .dataa (far_a), - .datab (far_b), - .result (t_far_res), - `UNUSED_PIN (fflags) - ); + // ── stage 4: per-axis lo/hi; an axis whose slabs are both NaN drops out ── + reg [2:0][31:0] lo_r, hi_r; + always_ff @(posedge clk) begin + if (enable) begin + for (integer a = 0; a < 3; ++a) begin + lo_r[a] <= f32_min(t0[a], t1[a], F32_NINF); + hi_r[a] <= f32_max(t0[a], t1[a], F32_PINF); + end + end + end - // ── stage 6: hit = (t_near <= t_far) ────────────────────────────── - wire [`VX_CFG_XLEN-1:0] cmp_res; - `UNUSED_VAR (cmp_res) - VX_fncp_unit #( - .LATENCY (FNCP_SIZE) - ) fncp_cmp ( - .clk (clk), - .reset (reset), - .enable (enable), - .mask (1'b1), - .op_type (INST_FPU_CMP), - .fmt ('0), - .frm (3'd0 /*LE*/), - .dataa (t_near_w), - .datab (t_far_w), - .result (cmp_res), - `UNUSED_PIN (fflags) - ); + // ── stage 5/6: lo = max(t_min, axes), hi = min(t_max, axes) ────── + reg [31:0] near_a_r, near_b_r, far_a_r, far_b_r; + reg [31:0] lo_all_r, hi_all_r; + always_ff @(posedge clk) begin + if (enable) begin + near_a_r <= f32_max(lo_r[0], lo_r[1], F32_NINF); + near_b_r <= f32_max(lo_r[2], tmin_r, F32_NINF); + far_a_r <= f32_min(hi_r[0], hi_r[1], F32_PINF); + far_b_r <= f32_min(hi_r[2], tmax_r, F32_PINF); + lo_all_r <= f32_max(near_a_r, near_b_r, F32_NINF); + hi_all_r <= f32_min(far_a_r, far_b_r, F32_PINF); + end + end - // carry t_near alongside the compare result, plus the overall valid pipe - wire [31:0] t_near_cmp; - VX_shift_register #( - .DATAW (32), - .DEPTH (LAT_CMP) - ) sr_tnear ( - .clk (clk), - .reset (reset), - .enable (enable), - .data_in (t_near_w), - .data_out (t_near_cmp) - ); + // ── stage 7: hit = lo <= hi; t_near = lo ───────────────────────── + // lo and hi are never NaN; t_near is non-negative for t_min >= 0 and +0 for + // a zero, so the consumer may order it as an unsigned integer. + reg hit_r; + reg [31:0] t_near_r; + always_ff @(posedge clk) begin + if (enable) begin + hit_r <= f32_le(lo_all_r, hi_all_r); + t_near_r <= (lo_all_r[30:0] == 31'd0) ? 32'd0 : lo_all_r; + end + end reg [LATENCY-1:0] valid_pipe_r; always_ff @(posedge clk) begin @@ -503,7 +366,7 @@ module VX_rtu_box_pe import VX_gpu_pkg::*, VX_fpu_pkg::*, VX_rtu_pkg::*; #( assign valid_out = valid_pipe_r[LATENCY-1]; assign tag_out = tag_out_w; assign tag_out_pre = tag_out_pre_w; - assign hit = cmp_res[0]; - assign t_near = t_near_cmp; + assign hit = hit_r; + assign t_near = t_near_r; endmodule diff --git a/hw/rtl/rtu/VX_rtu_core.sv b/hw/rtl/rtu/VX_rtu_core.sv index f7bd8af8ea..d2dedd1238 100644 --- a/hw/rtl/rtu/VX_rtu_core.sv +++ b/hw/rtl/rtu/VX_rtu_core.sv @@ -126,6 +126,15 @@ module VX_rtu_core import VX_gpu_pkg::*, VX_rtu_pkg::*; #( `STATIC_ASSERT((`VX_CFG_RTU_MERGE_DEPTH == 0), ("VX_CFG_RTU_MERGE_DEPTH > 0 is not implemented: this core does not merge node fetches")) + // A trace's warp-uniform config (scene, payload, flags|cull) arrives in + // lanes 1-3 of a WGATHER'd register, which must be written even when the + // warp's low lanes are masked. The ALU guarantees that only when it runs + // the whole warp in one packet: a narrower ALU skips all-masked packets and + // resolves its fallback source lane per packet, so the RTU would read a + // stale config. Fail the build instead of tracing the wrong scene. + `STATIC_ASSERT((`VX_CFG_NUM_ALU_LANES == `VX_CFG_NUM_THREADS), + ("the RTU needs a full-warp ALU (VX_CFG_NUM_ALU_LANES == VX_CFG_NUM_THREADS) for its WGATHER'd config")) + // ── ray staging: one entry per {src, wid} ───────────────────────── localparam NUM_STG = NUM_SRCS * NUM_WARPS; localparam STG_IDX_W = `LOG2UP(NUM_STG); @@ -143,7 +152,7 @@ module VX_rtu_core import VX_gpu_pkg::*, VX_rtu_pkg::*; #( T_CBWAIT = 3'd4, // candidate returned; await the CONTINUE's t T_CBATTR = 3'd5, // ... and its hitAttribute (CONT beat 1) T_RESUME = 3'd6, // release this slot's yield barrier - T_RWAIT = 3'd7; // await the resume commit -> terminal record + T_RWAIT = 3'd7; // await the resume commit -> next batch | terminal record reg [NUM_SLOTS-1:0][2:0] tstate; reg [NUM_SLOTS-1:0][NUM_LANES-1:0] req_mask; @@ -165,7 +174,7 @@ module VX_rtu_core import VX_gpu_pkg::*, VX_rtu_pkg::*; #( wire [RTU_RAY_BEATS*32-1:0] rw_data; wire [NUM_SLOTS-1:0] sch_busy, sch_done, sch_yield; - wire [NUM_CTX-1:0] sch_hit, sch_yld, sch_attrv; + wire [NUM_CTX-1:0] sch_hit, sch_yld, sch_attrv, sch_objv; wire [NUM_CTX-1:0][RTU_CB_TYPE_BITS-1:0] sch_cbtype; wire [NUM_SLOTS-1:0] sch_resume; wire [NUM_CTX-1:0][RTU_CB_ACTION_BITS-1:0] sch_action; @@ -209,6 +218,7 @@ module VX_rtu_core import VX_gpu_pkg::*, VX_rtu_pkg::*; #( .hit_bits (sch_hit), .yld_bits (sch_yld), .cb_types (sch_cbtype), + .obj_vld (sch_objv), .attr_vld (sch_attrv), .resume (sch_resume), .action (sch_action), @@ -502,10 +512,15 @@ module VX_rtu_core import VX_gpu_pkg::*, VX_rtu_pkg::*; #( wire wr_objray = is_cand[ws] && !wr_hitspan && (wr_idx < RTU_IDX_BITS'(13)); wire wr_cbtype = is_cand[ws] && (wr_idx == RTU_IDX_BITS'(13)); wire wr_sbt = is_cand[ws] && (wr_idx == RTU_IDX_BITS'(14)); + // Object ray: a candidate from inside a BLAS staged its transformed ray in + // the window store; any other lane's object ray is its world ray (staging). + wire [NUM_LANES-1:0] ws_objv = sch_objv[32'(ws)*NUM_LANES +: NUM_LANES] + & sch_yld[32'(ws)*NUM_LANES +: NUM_LANES]; + wire wr_objrow = wr_objray && (ws_objv != '0); // reads this word needs wire wb_need_w1 = (wb_state == WB_RD1) - && (wr_hitspan || wr_sbt || wr_attr) && !wr_status && !wr_payload; + && (wr_hitspan || wr_sbt || wr_attr || wr_objrow) && !wr_status && !wr_payload; wire wb_need_w2 = is_cand[ws] && wr_hitspan; // the hit row wire wb_need_st = (wr_hitspan && (wr_idx == '0)) || wr_objray; // t_max / object ray @@ -518,6 +533,8 @@ module VX_rtu_core import VX_gpu_pkg::*, VX_rtu_pkg::*; #( : RTU_WS_WORD_BITS'(RTU_WS_HIT_BASE)) + RTU_WS_WORD_BITS'(wr_idx)} : wr_sbt ? {ws, RTU_WS_WORD_BITS'(RTU_WS_YLD_SBT)} + : wr_objray ? {ws, RTU_WS_WORD_BITS'(RTU_WS_YLD_OBJ) + + RTU_WS_WORD_BITS'(wr_idx - RTU_IDX_BITS'(RTU_RES_HIT))} : {ws, RTU_WS_WORD_BITS'(RTU_WS_RES_ATTR)}; assign wb_stg_req = (wb_state == WB_RD1) && wb_need_st; @@ -567,7 +584,7 @@ module VX_rtu_core import VX_gpu_pkg::*, VX_rtu_pkg::*; #( win_word[i] = (wr_idx == '0) ? wb_ds[i] : 32'd0; end end else if (wr_objray) begin - win_word[i] = wb_ds[i]; + win_word[i] = ws_objv[i] ? wb_d1[i*32 +: 32] : wb_ds[i]; end else if (wr_cbtype) begin win_word[i] = {{(32-RTU_CB_TYPE_BITS){1'b0}}, sch_cbtype[wctx[i]]}; end else if (wr_sbt) begin @@ -692,7 +709,12 @@ module VX_rtu_core import VX_gpu_pkg::*, VX_rtu_pkg::*; #( tstate[s] <= T_RWAIT; end T_RWAIT: begin - if (sch_done[s]) begin + // a verdict that did not end a lane's ray re-walks it: + // the next candidate batch yields like the first + if (sch_yield[s]) begin + is_cand[s] <= 1'b1; + tstate[s] <= T_WRITE; + end else if (sch_done[s]) begin is_cand[s] <= 1'b0; tstate[s] <= T_WRITE; end diff --git a/hw/rtl/rtu/VX_rtu_fcross3.sv b/hw/rtl/rtu/VX_rtu_fcross3.sv deleted file mode 100644 index e68924eab0..0000000000 --- a/hw/rtl/rtu/VX_rtu_fcross3.sv +++ /dev/null @@ -1,127 +0,0 @@ -// Copyright © 2019-2023 -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// VX_rtu_fcross3 — fused fp32 3-vector cross product result = a × b. -// result[i] = a[(i+1)%3]*b[(i+2)%3] - a[(i+2)%3]*b[(i+1)%3] -// Each axis forms its two lane products and sums them (the second negated) in a -// shared VX_rtu_fmac3, normalizing+rounding ONCE rather than per FMA. Inputs in -// the RTU geometry path are finite; subnormals flushed to zero. Latency padded -// to 2*LATENCY_FMA so the consuming PE keeps its side-band alignment. - -`include "VX_define.vh" - -module VX_rtu_fcross3 import VX_gpu_pkg::*, VX_fpu_pkg::*; #( - parameter LATENCY_FMA = `VX_CFG_FMA_LATENCY, - parameter LATENCY = 2 * LATENCY_FMA -) ( - input wire clk, - input wire reset, - input wire enable, - input wire [2:0][31:0] a, - input wire [2:0][31:0] b, - output wire [2:0][31:0] result -); - // output pad must cover the pipelined multiply + fmac3 depth - `STATIC_ASSERT((LATENCY >= (`LATENCY_IMUL + 7)), ("VX_rtu_fcross3: LATENCY too small for the pipelined multiply")) - - for (genvar i = 0; i < 3; ++i) begin : g_axis - localparam I1 = (i + 1) % 3; - localparam I2 = (i + 2) % 3; - - // term0 = +a[I1]*b[I2], term1 = -a[I2]*b[I1] - wire [7:0] e0a = a[I1][30:23], e0b = b[I2][30:23]; - wire z0a = (e0a == 8'd0), z0b = (e0b == 8'd0); - wire [23:0] m0a = {1'b1, a[I1][22:0]}; - wire [23:0] m0b = {1'b1, b[I2][22:0]}; - - wire [7:0] e1a = a[I2][30:23], e1b = b[I1][30:23]; - wire z1a = (e1a == 8'd0), z1b = (e1b == 8'd0); - wire [23:0] m1a = {1'b1, a[I2][22:0]}; - wire [23:0] m1b = {1'b1, b[I1][22:0]}; - - wire [2:0] m_sign = {1'b0, - ~(a[I2][31] ^ b[I1][31]), // negated (subtraction) - (a[I1][31] ^ b[I2][31])}; - wire [2:0][8:0] m_pe = {9'd0, - (z1a | z1b) ? 9'd0 : ({1'b0, e1a} + {1'b0, e1b}), - (z0a | z0b) ? 9'd0 : ({1'b0, e0a} + {1'b0, e0b})}; - // 24x24 mantissa products pipelined into the DSP48 (LATENCY_IMUL deep) - // so the multiply is registered rather than combinational. The operands - // are the raw mantissas: a flushed term is discarded downstream by its - // pe=0, so nothing selects in front of the multiplier inputs and the - // DSP is driven straight from the source flops. - wire [47:0] pp0, pp1; - VX_multiplier #( - .A_WIDTH (24), - .B_WIDTH (24), - .SIGNED (0), - .LATENCY (`LATENCY_IMUL) - ) mul0 ( - .clk (clk), - .enable (enable), - .dataa (m0a), - .datab (m0b), - .result (pp0) - ); - VX_multiplier #( - .A_WIDTH (24), - .B_WIDTH (24), - .SIGNED (0), - .LATENCY (`LATENCY_IMUL) - ) mul1 ( - .clk (clk), - .enable (enable), - .dataa (m1a), - .datab (m1b), - .result (pp1) - ); - wire [2:0][47:0] q_prod = {48'd0, pp1, pp0}; - - // sign/exponent side-band delayed to align with the multiply latency - wire [2:0] q_sign; - wire [2:0][8:0] q_pe; - VX_pipe_register #( - .DATAW (3 + 3*9), - .DEPTH (`LATENCY_IMUL) - ) p0 ( - .clk (clk), - .reset (reset), - .enable (enable), - .data_in ({m_sign, m_pe}), - .data_out ({q_sign, q_pe}) - ); - - wire [31:0] crs; - VX_rtu_fmac3 mac ( - .clk (clk), - .reset (reset), - .enable (enable), - .sign (q_sign), - .pe (q_pe), - .prod (q_prod), - .result (crs) - ); - - VX_shift_register #( - .DATAW (32), - .DEPTH (LATENCY - (`LATENCY_IMUL + 7)) // 7 = VX_rtu_fmac3 depth - ) sr_pad ( - .clk (clk), - .reset (reset), - .enable (enable), - .data_in (crs), - .data_out (result[i]) - ); - end - -endmodule diff --git a/hw/rtl/rtu/VX_rtu_fdot3.sv b/hw/rtl/rtu/VX_rtu_fdot3.sv deleted file mode 100644 index 79400df589..0000000000 --- a/hw/rtl/rtu/VX_rtu_fdot3.sv +++ /dev/null @@ -1,101 +0,0 @@ -// Copyright © 2019-2023 -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// VX_rtu_fdot3 — fused fp32 3-vector dot product result = a·b. The three lane -// products feed VX_rtu_fmac3, which aligns and sums them, then normalizes and -// rounds ONCE — versus a chain of three FMAs that each normalize+round. Inputs -// in the RTU geometry path are finite; subnormals are flushed to zero. Latency -// padded to 3*LATENCY_FMA so the consuming PEs keep their side-band alignment. - -`include "VX_define.vh" - -module VX_rtu_fdot3 import VX_gpu_pkg::*, VX_fpu_pkg::*; #( - parameter LATENCY_FMA = `VX_CFG_FMA_LATENCY, - parameter LATENCY = 3 * LATENCY_FMA -) ( - input wire clk, - input wire reset, - input wire enable, - input wire [2:0][31:0] a, - input wire [2:0][31:0] b, - output wire [31:0] result -); - // output pad must cover the pipelined multiply + fmac3 depth - `STATIC_ASSERT((LATENCY >= (`LATENCY_IMUL + 7)), ("VX_rtu_fdot3: LATENCY too small for the pipelined multiply")) - - wire [2:0] m_sign; - wire [2:0][8:0] m_pe; - wire [2:0][47:0] m_prod; - for (genvar i = 0; i < 3; ++i) begin : g_mul - wire [7:0] ea = a[i][30:23], eb = b[i][30:23]; - wire az = (ea == 8'd0), bz = (eb == 8'd0); - wire [23:0] ma = {1'b1, a[i][22:0]}; - wire [23:0] mb = {1'b1, b[i][22:0]}; - assign m_sign[i] = a[i][31] ^ b[i][31]; - assign m_pe[i] = (az | bz) ? 9'd0 : ({1'b0, ea} + {1'b0, eb}); - // 24x24 mantissa product pipelined into the DSP48 (LATENCY_IMUL deep). - // The operands are the raw mantissas: a flushed term is discarded - // downstream by its pe=0, so nothing selects in front of the multiplier - // inputs and the DSP is driven straight from the source flops. - VX_multiplier #( - .A_WIDTH (24), - .B_WIDTH (24), - .SIGNED (0), - .LATENCY (`LATENCY_IMUL) - ) mul ( - .clk (clk), - .enable (enable), - .dataa (ma), - .datab (mb), - .result (m_prod[i]) - ); - end - - // sign/exponent side-band delayed to align with the multiply latency - wire [2:0] q_sign; - wire [2:0][8:0] q_pe; - wire [2:0][47:0] q_prod = m_prod; // products already registered by the DSPs - VX_pipe_register #( - .DATAW (3 + 3*9), - .DEPTH (`LATENCY_IMUL) - ) p0 ( - .clk (clk), - .reset (reset), - .enable (enable), - .data_in ({m_sign, m_pe}), - .data_out ({q_sign, q_pe}) - ); - - wire [31:0] dot; - VX_rtu_fmac3 mac ( - .clk (clk), - .reset (reset), - .enable (enable), - .sign (q_sign), - .pe (q_pe), - .prod (q_prod), - .result (dot) - ); - - VX_shift_register #( - .DATAW (32), - .DEPTH (LATENCY - (`LATENCY_IMUL + 7)) // 7 = VX_rtu_fmac3 depth - ) sr_pad ( - .clk (clk), - .reset (reset), - .enable (enable), - .data_in (dot), - .data_out (result) - ); - -endmodule diff --git a/hw/rtl/rtu/VX_rtu_fmac3.sv b/hw/rtl/rtu/VX_rtu_fmac3.sv deleted file mode 100644 index 155cd7c366..0000000000 --- a/hw/rtl/rtu/VX_rtu_fmac3.sv +++ /dev/null @@ -1,208 +0,0 @@ -// Copyright © 2019-2023 -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// VX_rtu_fmac3 — fused sum of up to three signed fp32 products. Each term is a -// pre-formed 48-bit mantissa product with a 9-bit product exponent (ea+eb) and -// a sign; an unused term passes pe=0/prod=0. The terms are aligned to the common -// (max) exponent, summed in extended precision, then normalized and rounded -// ONCE. Finite inputs only; subnormals flushed to zero. Deeply pipelined — the -// carry-heavy ops (negate, abs, the wide add and the LZC) each get their own -// cycle — to hold 300 MHz; latency = 7. Shared by VX_rtu_fdot3 (3 terms) and -// VX_rtu_fcross3 (2 terms per axis). - -`include "VX_define.vh" - -module VX_rtu_fmac3 #( - parameter PW = 48 -) ( - input wire clk, - input wire reset, - input wire enable, - input wire [2:0] sign, - input wire [2:0][8:0] pe, // product exponent ea+eb (0 => unused term) - input wire [2:0][PW-1:0] prod, // 48-bit mantissa product - output wire [31:0] result -); - localparam GW = 32; // guard bits below the product - localparam FW = PW + GW; // aligned field width - localparam SW = FW + 2; // signed-sum magnitude width - localparam LZW = `LOG2UP(SW); - - // ── stage 1: max exponent + per-term right-shift (no negate yet) ───── - wire [8:0] pe01 = (pe[0] > pe[1]) ? pe[0] : pe[1]; - wire [8:0] max_pe = (pe01 > pe[2]) ? pe01 : pe[2]; - - // An unused term is identified by pe=0 alone: a term whose operands are both - // normal has pe >= 2, so pe=0 is unambiguous. Discarding its product here - // means the callers do not have to force their mantissas to zero, which - // would put a select in front of the multiplier inputs. The mask sits on the - // product rather than on the shift amount, which controls a full-width - // barrel shifter and is far more sensitive to an extra level. - wire [FW-1:0] field [3]; - for (genvar i = 0; i < 3; ++i) begin : g_shift - wire [8:0] sh = max_pe - pe[i]; - wire [PW-1:0] prod_z = (pe[i] != 9'd0) ? prod[i] : '0; - assign field[i] = ({{(FW-PW){1'b0}}, prod_z} << GW) >> sh; - end - - wire [FW-1:0] s1_f0, s1_f1, s1_f2; - wire [2:0] s1_sign; - wire [8:0] s1_max_pe; - VX_pipe_register #( - .DATAW (3*FW + 3 + 9), - .DEPTH (1) - ) p1 ( - .clk (clk), - .reset (reset), - .enable (enable), - .data_in ({field[0], field[1], field[2], sign, max_pe}), - .data_out ({s1_f0, s1_f1, s1_f2, s1_sign, s1_max_pe}) - ); - - // ── stage 2: apply sign (two's-complement negate) ──────────────────── - wire [FW-1:0] s1_field [3]; - assign s1_field[0] = s1_f0; - assign s1_field[1] = s1_f1; - assign s1_field[2] = s1_f2; - wire signed [SW:0] term [3]; - for (genvar i = 0; i < 3; ++i) begin : g_neg - wire signed [SW:0] fext = $signed({{(SW+1-FW){1'b0}}, s1_field[i]}); - assign term[i] = s1_sign[i] ? -fext : fext; - end - - wire signed [SW:0] s2_t0, s2_t1, s2_t2; - wire [8:0] s2_max_pe; - VX_pipe_register #( - .DATAW (3*(SW+1) + 9), - .DEPTH (1) - ) p2 ( - .clk (clk), - .reset (reset), - .enable (enable), - .data_in ({term[0], term[1], term[2], s1_max_pe}), - .data_out ({s2_t0, s2_t1, s2_t2, s2_max_pe}) - ); - - // ── stage 3: signed sum ────────────────────────────────────────────── - wire signed [SW:0] sum = s2_t0 + s2_t1 + s2_t2; - - wire signed [SW:0] s3_sum; - wire [8:0] s3_max_pe; - VX_pipe_register #( - .DATAW (SW+1 + 9), - .DEPTH (1) - ) p3 ( - .clk (clk), - .reset (reset), - .enable (enable), - .data_in ({sum, s2_max_pe}), - .data_out ({s3_sum, s3_max_pe}) - ); - - // ── stage 4: sign + magnitude (abs) ────────────────────────────────── - wire neg = s3_sum[SW]; - wire [SW-1:0] absS = neg ? (~s3_sum[SW-1:0] + 1'b1) : s3_sum[SW-1:0]; - - wire [SW-1:0] s4_abs; - wire s4_neg; - wire [8:0] s4_max_pe; - VX_pipe_register #( - .DATAW (SW + 1 + 9), - .DEPTH (1) - ) p4 ( - .clk (clk), - .reset (reset), - .enable (enable), - .data_in ({absS, neg, s3_max_pe}), - .data_out ({s4_abs, s4_neg, s4_max_pe}) - ); - - // ── stage 5: leading-zero count ────────────────────────────────────── - wire [LZW-1:0] lz; - wire lz_valid; - VX_lzc #( - .N (SW) - ) lzc_i ( - .data_in (s4_abs), - .data_out (lz), - .valid_out (lz_valid) - ); - - wire [SW-1:0] s5_abs; - wire [LZW-1:0] s5_lz; - wire s5_lzv, s5_neg; - wire [8:0] s5_max_pe; - VX_pipe_register #( - .DATAW (SW + LZW + 1 + 1 + 9), - .DEPTH (1) - ) p5 ( - .clk (clk), - .reset (reset), - .enable (enable), - .data_in ({s4_abs, lz, lz_valid, s4_neg, s4_max_pe}), - .data_out ({s5_abs, s5_lz, s5_lzv, s5_neg, s5_max_pe}) - ); - - // ── stage 6: normalize, extract mantissa + GRS, base exponent ──────── - wire [SW-1:0] norm = s5_abs << s5_lz; // leading 1 at bit SW-1 - wire [23:0] mant = norm[SW-1 -: 24]; - wire g_bit = norm[SW-1-24]; - wire r_bit = norm[SW-1-25]; - wire stky = |norm[SW-1-26 : 0]; - wire zero = (s5_abs == '0); - wire signed [10:0] rexp_b = $signed({2'b0, s5_max_pe}) - 11'sd124 - - $signed({{(11-LZW){1'b0}}, s5_lz}); - - wire [23:0] s6_mant; - wire s6_g, s6_r, s6_s, s6_zero, s6_lzv, s6_neg; - wire signed [10:0] s6_rexp; - VX_pipe_register #( - .DATAW (24 + 6 + 11), - .DEPTH (1) - ) p6 ( - .clk (clk), - .reset (reset), - .enable (enable), - .data_in ({mant, g_bit, r_bit, stky, zero, s5_lzv, s5_neg, rexp_b}), - .data_out ({s6_mant, s6_g, s6_r, s6_s, s6_zero, s6_lzv, s6_neg, s6_rexp}) - ); - - // ── stage 7: round (RNE) + pack ────────────────────────────────────── - wire round_up = s6_g & (s6_r | s6_s | s6_mant[0]); - wire [24:0] mant_r = s6_mant + round_up; - wire carry = mant_r[24]; - wire [22:0] frac = carry ? mant_r[23:1] : mant_r[22:0]; - wire signed [10:0] rexp = s6_rexp + $signed({10'd0, carry}); - - reg [31:0] res; - always @(*) begin - if (s6_zero || !s6_lzv || rexp <= 11'sd0) - res = {s6_neg, 31'd0}; - else if (rexp >= 11'sd255) - res = {s6_neg, 8'hFF, 23'd0}; - else - res = {s6_neg, rexp[7:0], frac}; - end - - VX_pipe_register #( - .DATAW (32), - .DEPTH (1) - ) p7 ( - .clk (clk), - .reset (reset), - .enable (enable), - .data_in (res), - .data_out (result) - ); - -endmodule diff --git a/hw/rtl/rtu/VX_rtu_pkg.sv b/hw/rtl/rtu/VX_rtu_pkg.sv index d21d89be69..4a061f53b5 100644 --- a/hw/rtl/rtu/VX_rtu_pkg.sv +++ b/hw/rtl/rtu/VX_rtu_pkg.sv @@ -186,10 +186,9 @@ package VX_rtu_pkg; localparam RTU_FLAT_LINES_BITS = `CLOG2(RTU_FLAT_LINES + 1); // ───────────────────────────────────────────────────────────────── - // TLAS instance record (64 B). The 3x4 row-major affine transform - // (object→world) occupies floats 0..11; - // the walker applies its inverse (VX_rtu_xform) to bring the world ray into - // object space. The two TLAS variants share xform/blas/custom but differ in + // TLAS instance record (64 B). The 3x4 row-major world→object affine + // occupies floats 0..11; the walker applies it (VX_rtu_xform) to bring the + // world ray into object space. The two TLAS variants share xform/blas/custom but differ in // where instance_id and cull_mask sit: // flat TLAS : blas_off@48, custom_id@52, cull_mask@56; instance_id = loop idx // BVH inst : blas_root@48, custom_id@52, instance_id@56, cull_mask@60 @@ -299,6 +298,7 @@ package VX_rtu_pkg; localparam RTU_WS_CONT_T = 16; // CONTINUE beat 0: the shader's own t localparam RTU_WS_CONT_ATTR = 17; // CONTINUE beat 1: the shader's hitAttribute localparam RTU_WS_RES_ATTR = 18; // accepted candidate's bound hitAttribute + localparam RTU_WS_YLD_OBJ = 19; // instanced candidate's object ray: o.xyz, d.xyz (19..24) localparam RTU_WS_WORDS = 32; // rows per slot (power of two for addressing) localparam RTU_WS_WORD_BITS = `CLOG2(RTU_WS_WORDS); diff --git a/hw/rtl/rtu/VX_rtu_recip.sv b/hw/rtl/rtu/VX_rtu_recip.sv index d5c2094cc3..86dec04f5d 100644 --- a/hw/rtl/rtu/VX_rtu_recip.sv +++ b/hw/rtl/rtu/VX_rtu_recip.sv @@ -21,6 +21,10 @@ // map to DSP48. Trades ~2K LUT/unit onto the idle BRAM + DSP blocks. // ~9e-8 max relative error (well inside the RTU's 1e-4 tolerance). // +// A zero (or, flushed, subnormal) operand returns +FLT_MAX rather than inf +// (SimX rtu::ray_recip): a slab along a zero direction component then stays +// finite, (b - o) * FLT_MAX, instead of 0 * inf = NaN. +// // The input is presented combinationally and held stable for the whole setup // span by the scheduler; the result is a fixed-latency pipeline output, valid // after the backend's pipeline depth (<= the scheduler's SETUP_LAT wait). @@ -36,8 +40,10 @@ module VX_rtu_recip import VX_gpu_pkg::*, VX_fpu_pkg::*; #( input wire enable, input wire mask, input wire [31:0] x, // operand (dir component) - output wire [31:0] result // 1 / x + output wire [31:0] result // 1 / x (+FLT_MAX for x == 0) ); + localparam [31:0] F32_MAX = 32'h7F7FFFFF; + if (DSP_SEED != 0) begin : g_dsp_seed `UNUSED_VAR (mask) // ── seed ROM: 1/a for a = 1.fraction in [1,2), indexed by the top 10 @@ -68,16 +74,17 @@ module VX_rtu_recip import VX_gpu_pkg::*, VX_fpu_pkg::*; #( wire [22:0] s0_frac = x[22:0]; wire [23:0] s0_A = {1'b1, s0_frac}; // significand a*2^23 wire [30:0] s0_afx = {s0_A, 7'b0}; // a in Q2.30 - wire s0_inf = (s0_exp == 8'h00); // 1/0 -> inf - wire s0_zero = (s0_exp == 8'hFF); // 1/inf -> 0 + wire s0_inf = (s0_exp == 8'h00); // 1/0 -> FLT_MAX + wire s0_zero = (s0_exp == 8'hFF); // 1/inf -> 0, 1/NaN -> NaN + wire s0_nan = s0_zero && (s0_frac != 23'd0); wire [KIDX-1:0] s0_idx = s0_frac[22 -: KIDX]; - reg s1_sign, s1_inf, s1_zero; + reg s1_sign, s1_inf, s1_zero, s1_nan; reg [7:0] s1_exp; reg [30:0] s1_afx; reg [31:0] s1_y; // seed, Q1.31 always_ff @(posedge clk) if (enable) begin - s1_sign <= s0_sign; s1_inf <= s0_inf; s1_zero <= s0_zero; + s1_sign <= s0_sign; s1_inf <= s0_inf; s1_zero <= s0_zero; s1_nan <= s0_nan; s1_exp <= s0_exp; s1_afx <= s0_afx; s1_y <= seed_rom[s0_idx]; // registered ROM read -> BRAM end @@ -86,23 +93,23 @@ module VX_rtu_recip import VX_gpu_pkg::*, VX_fpu_pkg::*; #( wire [62:0] s1_ay = s1_afx * s1_y; // -> DSP (31b * 32b) wire [31:0] s1_p = 32'(s1_ay >> 31); // a*y (Q2.30) wire [31:0] s1_t = 32'h8000_0000 - s1_p; // 2 - p (2 == 2^31 in Q2.30) - reg s2_sign, s2_inf, s2_zero; + reg s2_sign, s2_inf, s2_zero, s2_nan; reg [7:0] s2_exp; reg [30:0] s2_afx; reg [31:0] s2_y0, s2_t; always_ff @(posedge clk) if (enable) begin - s2_sign <= s1_sign; s2_inf <= s1_inf; s2_zero <= s1_zero; + s2_sign <= s1_sign; s2_inf <= s1_inf; s2_zero <= s1_zero; s2_nan <= s1_nan; s2_exp <= s1_exp; s2_afx <= s1_afx; s2_y0 <= s1_y; s2_t <= s1_t; end wire [63:0] s2_yt = s2_y0 * s2_t; // -> DSP (32b * 32b) wire [31:0] s2_y1 = 32'(s2_yt >> 30); // y*(2-a*y) (Q1.31) - reg s3_sign, s3_inf, s3_zero; + reg s3_sign, s3_inf, s3_zero, s3_nan; reg [7:0] s3_exp; reg [30:0] s3_afx; reg [31:0] s3_y1; always_ff @(posedge clk) if (enable) begin - s3_sign <= s2_sign; s3_inf <= s2_inf; s3_zero <= s2_zero; + s3_sign <= s2_sign; s3_inf <= s2_inf; s3_zero <= s2_zero; s3_nan <= s2_nan; s3_exp <= s2_exp; s3_afx <= s2_afx; s3_y1 <= s2_y1; end @@ -110,20 +117,20 @@ module VX_rtu_recip import VX_gpu_pkg::*, VX_fpu_pkg::*; #( wire [62:0] s3_ay = s3_afx * s3_y1; // -> DSP (31b * 32b) wire [31:0] s3_p = 32'(s3_ay >> 31); wire [31:0] s3_t = 32'h8000_0000 - s3_p; - reg s4_sign, s4_inf, s4_zero; + reg s4_sign, s4_inf, s4_zero, s4_nan; reg [7:0] s4_exp; reg [31:0] s4_y1, s4_t; always_ff @(posedge clk) if (enable) begin - s4_sign <= s3_sign; s4_inf <= s3_inf; s4_zero <= s3_zero; + s4_sign <= s3_sign; s4_inf <= s3_inf; s4_zero <= s3_zero; s4_nan <= s3_nan; s4_exp <= s3_exp; s4_y1 <= s3_y1; s4_t <= s3_t; end wire [63:0] s4_yt = s4_y1 * s4_t; // -> DSP (32b * 32b) wire [31:0] s4_y2 = 32'(s4_yt >> 30); // 1/a in Q1.31 - reg s5_sign, s5_inf, s5_zero; + reg s5_sign, s5_inf, s5_zero, s5_nan; reg [7:0] s5_exp; reg [31:0] s5_y2; always_ff @(posedge clk) if (enable) begin - s5_sign <= s4_sign; s5_inf <= s4_inf; s5_zero <= s4_zero; + s5_sign <= s4_sign; s5_inf <= s4_inf; s5_zero <= s4_zero; s5_nan <= s4_nan; s5_exp <= s4_exp; s5_y2 <= s4_y2; end @@ -135,18 +142,20 @@ module VX_rtu_recip import VX_gpu_pkg::*, VX_fpu_pkg::*; #( wire s5_ovf = s5_fr[23]; // 2y rounded up to 2.0 wire [7:0] s5_expf = s5_ovf ? (s5_exf[7:0] + 8'd1) : s5_exf[7:0]; wire [22:0] s5_frac = s5_ovf ? 23'd0 : s5_fr[22:0]; - assign result = s5_inf ? {s5_sign, 8'hFF, 23'd0} + assign result = s5_inf ? F32_MAX + : s5_nan ? 32'h7FC00000 : s5_zero ? {s5_sign, 8'h00, 23'd0} : {s5_sign, s5_expf, s5_frac}; `UNUSED_PARAM (LATENCY) end else begin : g_lut_nr // portable baseline: 1.0 / x via the shared divide unit — vendor xil_fdiv // on Vivado (USE_DSP=VX_CFG_RTU_USE_DSP, LATENCY 28), soft NR in sim (17). + wire [31:0] quot; VX_fdiv_unit #( .USE_DSP (`VX_CFG_RTU_USE_DSP), .LATENCY (LATENCY), .SUBNORM_ENABLE (0), - .EXCEPT_ENABLE (0) + .EXCEPT_ENABLE (1) ) u_recip ( .clk (clk), .reset (reset), @@ -156,9 +165,21 @@ module VX_rtu_recip import VX_gpu_pkg::*, VX_fpu_pkg::*; #( .frm (INST_FRM_RNE), .dataa (32'h3F800000 /*1.0*/), .datab (x), - .result (result), + .result (quot), `UNUSED_PIN (fflags) ); + wire x_zero_q; + VX_shift_register #( + .DATAW (1), + .DEPTH (LATENCY) + ) sr_zero ( + .clk (clk), + .reset (reset), + .enable (enable), + .data_in (x[30:23] == 8'd0), + .data_out (x_zero_q) + ); + assign result = x_zero_q ? F32_MAX : quot; end endmodule diff --git a/hw/rtl/rtu/VX_rtu_scheduler.sv b/hw/rtl/rtu/VX_rtu_scheduler.sv index cec290c883..8477773ae5 100644 --- a/hw/rtl/rtu/VX_rtu_scheduler.sv +++ b/hw/rtl/rtu/VX_rtu_scheduler.sv @@ -85,6 +85,7 @@ module VX_rtu_scheduler import VX_gpu_pkg::*, VX_fpu_pkg::*, VX_rtu_pkg::*; #( output wire [NUM_CTX-1:0] hit_bits, output wire [NUM_CTX-1:0] yld_bits, output wire [NUM_CTX-1:0][RTU_CB_TYPE_BITS-1:0] cb_types, + output wire [NUM_CTX-1:0] obj_vld, // candidate staged an object ray output wire [NUM_CTX-1:0] attr_vld, // callback resume: the warp's per-lane actions, held stable by the core @@ -133,10 +134,23 @@ module VX_rtu_scheduler import VX_gpu_pkg::*, VX_fpu_pkg::*, VX_rtu_pkg::*; #( localparam RECIP_LAT = (`VX_CFG_RTU_RECIP_DSP_SEED != 0) ? 5 : RTU_FDIV_LAT; - localparam RTU_RESTART_CAP = 8; - localparam RST_CNTW = `CLOG2(RTU_RESTART_CAP + 1); localparam STK_IDXW = `CLOG2(RTU_STACK_DEPTH); + // Short-stack overflow restart. A walk visits the tree depth first, nearer + // child first, so every node has a rank path (its child rank at each level, + // an instance's index within its leaf) and the walk proceeds in ascending + // rank-path order. A child that does not fit on the stack is dropped; the + // walk then only descends (the stack stays full), so when it would next pop + // (a node past the drop) it restarts from the root instead, following the + // smallest dropped rank path: every node before it has been visited. That + // path is the per-level ranks the walk recorded on its way down (unchanged, + // as nothing was popped) plus the dropped child's own rank. A tightened + // best_t only culls a suffix of a node's ordered children, so the ranks stay + // valid across restarts, and each restart starts strictly further along. + localparam LVLW = 6; // tree levels a walk can record + localparam PATHW = 8; // rank at a level (instance index <= 255) + localparam STK_ENTW = 32 + LVLW + RTU_CHILD_BITS; + // box collections in flight; sized so the collector never caps the node // rate the pipelined front end can sustain over the box-PE latency. localparam COLL_SIZE = 16; @@ -184,6 +198,14 @@ module VX_rtu_scheduler import VX_gpu_pkg::*, VX_fpu_pkg::*, VX_rtu_pkg::*; #( logic [2:0][31:0] inv_d; logic [31:0] best_t; logic [31:0] yld_t; // staged candidate's t (compare copy) + logic [31:0] yld_ki; // staged candidate's key: instance id + logic [31:0] yld_ko; // ... and record offset + // resume floor: a re-walk offers only candidates above the last + // decided one in (t, key) order + logic has_floor; + logic [31:0] floor_t; + logic [31:0] floor_ki; + logic [31:0] floor_ko; logic [31:0] cur_off; logic [LB-1:0] f_idx; logic [LB-1:0] f_total; @@ -204,10 +226,18 @@ module VX_rtu_scheduler import VX_gpu_pkg::*, VX_fpu_pkg::*, VX_rtu_pkg::*; #( logic [31:0] inst_cust; logic [7:0] inst_flags; logic [31:0] root_off; - logic ovf_w; - logic ovf_o; - logic [RST_CNTW-1:0] rst_w; - logic [RST_CNTW-1:0] rst_o; + // overflow restart: the current node's level, the instance level of + // the current TLAS leaf, the smallest dropped child (level, rank), and + // the rank path a restarted walk follows down to it + logic [LVLW-1:0] lvl; + logic [LVLW-1:0] ilvl; + logic ovf; + logic [LVLW-1:0] nd_lvl; + logic [RTU_CHILD_BITS-1:0] nd_rank; + logic follow; + logic [LVLW-1:0] trl_lvl; + logic [RTU_CHILD_BITS-1:0] trl_rank; + logic [RTU_CHILD_BITS-1:0] dsc; // the child CS_PUSH descends into logic [2:0][31:0] obj_o; logic [2:0][31:0] obj_d; logic [2:0][31:0] obj_inv_d; @@ -228,13 +258,19 @@ module VX_rtu_scheduler import VX_gpu_pkg::*, VX_fpu_pkg::*, VX_rtu_pkg::*; #( reg [NUM_CTX-1:0] rdy_set; // wake bits (event-driven) wire [NUM_CTX-1:0] rdy_next; // next-cycle wake vector (see SELECT) reg [NUM_CTX-1:0] fresh_set; // first pass runs the launch init + reg [NUM_CTX-1:0] rewalk_set;// first pass runs the re-walk init + reg [NUM_CTX-1:0] live_q; // lane walks in the current round + reg [NUM_CTX-1:0] seed_v_q; // a resume committed seed_t_q + reg [NUM_CTX-1:0][31:0] seed_t_q; reg [NUM_CTX-1:0] done_q; reg [NUM_CTX-1:0] mask_q; reg [NUM_CTX-1:0] hit_q; reg [NUM_CTX-1:0] yld_q; reg [NUM_CTX-1:0][RTU_CB_TYPE_BITS-1:0] cbtype_q; + reg [NUM_CTX-1:0] objv_q; // candidate came from inside a BLAS reg [NUM_CTX-1:0] attr_q; reg [NUM_CTX-1:0][RTU_STACK_BITS-1:0] sp_q_arr; + reg [NUM_CTX-1:0][LVLW-1:0] lvl_q_arr; reg [NUM_CTX-1:0][LB-1:0] f_slot_q; reg [NUM_CTX-1:0][RTU_CB_ACTION_BITS-1:0] act_q; @@ -274,16 +310,24 @@ module VX_rtu_scheduler import VX_gpu_pkg::*, VX_fpu_pkg::*, VX_rtu_pkg::*; #( reg s1_valid; reg [CTX_TAG_W-1:0] s1_sel; reg s1_fresh; + reg s1_rewalk; wire [SLOT_W-1:0] s1_slot = SLOT_W'(32'(s1_sel) / NUM_LANES); // ═══════════════════════ ALIGN snapshot ═══════════════════════════ reg x_valid; reg [CTX_TAG_W-1:0] sel_q; reg fresh_q; + reg rewalk_q; + // candidate key order, precomputed at ALIGN off the store row: the key of + // the record under test ({instance, record offset}) against the floor and + // the staged candidate, so EXEC only adds the t compares + reg key_gt_floor_q; + reg key_lt_yld_q; ctx_state_t word_q; lane_ray_t ray_q; reg [BUF_BITS-1:0] fbuf_q; - reg [31:0] stacktop_q; + reg [STK_ENTW-1:0] stacktop_q; + reg [PATHW-1:0] pathv_q; // rank path entry below the current node reg [ADDRW-1:0] structaddr_q; reg [RTU_STACK_BITS-1:0] sp_q; reg [15:0] flags_q; @@ -422,12 +466,16 @@ module VX_rtu_scheduler import VX_gpu_pkg::*, VX_fpu_pkg::*, VX_rtu_pkg::*; #( wire [95:0] recip_rdata; // ── short stack (BVH only) ──────────────────────────────────────── - wire stk_wr; - wire [31:0] stk_wdata; - wire [31:0] stk_rdata; + wire stk_wr; + wire [STK_ENTW-1:0] stk_wdata; + wire [STK_ENTW-1:0] stk_rdata; + wire path_wr; + wire [LVLW-1:0] path_wlvl; + wire [PATHW-1:0] path_wdata; + wire [PATHW-1:0] path_rdata; if (!FLAT) begin : g_stack VX_dp_ram #( - .DATAW (32), + .DATAW (STK_ENTW), .SIZE (NUM_CTX << STK_IDXW), .OUT_REG (1), .RDW_MODE ("W") @@ -442,9 +490,27 @@ module VX_rtu_scheduler import VX_gpu_pkg::*, VX_fpu_pkg::*, VX_rtu_pkg::*; #( .raddr ({g1_idx, STK_IDXW'(sp_q_arr[g1_idx] - RTU_STACK_BITS'(1))}), .rdata (stk_rdata) ); + // the rank recorded at each level of the current path + VX_dp_ram #( + .DATAW (PATHW), + .SIZE (NUM_CTX << LVLW), + .OUT_REG (1), + .RDW_MODE ("W") + ) path_ram ( + .clk (clk), + .reset (reset), + .read (g1_valid), + .write (path_wr), + .wren (1'b1), + .waddr ({sel_q, path_wlvl}), + .wdata (path_wdata), + .raddr ({g1_idx, LVLW'(lvl_q_arr[g1_idx] + LVLW'(1))}), + .rdata (path_rdata) + ); end else begin : g_no_stack - assign stk_rdata = '0; - `UNUSED_VAR ({stk_wr, stk_wdata, sp_q_arr}) + assign stk_rdata = '0; + assign path_rdata = '0; + `UNUSED_VAR ({stk_wr, stk_wdata, sp_q_arr, path_wr, path_wlvl, path_wdata, lvl_q_arr}) end reg [COLL_SIZE-1:0] coll_busy; @@ -457,6 +523,8 @@ module VX_rtu_scheduler import VX_gpu_pkg::*, VX_fpu_pkg::*, VX_rtu_pkg::*; #( reg [COLL_SIZE-1:0][NODE_W-1:0][31:0] coll_ordt; reg [COLL_SIZE-1:0] coll_prochit; + wire [31:0] cand_ki_al = cs_word.in_blas ? cs_word.inst_id : 32'd0; + // ═══════════════════════ stage advance ════════════════════════════ always_ff @(posedge clk) begin if (reset) begin @@ -471,18 +539,23 @@ module VX_rtu_scheduler import VX_gpu_pkg::*, VX_fpu_pkg::*, VX_rtu_pkg::*; #( s1_valid <= g1_valid; s1_sel <= g1_idx; s1_fresh <= g1_valid && fresh_set[g1_idx]; + s1_rewalk <= g1_valid && rewalk_set[g1_idx]; x_valid <= s1_valid; if (s1_valid) begin sel_q <= s1_sel; fresh_q <= s1_fresh; + rewalk_q <= s1_rewalk; + key_gt_floor_q <= {cand_ki_al, cs_word.cur_off} > {cs_word.floor_ki, cs_word.floor_ko}; + key_lt_yld_q <= {cand_ki_al, cs_word.cur_off} < {cs_word.yld_ki, cs_word.yld_ko}; word_q <= cs_word; ray_q <= lane_ray_t'(ray_rdata); fbuf_q <= fbuf; stacktop_q <= stk_rdata; - // a fresh context's store row is stale: its walk starts at the - // scene base (the init template's cur_off is 0) + pathv_q <= path_rdata; + // a fresh context's store row is stale and a re-walk restarts: + // either walk starts at the scene base (the template's cur_off is 0) structaddr_q <= slot_scene[s1_slot] - + (s1_fresh ? ADDRW'(0) : ADDRW'(cs_word.cur_off)); + + ((s1_fresh || s1_rewalk) ? ADDRW'(0) : ADDRW'(cs_word.cur_off)); sp_q <= sp_q_arr[s1_sel]; flags_q <= slot_flags[s1_slot]; cull_q <= slot_cull[s1_slot]; @@ -598,7 +671,9 @@ module VX_rtu_scheduler import VX_gpu_pkg::*, VX_fpu_pkg::*, VX_rtu_pkg::*; #( && (inst_culldis || !(eff_back && cull_back)) && (inst_culldis || !(!eff_back && cull_front)) && !cls_cull; - wire tri_committable = tri_pass && (trit_q < word_q.best_t); + // the reported geometry word: the leaf's index plus the hit's facing + wire [31:0] tri_geom = (word_q.geom_r & `VX_RT_HIT_GEOMETRY_MASK) + | (eff_back ? `VX_RT_HIT_BACK_FACING : 32'd0); // BLAS traversal runs the object-space ray wire [2:0][31:0] walk_ro = word_q.in_blas ? word_q.obj_o : ray_q.origin; @@ -666,7 +741,7 @@ module VX_rtu_scheduler import VX_gpu_pkg::*, VX_fpu_pkg::*, VX_rtu_pkg::*; #( assign box_tag_pre = '0; `UNUSED_VAR ({box_feed, box_feed_raw, feed_ci, walk_inv_d}) `UNUSED_VAR ({leaf_v0, leaf_v1, leaf_geom, leaf_prim, leaf_flags, leaf_count}) - `UNUSED_VAR ({node, node_kind, node_lines, leaf_lines, stacktop_q}) + `UNUSED_VAR ({node, node_kind, node_lines, leaf_lines, stacktop_q, pathv_q}) end // Row select for the insertion read. It depends only on the tag, which the @@ -674,6 +749,7 @@ module VX_rtu_scheduler import VX_gpu_pkg::*, VX_fpu_pkg::*, VX_rtu_pkg::*; #( // here rather than repeated on the result cycle -- where it fans out across // every row and lands on the path that ends at the ordering registers. wire [COLL_IDW-1:0] box_coll_pre = box_tag_pre[COLL_IDW+32-1 : 32]; + `UNUSED_VAR (box_tag_pre[31:0]) reg [COLL_SIZE-1:0] box_coll_hot; always @(posedge clk) begin if (reset) begin @@ -736,6 +812,8 @@ module VX_rtu_scheduler import VX_gpu_pkg::*, VX_fpu_pkg::*, VX_rtu_pkg::*; #( .v1 (ltri_v1), .v2 (ltri_v2), .t_min (ray_q.t_min), + // the interval shrinks to the committed hit: only a strictly nearer + // hit is reported, so on equal t the first one found stays .t_max (word_q.best_t), .valid_out (tri_valid_out), .tag_out (tri_tag_out), @@ -823,13 +901,15 @@ module VX_rtu_scheduler import VX_gpu_pkg::*, VX_fpu_pkg::*, VX_rtu_pkg::*; #( // one field-row per cycle, per-lane write enable. localparam [1:0] CK_HIT = 2'd0, // committed opaque hit: the 7 hit rows CK_YLDA = 2'd1, // any-hit candidate: 7 yld rows + sbt - CK_YLDP = 2'd2; // IS candidate: t/u/v/prim/geom + sbt + CK_YLDP = 2'd2; // IS candidate: t/u/v/prim/inst/geom/cust + sbt (+ obj ray) typedef struct packed { logic [1:0] kind; logic [CTX_TAG_W-1:0] ctx; logic [RTU_CB_SBT_BITS-1:0] sbt; logic [31:0] t, u, v, prim, inst, geom, cust; + logic objv; // candidate inside a BLAS: stage obj + logic [5:0][31:0] obj; // its object ray: o.xyz, d.xyz } commit_t; wire cf_push, cf_pop, cf_empty, cf_full; @@ -883,7 +963,7 @@ module VX_rtu_scheduler import VX_gpu_pkg::*, VX_fpu_pkg::*, VX_rtu_pkg::*; #( assign win_rd_data = ws_rdata; // commit engine sequencing (one row per granted cycle) - reg [2:0] ce_step; + reg [3:0] ce_step; wire ce_active = ~cf_empty; wire [SLOT_W-1:0] ce_slot = SLOT_W'(32'(cf_dout.ctx) / NUM_LANES); wire [NUM_LANES-1:0] ce_lane = NUM_LANES'(1) << (32'(cf_dout.ctx) % NUM_LANES); @@ -892,8 +972,12 @@ module VX_rtu_scheduler import VX_gpu_pkg::*, VX_fpu_pkg::*, VX_rtu_pkg::*; #( reg [31:0] ce_data; reg ce_last; always @(*) begin - // per-kind (row, field) walk; CK_YLDP skips inst/custom — an IS - // candidate leaves those rows holding whatever was last staged + // per-kind (row, field) walk. Both candidate kinds stage the full + // attribute record (an IS shader reads gl_InstanceID and + // gl_InstanceCustomIndexEXT too) plus the SBT row; a committed hit + // stops after custom. A candidate from inside a BLAS then stages its + // object-space ray (gl_ObjectRay*): outside one it IS the world ray, + // which the core already holds, so those candidates skip the rows. logic [RTU_WS_WORD_BITS-1:0] base; base = (cf_dout.kind == CK_HIT) ? RTU_WS_WORD_BITS'(RTU_WS_HIT_BASE) : RTU_WS_WORD_BITS'(RTU_WS_YLD_BASE); @@ -901,37 +985,25 @@ module VX_rtu_scheduler import VX_gpu_pkg::*, VX_fpu_pkg::*, VX_rtu_pkg::*; #( ce_data = 32'd0; ce_last = 1'b0; case (ce_step) - 3'd0: ce_data = cf_dout.t; - 3'd1: ce_data = cf_dout.u; - 3'd2: ce_data = cf_dout.v; - 3'd3: ce_data = cf_dout.prim; - 3'd4: begin - if (cf_dout.kind == CK_YLDP) begin - ce_word = base + RTU_WS_WORD_BITS'(RTU_WS_F_GEOM); - ce_data = cf_dout.geom; - end else begin - ce_data = cf_dout.inst; - end - end - 3'd5: begin - if (cf_dout.kind == CK_YLDP) begin - ce_word = RTU_WS_WORD_BITS'(RTU_WS_YLD_SBT); - ce_data = 32'(cf_dout.sbt); - ce_last = 1'b1; - end else begin - ce_word = base + RTU_WS_WORD_BITS'(RTU_WS_F_GEOM); - ce_data = cf_dout.geom; - end - end - 3'd6: begin - ce_word = base + RTU_WS_WORD_BITS'(RTU_WS_F_CUST); + 4'd0: ce_data = cf_dout.t; + 4'd1: ce_data = cf_dout.u; + 4'd2: ce_data = cf_dout.v; + 4'd3: ce_data = cf_dout.prim; + 4'd4: ce_data = cf_dout.inst; + 4'd5: ce_data = cf_dout.geom; + 4'd6: begin ce_data = cf_dout.cust; ce_last = (cf_dout.kind == CK_HIT); end - default: begin + 4'd7: begin ce_word = RTU_WS_WORD_BITS'(RTU_WS_YLD_SBT); ce_data = 32'(cf_dout.sbt); - ce_last = 1'b1; + ce_last = ~cf_dout.objv; + end + default: begin // 8..13: object ray + ce_word = RTU_WS_WORD_BITS'(RTU_WS_YLD_OBJ) + RTU_WS_WORD_BITS'(32'(ce_step) - 8); + ce_data = cf_dout.obj[3'(32'(ce_step) - 8)]; + ce_last = (ce_step == 4'd13); end endcase end @@ -939,8 +1011,11 @@ module VX_rtu_scheduler import VX_gpu_pkg::*, VX_fpu_pkg::*, VX_rtu_pkg::*; #( // ═══════════════════════ barrier walker ═══════════════════════════ // Runs the two whole-slot record operations one row at a time: // FIN — stage CHS (hit->yld row copy) and MISS (zeroed attributes) - // RES — commit accepted candidates (yld->hit row copy, attr merge) + // RES — commit accepted candidates (yld->hit row copy, attr merge), then + // re-walk every lane whose verdict did not end its ray localparam [3:0] BW_IDLE = 4'd0, + BW_RD3 = 4'd10, // RES: request the committed-t row + BW_CAP3 = 4'd11, // ... and drop out-of-range IS accepts BW_RD = 4'd1, // request the source row BW_CAP = 4'd2, // capture it BW_RD2 = 4'd3, // request the CONT-t row (RES field 0) @@ -997,11 +1072,11 @@ module VX_rtu_scheduler import VX_gpu_pkg::*, VX_fpu_pkg::*, VX_rtu_pkg::*; #( always @(*) begin for (integer j = 0; j < NUM_LANES; j = j + 1) begin fin_chs_mask[j] = fin_req && fin_chs_en - && mask_q[32'(fin_slot)*NUM_LANES + j] + && live_q[32'(fin_slot)*NUM_LANES + j] && !yld_q[32'(fin_slot)*NUM_LANES + j] && hit_q[32'(fin_slot)*NUM_LANES + j]; fin_miss_mask[j] = fin_req && fin_miss_en - && mask_q[32'(fin_slot)*NUM_LANES + j] + && live_q[32'(fin_slot)*NUM_LANES + j] && !yld_q[32'(fin_slot)*NUM_LANES + j] && !hit_q[32'(fin_slot)*NUM_LANES + j]; end @@ -1030,6 +1105,29 @@ module VX_rtu_scheduler import VX_gpu_pkg::*, VX_fpu_pkg::*, VX_rtu_pkg::*; #( end end + // An intersection shader reports its own t: it commits only if nearer than + // the committed hit (the captured row at BW_CAP3). The shader already + // checked it against the ray interval it knows of, which cannot include + // the opaque hits the walk committed on its own. + reg [NUM_LANES-1:0] bw_keep_mask; + // the lanes whose verdict did not end the ray: they re-walk above it + reg [NUM_LANES-1:0] res_rewalk; + wire bw_term_first = ((32'(slot_flags[bw_slot]) & 32'(`VX_RT_FLAG_TERMINATE_ON_FIRST_HIT)) != 0); + always @(*) begin + for (integer j = 0; j < NUM_LANES; j = j + 1) begin + logic [CTX_TAG_W-1:0] c; + logic is_proc, decides, ends; + c = CTX_TAG_W'(32'(bw_slot) * NUM_LANES + j); + is_proc = (cbtype_q[c] == RTU_CB_TYPE_BITS'(`VX_RT_CB_TYPE_PROC)); + decides = is_proc || (cbtype_q[c] == RTU_CB_TYPE_BITS'(`VX_RT_CB_TYPE_ANYHIT)); + bw_keep_mask[j] = bw_copy_mask[j] + && !(is_proc && hit_q[c] && !(bw_data2[j*32 +: 32] < ws_rdata[j*32 +: 32])); + ends = (act_q[c] == RTU_CB_ACTION_BITS'(`VX_RT_CB_TERMINATE)) + || (bw_copy_mask[j] && bw_term_first); + res_rewalk[j] = bw_is_res && yld_q[c] && decides && !ends; + end + end + // walker window-port requests reg bw_rd_req, bw_wr_req; reg [WS_ADDRW-1:0] bw_rd_addr, bw_wr_addr; @@ -1057,6 +1155,10 @@ module VX_rtu_scheduler import VX_gpu_pkg::*, VX_fpu_pkg::*, VX_rtu_pkg::*; #( bw_rd_req = 1'b1; bw_rd_addr = {bw_slot, RTU_WS_WORD_BITS'(RTU_WS_CONT_T)}; end + BW_RD3: begin + bw_rd_req = 1'b1; + bw_rd_addr = {bw_slot, RTU_WS_WORD_BITS'(RTU_WS_HIT_BASE)}; + end BW_WR: begin bw_wr_req = 1'b1; bw_wr_addr = {bw_slot, bw_dst_base + RTU_WS_WORD_BITS'(32'(bw_field))}; @@ -1120,13 +1222,16 @@ module VX_rtu_scheduler import VX_gpu_pkg::*, VX_fpu_pkg::*, VX_rtu_pkg::*; #( if (reset) begin ce_step <= '0; end else if (ce_wr_gnt) begin - ce_step <= ce_last ? 3'd0 : (ce_step + 3'd1); + ce_step <= ce_last ? 4'd0 : (ce_step + 4'd1); end end // ═══════════════════════ EXEC: the context FSM ════════════════════ // Effective word: a fresh (just-launched) context ignores the stale store - // row and starts from the init template. + // row and starts from the init template. A re-walk (resumed after a + // callback verdict that did not end the ray) restarts from the root with + // the committed hit as its t_max and the decided candidate as its floor; + // the world-ray reciprocals are unchanged, so it skips the setup. ctx_state_t word_x; always @(*) begin word_x = word_q; @@ -1135,9 +1240,33 @@ module VX_rtu_scheduler import VX_gpu_pkg::*, VX_fpu_pkg::*, VX_rtu_pkg::*; #( word_x.cstate = FLAT ? CS_HDR_REQ : CS_SETUP; word_x.best_t = ray_q.t_max; word_x.yld_t = ray_q.t_max; + end else if (rewalk_q) begin + word_x = '0; + word_x.cstate = CS_HDR_REQ; + word_x.inv_d = word_q.inv_d; + word_x.best_t = seed_v_q[sel_q] ? seed_t_q[sel_q] : word_q.best_t; + word_x.yld_t = ray_q.t_max; + word_x.has_floor = 1'b1; + word_x.floor_t = word_q.yld_t; + word_x.floor_ki = word_q.yld_ki; + word_x.floor_ko = word_q.yld_ko; end end + // Candidate order: ascending (t, key), key = {instance id, record offset}. + // A candidate is staged only above the floor and ahead of the staged one. + wire [31:0] cand_ki = word_q.in_blas ? word_q.inst_id : 32'd0; + function automatic logic above_floor(input logic [31:0] t); + above_floor = !word_q.has_floor + || (t > word_q.floor_t) + || ((t == word_q.floor_t) && key_gt_floor_q); + endfunction + function automatic logic before_yld(input logic [31:0] t); + before_yld = !yld_q[sel_q] + || (t < word_q.yld_t) + || ((t == word_q.yld_t) && key_lt_yld_q); + endfunction + // EXEC outcome (combinational) ctx_state_t word_n; reg wake_self; @@ -1146,6 +1275,7 @@ module VX_rtu_scheduler import VX_gpu_pkg::*, VX_fpu_pkg::*, VX_rtu_pkg::*; #( reg exec_yld_set; reg exec_yld_clr; reg [RTU_CB_TYPE_BITS-1:0] exec_cbtype; + reg exec_objv; reg mem_issue; reg [LB-1:0] mem_fslot; reg box_feed_r, box_raw_r, tri_feed_r, xform_feed_r; @@ -1155,13 +1285,27 @@ module VX_rtu_scheduler import VX_gpu_pkg::*, VX_fpu_pkg::*, VX_rtu_pkg::*; #( reg cf_push_r; commit_t cf_din_r; reg sp_inc, sp_dec; + reg sp_clr; reg stk_wr_r; - reg [31:0] stk_wdata_r; + reg [STK_ENTW-1:0] stk_wdata_r; + reg path_wr_r; + reg [LVLW-1:0] path_wlvl_r; + reg [PATHW-1:0] path_wdata_r; wire mem_fire = x_valid && mem_issue && mem_req_ready; wire [RTU_CHILD_BITS-1:0] last_child = node.n_children - RTU_CHILD_BITS'(1); + // overflow restart: the rank a restarted walk takes at the level below + // the current node (the dropped child's own at its level), the instance a + // TLAS leaf resumes at, and the popped stack entry's level and rank + wire [RTU_CHILD_BITS-1:0] follow_rank = + ((word_x.lvl + LVLW'(1)) == word_x.trl_lvl) ? word_x.trl_rank + : RTU_CHILD_BITS'(pathv_q); + wire [31:0] inst_start = word_x.follow ? 32'(pathv_q) : 32'd0; + wire [LVLW-1:0] stk_top_lvl = stacktop_q[STK_ENTW-1 -: LVLW]; + wire [RTU_CHILD_BITS-1:0] stk_top_rank = stacktop_q[32 +: RTU_CHILD_BITS]; + always @(*) begin word_n = word_x; wake_self = 1'b0; @@ -1170,6 +1314,7 @@ module VX_rtu_scheduler import VX_gpu_pkg::*, VX_fpu_pkg::*, VX_rtu_pkg::*; #( exec_yld_set = 1'b0; exec_yld_clr = 1'b0; exec_cbtype = '0; + exec_objv = 1'b0; mem_issue = 1'b0; mem_fslot = '0; box_feed_r = 1'b0; @@ -1184,8 +1329,12 @@ module VX_rtu_scheduler import VX_gpu_pkg::*, VX_fpu_pkg::*, VX_rtu_pkg::*; #( cf_din_r = '0; sp_inc = 1'b0; sp_dec = 1'b0; + sp_clr = 1'b0; stk_wr_r = 1'b0; stk_wdata_r = '0; + path_wr_r = 1'b0; + path_wlvl_r = '0; + path_wdata_r = '0; cf_din_r.ctx = sel_q; @@ -1336,11 +1485,14 @@ module VX_rtu_scheduler import VX_gpu_pkg::*, VX_fpu_pkg::*, VX_rtu_pkg::*; #( wake_self = 1'b1; end end else if (node_kind == RTU_KIND_LEAF_INST && leaf_count != 8'd0) begin + // a restart following its path resumes at the recorded instance word_n.inst_cnt = {24'd0, leaf_count}; - word_n.inst_idx = '0; + word_n.inst_idx = inst_start; word_n.inst_base = word_x.cur_off + 32'(RTU_LEAF_HDR_BYTES); word_n.blas_floor = sp_q; - word_n.cur_off = word_x.cur_off + 32'(RTU_LEAF_HDR_BYTES); + word_n.ilvl = word_x.lvl + LVLW'(1); + word_n.cur_off = word_x.cur_off + 32'(RTU_LEAF_HDR_BYTES) + + (inst_start * 32'(RTU_INST_STRIDE)); word_n.cstate = CS_INST_REQ; wake_self = 1'b1; end else begin @@ -1358,29 +1510,50 @@ module VX_rtu_scheduler import VX_gpu_pkg::*, VX_fpu_pkg::*, VX_rtu_pkg::*; #( end end CS_WAIT: begin - // woken by the collector: this node's ordering is complete - word_n.push_ptr = (coll_cnt_q == RTU_CHILD_BITS'(0)) - ? RTU_CHILD_BITS'(0) - : (coll_cnt_q - RTU_CHILD_BITS'(1)); - word_n.cstate = CS_PUSH; - wake_self = 1'b1; + // woken by the collector: this node's ordering is complete. The + // walk descends into the nearest child, or, restarting, into the + // child its path names; nearer children were visited before. + if (word_x.follow && (coll_cnt_q <= follow_rank)) begin + // that child (and every farther one) is culled by now + word_n.follow = 1'b0; + coll_free_r = 1'b1; + word_n.cstate = CS_POP; + end else begin + word_n.dsc = word_x.follow ? follow_rank : RTU_CHILD_BITS'(0); + word_n.push_ptr = (coll_cnt_q == RTU_CHILD_BITS'(0)) + ? RTU_CHILD_BITS'(0) + : (coll_cnt_q - RTU_CHILD_BITS'(1)); + word_n.cstate = CS_PUSH; + end + wake_self = 1'b1; end CS_PUSH: begin - if (word_x.push_ptr != RTU_CHILD_BITS'(0)) begin + if (word_x.push_ptr != word_x.dsc) begin if (sp_q != RTU_STACK_BITS'(RTU_STACK_DEPTH)) begin stk_wr_r = 1'b1; - stk_wdata_r = coll_ordoff[word_x.coll_id][word_x.push_ptr[IDXW-1:0]] - & RTU_CHILD_OFF_MASK; + stk_wdata_r = {word_x.lvl + LVLW'(1), word_x.push_ptr, + coll_ordoff[word_x.coll_id][word_x.push_ptr[IDXW-1:0]] + & RTU_CHILD_OFF_MASK}; sp_inc = 1'b1; - end else if (word_x.in_blas) begin - word_n.ovf_o = 1'b1; end else begin - word_n.ovf_w = 1'b1; + // dropped: pushes run farthest first, so the last one + // dropped is the nearest + word_n.ovf = 1'b1; + word_n.nd_lvl = word_x.lvl + LVLW'(1); + word_n.nd_rank = word_x.push_ptr; end word_n.push_ptr = word_x.push_ptr - RTU_CHILD_BITS'(1); wake_self = 1'b1; end else if (coll_cnt_q != RTU_CHILD_BITS'(0)) begin - word_n.cur_off = coll_ordoff[word_x.coll_id][0] & RTU_CHILD_OFF_MASK; + word_n.cur_off = coll_ordoff[word_x.coll_id][word_x.dsc[IDXW-1:0]] + & RTU_CHILD_OFF_MASK; + word_n.lvl = word_x.lvl + LVLW'(1); + path_wr_r = 1'b1; + path_wlvl_r = word_x.lvl + LVLW'(1); + path_wdata_r = PATHW'(word_x.dsc); + if (word_x.follow && ((word_x.lvl + LVLW'(1)) == word_x.trl_lvl)) begin + word_n.follow = 1'b0; // at the dropped child: walk on normally + end coll_free_r = 1'b1; word_n.cstate = CS_REQ0; wake_self = 1'b1; @@ -1394,17 +1567,25 @@ module VX_rtu_scheduler import VX_gpu_pkg::*, VX_fpu_pkg::*, VX_rtu_pkg::*; #( // woken by the collector: the raw AABB result landed if (coll_hit_q && (coll_t0_q < word_x.best_t) - && (!yld_q[sel_q] || (coll_t0_q < word_x.yld_t))) begin + && above_floor(coll_t0_q) + && before_yld(coll_t0_q)) begin cf_din_r.kind = CK_YLDP; cf_din_r.t = coll_t0_q; cf_din_r.prim = word_x.prim_base; - cf_din_r.geom = word_x.geom_r; + cf_din_r.inst = word_x.in_blas ? word_x.inst_id : 32'd0; + cf_din_r.cust = word_x.in_blas ? word_x.inst_cust : 32'd0; + cf_din_r.geom = word_x.geom_r & `VX_RT_HIT_GEOMETRY_MASK; cf_din_r.sbt = word_x.proc_sbt; + cf_din_r.objv = word_x.in_blas; + cf_din_r.obj = {word_x.obj_d, word_x.obj_o}; if (!cf_full) begin cf_push_r = 1'b1; exec_yld_set = 1'b1; exec_cbtype = RTU_CB_TYPE_BITS'(`VX_RT_CB_TYPE_PROC); + exec_objv = word_x.in_blas; word_n.yld_t = coll_t0_q; + word_n.yld_ki = cand_ki; + word_n.yld_ko = word_x.cur_off; coll_free_r = 1'b1; word_n.cstate = CS_POP; end @@ -1458,7 +1639,7 @@ module VX_rtu_scheduler import VX_gpu_pkg::*, VX_fpu_pkg::*, VX_rtu_pkg::*; #( CS_TRI_WAIT: begin // woken by the tri PE result (held in its result RAM, so a retry // on a full commit queue re-reads the same result) - if (tri_committable && tri_opaque) begin + if (tri_pass && tri_opaque) begin cf_din_r.kind = CK_HIT; cf_din_r.t = trit_q; cf_din_r.u = triu_q; @@ -1466,13 +1647,13 @@ module VX_rtu_scheduler import VX_gpu_pkg::*, VX_fpu_pkg::*, VX_rtu_pkg::*; #( cf_din_r.prim = word_x.prim_base + word_x.tri_i; cf_din_r.inst = word_x.in_blas ? word_x.inst_id : 32'd0; cf_din_r.cust = word_x.in_blas ? word_x.inst_cust : 32'd0; - cf_din_r.geom = word_x.geom_r; + cf_din_r.geom = tri_geom; if (cf_full) begin wake_self = 1'b1; end else begin cf_push_r = 1'b1; exec_hit_set = 1'b1; - word_n.best_t = trit_q; + word_n.best_t = trit_q; // a closer opaque hit occludes a farther candidate if (yld_q[sel_q] && (word_x.yld_t >= trit_q)) begin exec_yld_clr = 1'b1; @@ -1490,8 +1671,9 @@ module VX_rtu_scheduler import VX_gpu_pkg::*, VX_fpu_pkg::*, VX_rtu_pkg::*; #( wake_self = 1'b1; end end - end else if (tri_committable - && (!yld_q[sel_q] || (trit_q < word_x.yld_t))) begin + end else if (tri_pass + && above_floor(trit_q) + && before_yld(trit_q)) begin cf_din_r.kind = CK_YLDA; cf_din_r.t = trit_q; cf_din_r.u = triu_q; @@ -1499,15 +1681,20 @@ module VX_rtu_scheduler import VX_gpu_pkg::*, VX_fpu_pkg::*, VX_rtu_pkg::*; #( cf_din_r.prim = word_x.prim_base + word_x.tri_i; cf_din_r.inst = word_x.in_blas ? word_x.inst_id : 32'd0; cf_din_r.cust = word_x.in_blas ? word_x.inst_cust : 32'd0; - cf_din_r.geom = word_x.geom_r; + cf_din_r.geom = tri_geom; cf_din_r.sbt = cls_sbt; + cf_din_r.objv = word_x.in_blas; + cf_din_r.obj = {word_x.obj_d, word_x.obj_o}; if (cf_full) begin wake_self = 1'b1; end else begin cf_push_r = 1'b1; exec_yld_set = 1'b1; exec_cbtype = cls_cbtype; - word_n.yld_t = trit_q; + exec_objv = word_x.in_blas; + word_n.yld_t = trit_q; + word_n.yld_ki = cand_ki; + word_n.yld_ko = word_x.cur_off; if ((word_x.tri_i + 32'd1) < word_x.tri_n) begin word_n.tri_i = word_x.tri_i + 32'd1; word_n.cur_off = word_x.cur_off + 32'(RTU_TRI_STRIDE); @@ -1537,31 +1724,33 @@ module VX_rtu_scheduler import VX_gpu_pkg::*, VX_fpu_pkg::*, VX_rtu_pkg::*; #( word_n.cstate = CS_DONE; exec_done = 1'b1; end + end else if (word_x.ovf) begin + // every node before the dropped child has been visited: start + // over from the root along its rank path + word_n.ovf = 1'b0; + word_n.follow = 1'b1; + word_n.trl_lvl = word_x.nd_lvl; + word_n.trl_rank = word_x.nd_rank; + word_n.in_blas = 1'b0; + word_n.lvl = '0; + word_n.cur_off = word_x.root_off; + sp_clr = 1'b1; + word_n.cstate = CS_REQ0; + wake_self = 1'b1; end else if (word_x.in_blas && (sp_q == word_x.blas_floor)) begin - if (word_x.ovf_o && (word_x.rst_o != RST_CNTW'(RTU_RESTART_CAP))) begin - // an object-level subtree was dropped: re-descend the BLAS - // root pruning by the tightened best_t - word_n.ovf_o = 1'b0; - word_n.rst_o = word_x.rst_o + RST_CNTW'(1); - word_n.cur_off = word_x.blas_root; - word_n.cstate = CS_REQ0; - end else begin - word_n.cstate = CS_INST_NEXT; - end - wake_self = 1'b1; + word_n.follow = 1'b0; + word_n.cstate = CS_INST_NEXT; + wake_self = 1'b1; end else if (sp_q == '0) begin - if (word_x.ovf_w && (word_x.rst_w != RST_CNTW'(RTU_RESTART_CAP))) begin - word_n.ovf_w = 1'b0; - word_n.rst_w = word_x.rst_w + RST_CNTW'(1); - word_n.cur_off = word_x.root_off; - word_n.cstate = CS_REQ0; - wake_self = 1'b1; - end else begin - word_n.cstate = CS_DONE; - exec_done = 1'b1; - end + word_n.cstate = CS_DONE; + exec_done = 1'b1; end else begin - word_n.cur_off = stacktop_q; + word_n.follow = 1'b0; // popped off a restart's path + word_n.cur_off = stacktop_q[31:0]; + word_n.lvl = stk_top_lvl; + path_wr_r = 1'b1; + path_wlvl_r = stk_top_lvl; + path_wdata_r = PATHW'(stk_top_rank); sp_dec = 1'b1; word_n.cstate = CS_REQ0; wake_self = 1'b1; @@ -1641,18 +1830,21 @@ module VX_rtu_scheduler import VX_gpu_pkg::*, VX_fpu_pkg::*, VX_rtu_pkg::*; #( word_n.obj_inv_d = recip_q; word_n.in_blas = 1'b1; word_n.cur_off = word_x.blas_root; - word_n.rst_o = '0; - word_n.ovf_o = 1'b0; + // the BLAS root sits at the instance level, ranked by its index + word_n.lvl = word_x.ilvl; + path_wr_r = 1'b1; + path_wlvl_r = word_x.ilvl; + path_wdata_r = PATHW'(word_x.inst_idx); word_n.cstate = CS_REQ0; wake_self = 1'b1; end CS_INST_NEXT: begin + // every instance is scanned: a candidate staged in one instance + // does not hide a nearer one in a later instance. A restart's + // path ends at the instance it resumed. word_n.in_blas = 1'b0; - if ((FLAT_TLAS != 0) && yld_q[sel_q]) begin - // the flat instance loop stops on a staged candidate - word_n.cstate = CS_DONE; - exec_done = 1'b1; - end else if ((word_x.inst_idx + 32'd1) == word_x.inst_cnt) begin + word_n.follow = 1'b0; + if ((word_x.inst_idx + 32'd1) == word_x.inst_cnt) begin if (FLAT) begin word_n.cstate = CS_DONE; exec_done = 1'b1; @@ -1708,6 +1900,11 @@ module VX_rtu_scheduler import VX_gpu_pkg::*, VX_fpu_pkg::*, VX_rtu_pkg::*; #( assign cs_wdata = word_n; assign stk_wr = x_valid && stk_wr_r; assign stk_wdata = stk_wdata_r; + assign path_wr = x_valid && path_wr_r; + `RUNTIME_ASSERT(~(x_valid && path_wr_r && (path_wlvl_r == '0) && (word_x.cstate == CS_PUSH)), + ("%t: rtu walk deeper than %0d levels", $time, (1 << LVLW) - 1)) + assign path_wlvl = path_wlvl_r; + assign path_wdata = path_wdata_r; assign mem_req_valid = x_valid && mem_issue; assign mem_req_addr = structaddr_q + (ADDRW'(mem_fslot) << RTU_LINE_SEL_BITS); assign mem_req_tag = sel_q; @@ -1734,6 +1931,10 @@ module VX_rtu_scheduler import VX_gpu_pkg::*, VX_fpu_pkg::*, VX_rtu_pkg::*; #( end end + // the contexts a resume re-walks (bw_job_done's RES commit) + wire [NUM_CTX-1:0] rewalk_wake = (bw_job_done && bw_is_res) + ? (NUM_CTX'(res_rewalk) << (32'(bw_slot) * NUM_LANES)) : NUM_CTX'(0); + wire [NUM_CTX-1:0] rdy_wake_mask = (ray_wr_valid ? NUM_CTX'(1) << ray_wr_ctx : NUM_CTX'(0)) | (mem_rsp_valid ? NUM_CTX'(1) << mem_rsp_tag : NUM_CTX'(0)) @@ -1741,7 +1942,8 @@ module VX_rtu_scheduler import VX_gpu_pkg::*, VX_fpu_pkg::*, VX_rtu_pkg::*; #( | (xform_valid_out ? NUM_CTX'(1) << xform_tag_out : NUM_CTX'(0)) | ((recip_valid_out && recip_last_out) ? NUM_CTX'(1) << recip_tag_out : NUM_CTX'(0)) | (box_wake_r ? NUM_CTX'(1) << box_wake_ctx_r : NUM_CTX'(0)) - | (wake_self_r ? NUM_CTX'(1) << wake_self_ctx_r : NUM_CTX'(0)); + | (wake_self_r ? NUM_CTX'(1) << wake_self_ctx_r : NUM_CTX'(0)) + | rewalk_wake; assign rdy_next = (rdy_set & ~g1_onehot) | rdy_wake_mask; @@ -1750,10 +1952,14 @@ module VX_rtu_scheduler import VX_gpu_pkg::*, VX_fpu_pkg::*, VX_rtu_pkg::*; #( if (reset) begin rdy_set <= '0; fresh_set <= '0; + rewalk_set <= '0; + live_q <= '0; + seed_v_q <= '0; done_q <= '0; mask_q <= '0; hit_q <= '0; yld_q <= '0; + objv_q <= '0; attr_q <= '0; running <= '0; finalised <= '0; @@ -1778,6 +1984,8 @@ module VX_rtu_scheduler import VX_gpu_pkg::*, VX_fpu_pkg::*, VX_rtu_pkg::*; #( hit_q[32'(slot_start_slot)*NUM_LANES + k] <= 1'b0; yld_q[32'(slot_start_slot)*NUM_LANES + k] <= 1'b0; attr_q[32'(slot_start_slot)*NUM_LANES + k] <= 1'b0; + live_q[32'(slot_start_slot)*NUM_LANES + k] <= slot_start_mask[k]; + seed_v_q[32'(slot_start_slot)*NUM_LANES + k] <= 1'b0; end end @@ -1788,7 +1996,11 @@ module VX_rtu_scheduler import VX_gpu_pkg::*, VX_fpu_pkg::*, VX_rtu_pkg::*; #( // EXEC outcomes if (x_valid) begin - fresh_set[sel_q] <= 1'b0; + fresh_set[sel_q] <= 1'b0; + rewalk_set[sel_q] <= 1'b0; + if (rewalk_q) begin + seed_v_q[sel_q] <= 1'b0; + end if (exec_done) begin done_q[sel_q] <= 1'b1; end @@ -1798,11 +2010,13 @@ module VX_rtu_scheduler import VX_gpu_pkg::*, VX_fpu_pkg::*, VX_rtu_pkg::*; #( if (exec_yld_set) begin yld_q[sel_q] <= 1'b1; cbtype_q[sel_q] <= exec_cbtype; + objv_q[sel_q] <= exec_objv; end if (exec_yld_clr) begin yld_q[sel_q] <= 1'b0; end - if (fresh_q) begin + lvl_q_arr[sel_q] <= word_n.lvl; + if (fresh_q || rewalk_q || sp_clr) begin sp_q_arr[sel_q] <= '0; end else if (sp_inc) begin sp_q_arr[sel_q] <= sp_q + RTU_STACK_BITS'(1); @@ -1834,20 +2048,32 @@ module VX_rtu_scheduler import VX_gpu_pkg::*, VX_fpu_pkg::*, VX_rtu_pkg::*; #( if (bw_is_res) begin for (k = 0; k < NUM_LANES; k = k + 1) begin if (bw_copy_mask[k]) begin - hit_q[32'(bw_slot)*NUM_LANES + k] <= 1'b1; - attr_q[32'(bw_slot)*NUM_LANES + k] <= 1'b1; + hit_q[32'(bw_slot)*NUM_LANES + k] <= 1'b1; + attr_q[32'(bw_slot)*NUM_LANES + k] <= 1'b1; + seed_v_q[32'(bw_slot)*NUM_LANES + k] <= 1'b1; + end + yld_q[32'(bw_slot)*NUM_LANES + k] <= 1'b0; + live_q[32'(bw_slot)*NUM_LANES + k] <= res_rewalk[k]; + if (res_rewalk[k]) begin + done_q[32'(bw_slot)*NUM_LANES + k] <= 1'b0; + rewalk_set[32'(bw_slot)*NUM_LANES + k] <= 1'b1; end - yld_q[32'(bw_slot)*NUM_LANES + k] <= 1'b0; + end + // another round: its lanes finalise again when they finish + if (res_rewalk != '0) begin + finalised[bw_slot] <= 1'b0; end end else begin for (k = 0; k < NUM_LANES; k = k + 1) begin if (bw_copy_mask[k]) begin yld_q[32'(bw_slot)*NUM_LANES + k] <= 1'b1; cbtype_q[32'(bw_slot)*NUM_LANES + k] <= RTU_CB_TYPE_BITS'(`VX_RT_CB_TYPE_CHS); + objv_q[32'(bw_slot)*NUM_LANES + k] <= 1'b0; end if (bw_miss_mask[k]) begin yld_q[32'(bw_slot)*NUM_LANES + k] <= 1'b1; cbtype_q[32'(bw_slot)*NUM_LANES + k] <= RTU_CB_TYPE_BITS'(`VX_RT_CB_TYPE_MISS); + objv_q[32'(bw_slot)*NUM_LANES + k] <= 1'b0; end end finalised[bw_slot] <= 1'b1; @@ -1865,6 +2091,17 @@ module VX_rtu_scheduler import VX_gpu_pkg::*, VX_fpu_pkg::*, VX_rtu_pkg::*; #( end end + // a resume's committed t seeds the lane's re-walk + always_ff @(posedge clk) begin + if (bw_wr_gnt && (bw_state == BW_WR) && bw_is_res && (bw_field == 3'd0)) begin + for (integer j = 0; j < NUM_LANES; j = j + 1) begin + if (bw_copy_mask[j]) begin + seed_t_q[32'(bw_slot)*NUM_LANES + j] <= bw_wr_data[j*32 +: 32]; + end + end + end + end + // actions are captured at the resume pulse always_ff @(posedge clk) begin for (integer s2 = 0; s2 < NUM_SLOTS; s2 = s2 + 1) begin @@ -1960,7 +2197,21 @@ module VX_rtu_scheduler import VX_gpu_pkg::*, VX_fpu_pkg::*, VX_rtu_pkg::*; #( end BW_CAP2: begin bw_data2 <= ws_rdata; - bw_state <= BW_WR; + bw_state <= BW_RD3; + end + BW_RD3: begin + if (bw_rd_gnt) begin + bw_state <= BW_CAP3; + end + end + BW_CAP3: begin + bw_copy_mask <= bw_keep_mask; + if (bw_keep_mask == '0) begin + bw_job_done <= 1'b1; // every accept was out of range + bw_state <= BW_IDLE; + end else begin + bw_state <= BW_WR; + end end BW_WR: begin if (bw_wr_gnt) begin @@ -2016,12 +2267,15 @@ module VX_rtu_scheduler import VX_gpu_pkg::*, VX_fpu_pkg::*, VX_rtu_pkg::*; #( assign busy = running; assign done = done_r; for (genvar s = 0; s < NUM_SLOTS; ++s) begin : g_yield + // bw_job_done: the flags a resume clears (yld/done) update on this + // edge, so the old values must not read as a fresh yield assign yield[s] = running[s] && all_done[s] && finalised[s] && yld_any[s] - && ce_idle && bw_idle && !pend_resume[s]; + && ce_idle && bw_idle && !pend_resume[s] && !bw_job_done; end assign hit_bits = hit_q; assign yld_bits = yld_q; assign cb_types = cbtype_q; + assign obj_vld = objv_q; assign attr_vld = attr_q; `UNUSED_VAR ({f_aligned, s1_fresh, ray_q, ins_le, ins_here, ins_shift}) diff --git a/hw/rtl/rtu/VX_rtu_tri_pe.sv b/hw/rtl/rtu/VX_rtu_tri_pe.sv index 341e299437..2bacf98c10 100644 --- a/hw/rtl/rtu/VX_rtu_tri_pe.sv +++ b/hw/rtl/rtu/VX_rtu_tri_pe.sv @@ -11,23 +11,25 @@ // See the License for the specific language governing permissions and // limitations under the License. -// VX_rtu_tri_pe — pipelined Möller-Trumbore ray-triangle intersector. Streams -// one triangle per cycle and emits {hit, t, u, v, back_facing} after a fixed -// latency. +// VX_rtu_tri_pe — pipelined watertight ray-triangle intersector (Woop, Benthin, +// Wald, "Watertight Ray/Triangle Intersection", JCGT 2013), all F32. Streams one +// triangle per cycle and emits {hit, t, u, v, back_facing} after a fixed +// latency. Mirrors SimX rtu::ray_triangle op for op: // -// e1 = v1 - v0 e2 = v2 - v0 T = origin - v0 -// P = dir × e2 det = e1 · P invDet = 1/det -// u = (T · P) * invDet -// Q = T × e1 v = (dir · Q) * invDet -// t = (e2 · Q) * invDet -// hit = |det| >= EPS && 0<=u<=1 && 0<=v && u+v<=1 && tmin<=t<=tmax +// kz = argmax|dir|, kx/ky follow (swapped when dir[kz] < 0) +// sz = 1/dir[kz], sx = dir[kx]*sz, sy = dir[ky]*sz +// r = vertex - origin, px = fma(-sx, rz, rx), py = fma(-sy, rz, ry), pz = sz*rz +// w0 = px2*py1 - py2*px1, w1 = px0*py2 - py0*px2, w2 = px1*py0 - py1*px0 +// det = (w0 + w1) + w2, T = fma(w2, pz2, fma(w1, pz1, w0*pz0)) +// rcp = 1/det, t = T*rcp, (u, v) = (w1*rcp, w2*rcp) +// hit = !(any w < 0 && any w > 0) && det != 0 && t_min < t < t_max // back_facing = det < 0 // -// The FP datapath reuses VX_fma_unit (a*b±c), VX_fdiv_unit (1/det) and -// VX_fncp_unit (compares); the dot/cross products are VX_rtu_fdot3 / -// VX_rtu_fcross3. Side-band operands are delayed through shift registers so -// each stage consumes time-aligned inputs, keeping the whole pipe at a fixed -// latency the scheduler tracks via valid_out. +// Each edge function is two rounded products and a rounded difference of the +// sheared vertices alone, so an edge shared by two triangles evaluates to +// exactly negated weights in both: no ray slips between them. t_min < t < t_max +// is the Vulkan ray interval for triangles (open at both ends). The FP units +// flush subnormals. `include "VX_define.vh" @@ -58,520 +60,481 @@ module VX_rtu_tri_pe import VX_gpu_pkg::*, VX_fpu_pkg::*, VX_rtu_pkg::*; #( output wire [31:0] v, output wire back_facing ); - // VX_fncp_unit result latency is 1 (input pipe reg, OUT_REG=0); the LATENCY - // param only sizes the unused mask pipe, so size it to 2 to avoid a - // degenerate [-1:0] slice while the result still lands after one cycle. - localparam FNCP_LAT = 1; // result latency for alignment - localparam FNCP_SIZE = 2; // mask-pipe sizing param - localparam F = LATENCY_FMA; - localparam V = LATENCY_FDIV; - localparam LATENCY = 8*F + V + 2; - - localparam [INST_FMT_BITS-1:0] FMT_ADD = 2'b00; // F32, a*b + c - localparam [INST_FMT_BITS-1:0] FMT_SUB = 2'b10; // F32, a*b - c - - localparam [31:0] FP_ZERO = 32'h00000000; - localparam [31:0] FP_ONE = 32'h3F800000; - localparam [31:0] FP_EPS = 32'h358637BD; // 1e-6 - localparam [31:0] FP_NEG_EPS = 32'hB58637BD; // -1e-6 - - // ── stage e (@F): edge vectors and ray-origin offset ────────────── - wire [2:0][31:0] e1, e2, tvec; - for (genvar a = 0; a < 3; ++a) begin : g_edges + localparam F = LATENCY_FMA; + localparam V = LATENCY_FDIV; + + // stage start times (cycles after valid_in) + localparam T_B = 1; // axis select registered + localparam T_C = T_B + V; // sz ready + localparam T_D = T_C + F; // sx, sy ready + localparam T_E = T_D + F; // px, py, pz ready + localparam T_F = T_E + 2 * F; // w ready + localparam T_G = T_F + 2 * F + V; // 1/det ready (T at T_F + 3F) + localparam T_H = T_G + F; // t, u, v ready + localparam LATENCY = T_H + 1; // verdict registered + + `STATIC_ASSERT(V >= F, ("tri PE: FDIV latency must cover the r subtract and T")) + + localparam [INST_FMT_BITS-1:0] FMT_ADD = 2'b00; + localparam [INST_FMT_BITS-1:0] FMT_SUB = 2'b10; + localparam [31:0] F32_ONE = 32'h3F800000; + + // IEEE ordering on F32 (+0 == -0); NaN compares false + function automatic logic f32_lt(input logic [31:0] a, input logic [31:0] b); + logic [31:0] ka, kb; + ka = (a[30:0] == 31'd0) ? 32'h80000000 : (a[31] ? ~a : {1'b1, a[30:0]}); + kb = (b[30:0] == 31'd0) ? 32'h80000000 : (b[31] ? ~b : {1'b1, b[30:0]}); + f32_lt = !((a[30:23] == 8'hff) && (a[22:0] != 23'd0)) + && !((b[30:23] == 8'hff) && (b[22:0] != 23'd0)) + && (ka < kb); + endfunction + + // ── stage A (@0 -> @T_B): axis select ───────────────────────────── + wire [30:0] ad0 = dir[0][30:0]; + wire [30:0] ad1 = dir[1][30:0]; + wire [30:0] ad2 = dir[2][30:0]; + wire [1:0] kz_w = (ad0 >= ad1) ? ((ad0 >= ad2) ? 2'd0 : 2'd2) + : ((ad1 >= ad2) ? 2'd1 : 2'd2); + wire [1:0] kx0 = (kz_w == 2'd2) ? 2'd0 : (kz_w + 2'd1); + wire [1:0] ky0 = (kx0 == 2'd2) ? 2'd0 : (kx0 + 2'd1); + wire dz_neg = dir[kz_w][31] && (dir[kz_w][30:0] != 31'd0); + wire [1:0] kx_w = dz_neg ? ky0 : kx0; + wire [1:0] ky_w = dz_neg ? kx0 : ky0; + + // per vertex: (x, y, z) components in the sheared frame's axes + wire [2:0][2:0][31:0] q_w; // [vertex][axis x/y/z] + wire [2:0][2:0][31:0] cvs = {v2, v1, v0}; + for (genvar i = 0; i < 3; ++i) begin : g_q + assign q_w[i][0] = cvs[i][kx_w]; + assign q_w[i][1] = cvs[i][ky_w]; + assign q_w[i][2] = cvs[i][kz_w]; + end + wire [2:0][31:0] o_w = {origin[kz_w], origin[ky_w], origin[kx_w]}; + wire [2:0][31:0] d_w = {dir[kz_w], dir[ky_w], dir[kx_w]}; + + reg [2:0][2:0][31:0] q_a; + reg [2:0][31:0] o_a, d_a; + reg [31:0] tmin_a, tmax_a; + always_ff @(posedge clk) begin + if (enable) begin + q_a <= q_w; + o_a <= o_w; + d_a <= d_w; + tmin_a <= t_min; + tmax_a <= t_max; + end + end + + // ── stage B (@T_B): sz = 1/dir[kz]; r = vertex - origin ─────────── + wire [31:0] sz_c; + VX_fdiv_unit #( + .LATENCY (V), + .FLEN (32), + .USE_DSP (`VX_CFG_RTU_USE_DSP), + .SUBNORM_ENABLE (0), + .EXCEPT_ENABLE (1) + ) fdiv_sz ( + .clk (clk), + .reset (reset), + .enable (enable), + .mask (1'b1), + .fmt ('0), + .frm (INST_FRM_RNE), + .dataa (F32_ONE), + .datab (d_a[2]), + .result (sz_c), + `UNUSED_PIN (fflags) + ); + + wire [2:0][2:0][31:0] r_f; + for (genvar i = 0; i < 3; ++i) begin : g_r + for (genvar a = 0; a < 3; ++a) begin : g_ax + VX_fma_unit #( + .LATENCY (F), + .USE_DSP (`VX_CFG_RTU_USE_DSP), + .SUBNORM_ENABLE (0), + .EXCEPT_ENABLE (1) + ) fsub_r ( + .clk (clk), + .reset (reset), + .enable (enable), + .mask (1'b1), + .op_type (INST_FPU_ADD), + .fmt (FMT_SUB), + .frm (INST_FRM_RNE), + .dataa (q_a[i][a]), + .datab (o_a[a]), + .datac ('0), + .result (r_f[i][a]), + `UNUSED_PIN (fflags) + ); + end + end + + // r from @T_B+F to @T_D (consumed by the shear stage) + wire [2:0][2:0][31:0] r_d; + VX_shift_register #( + .DATAW (9 * 32), + .DEPTH (T_D - (T_B + F)) + ) sr_r ( + .clk (clk), + .reset (reset), + .enable (enable), + .data_in (r_f), + .data_out (r_d) + ); + + wire [1:0][31:0] dxy_c; + VX_shift_register #( + .DATAW (64), + .DEPTH (T_C - T_B) + ) sr_dxy ( + .clk (clk), + .reset (reset), + .enable (enable), + .data_in ({d_a[1], d_a[0]}), + .data_out (dxy_c) + ); + + // ── stage C (@T_C): sx = dir[kx]*sz, sy = dir[ky]*sz ─────────────── + wire [1:0][31:0] sxy_d; + for (genvar a = 0; a < 2; ++a) begin : g_sxy VX_fma_unit #( - .USE_DSP (`VX_CFG_RTU_USE_DSP), // vendor xil_fma on Vivado (soft in sim), like the FPU .LATENCY (F), + .USE_DSP (`VX_CFG_RTU_USE_DSP), .SUBNORM_ENABLE (0), - .EXCEPT_ENABLE (0) - ) fma_e1 ( + .EXCEPT_ENABLE (1) + ) fmul_s ( .clk (clk), .reset (reset), .enable (enable), .mask (1'b1), - .op_type (INST_FPU_MADD), - .fmt (FMT_SUB), + .op_type (INST_FPU_MUL), + .fmt (FMT_ADD), .frm (INST_FRM_RNE), - .dataa (v1[a]), - .datab (FP_ONE), - .datac (v0[a]), - .result (e1[a]), + .dataa (dxy_c[a]), + .datab (sz_c), + .datac ('0), + .result (sxy_d[a]), `UNUSED_PIN (fflags) ); + end + + wire [31:0] sz_d; + VX_shift_register #( + .DATAW (32), + .DEPTH (T_D - T_C) + ) sr_sz ( + .clk (clk), + .reset (reset), + .enable (enable), + .data_in (sz_c), + .data_out (sz_d) + ); + + // ── stage D (@T_D): px/py = fma(-s, rz, r), pz = sz*rz ──────────── + wire [2:0][1:0][31:0] p_e; // [vertex][x/y] + wire [2:0][31:0] pz_e; + for (genvar i = 0; i < 3; ++i) begin : g_shear + for (genvar a = 0; a < 2; ++a) begin : g_ax + VX_fma_unit #( + .LATENCY (F), + .USE_DSP (`VX_CFG_RTU_USE_DSP), + .SUBNORM_ENABLE (0), + .EXCEPT_ENABLE (1) + ) fma_p ( + .clk (clk), + .reset (reset), + .enable (enable), + .mask (1'b1), + .op_type (INST_FPU_MADD), + .fmt (FMT_ADD), + .frm (INST_FRM_RNE), + .dataa ({~sxy_d[a][31], sxy_d[a][30:0]}), + .datab (r_d[i][2]), + .datac (r_d[i][a]), + .result (p_e[i][a]), + `UNUSED_PIN (fflags) + ); + end VX_fma_unit #( - .USE_DSP (`VX_CFG_RTU_USE_DSP), // vendor xil_fma on Vivado (soft in sim), like the FPU .LATENCY (F), + .USE_DSP (`VX_CFG_RTU_USE_DSP), .SUBNORM_ENABLE (0), - .EXCEPT_ENABLE (0) - ) fma_e2 ( + .EXCEPT_ENABLE (1) + ) fmul_pz ( .clk (clk), .reset (reset), .enable (enable), .mask (1'b1), - .op_type (INST_FPU_MADD), - .fmt (FMT_SUB), + .op_type (INST_FPU_MUL), + .fmt (FMT_ADD), .frm (INST_FRM_RNE), - .dataa (v2[a]), - .datab (FP_ONE), - .datac (v0[a]), - .result (e2[a]), + .dataa (sz_d), + .datab (r_d[i][2]), + .datac ('0), + .result (pz_e[i]), `UNUSED_PIN (fflags) ); + end + + // ── stage E (@T_E): w_i = px_a*py_b - py_a*px_b ─────────────────── + // (a, b) per weight: w0 <- (2, 1), w1 <- (0, 2), w2 <- (1, 0) + wire [2:0][1:0][31:0] cp_e; // [weight][px_a*py_b, py_a*px_b] + wire [2:0][31:0] w_f; + for (genvar i = 0; i < 3; ++i) begin : g_w + localparam IA = (i == 0) ? 2 : ((i == 1) ? 0 : 1); + localparam IB = (i == 0) ? 1 : ((i == 1) ? 2 : 0); + for (genvar k = 0; k < 2; ++k) begin : g_prod + VX_fma_unit #( + .LATENCY (F), + .USE_DSP (`VX_CFG_RTU_USE_DSP), + .SUBNORM_ENABLE (0), + .EXCEPT_ENABLE (1) + ) fmul_c ( + .clk (clk), + .reset (reset), + .enable (enable), + .mask (1'b1), + .op_type (INST_FPU_MUL), + .fmt (FMT_ADD), + .frm (INST_FRM_RNE), + .dataa (p_e[IA][k]), + .datab (p_e[IB][1-k]), + .datac ('0), + .result (cp_e[i][k]), + `UNUSED_PIN (fflags) + ); + end VX_fma_unit #( - .USE_DSP (`VX_CFG_RTU_USE_DSP), // vendor xil_fma on Vivado (soft in sim), like the FPU .LATENCY (F), + .USE_DSP (`VX_CFG_RTU_USE_DSP), .SUBNORM_ENABLE (0), - .EXCEPT_ENABLE (0) - ) fma_t ( + .EXCEPT_ENABLE (1) + ) fsub_w ( .clk (clk), .reset (reset), .enable (enable), .mask (1'b1), - .op_type (INST_FPU_MADD), + .op_type (INST_FPU_ADD), .fmt (FMT_SUB), .frm (INST_FRM_RNE), - .dataa (origin[a]), - .datab (FP_ONE), - .datac (v0[a]), - .result (tvec[a]), + .dataa (cp_e[i][0]), + .datab (cp_e[i][1]), + .datac ('0), + .result (w_f[i]), `UNUSED_PIN (fflags) ); end - // dir aligned to the cross/dot consumers - wire [2:0][31:0] dir_f, dir_3f; - VX_shift_register #( - .DATAW (96), - .DEPTH (F) - ) sr_dir_f ( - .clk (clk), - .reset (reset), - .enable (enable), - .data_in (dir), - .data_out (dir_f) - ); + // pz from @T_E to @T_F + wire [2:0][31:0] pz_f; VX_shift_register #( - .DATAW (96), - .DEPTH (3*F) - ) sr_dir_3f ( + .DATAW (3 * 32), + .DEPTH (T_F - T_E) + ) sr_pz ( .clk (clk), .reset (reset), .enable (enable), - .data_in (dir), - .data_out (dir_3f) + .data_in (pz_e), + .data_out (pz_f) ); - // ── stage cross (@3F): P = dir × e2, Q = T × e1 ─────────────────── - wire [2:0][31:0] pvec, qvec; - VX_rtu_fcross3 #( - .LATENCY_FMA (F) - ) cross_p ( - .clk (clk), - .reset (reset), - .enable (enable), - .a (dir_f), - .b (e2), - .result (pvec) - ); - VX_rtu_fcross3 #( - .LATENCY_FMA (F) - ) cross_q ( - .clk (clk), - .reset (reset), - .enable (enable), - .a (tvec), - .b (e1), - .result (qvec) - ); - - // e1/e2/T aligned from @F to @3F to feed the dot products - wire [2:0][31:0] e1_3f, e2_3f, t_3f; - VX_shift_register #( - .DATAW (96), - .DEPTH (2*F) - ) sr_e1 ( - .clk (clk), - .reset (reset), - .enable (enable), - .data_in (e1), - .data_out (e1_3f) - ); + // ── stage F (@T_F): det = (w0 + w1) + w2; T = fma chain over w*pz ── + // w, pz delayed one and two FMA stages for the later chain links + wire [2:0][31:0] w_f1, pz_f1; + wire [31:0] w1_f2, w2_f2, pz2_f2; VX_shift_register #( - .DATAW (96), - .DEPTH (2*F) - ) sr_e2 ( + .DATAW (6 * 32), + .DEPTH (F) + ) sr_wpz1 ( .clk (clk), .reset (reset), .enable (enable), - .data_in (e2), - .data_out (e2_3f) + .data_in ({w_f, pz_f}), + .data_out ({w_f1, pz_f1}) ); VX_shift_register #( - .DATAW (96), - .DEPTH (2*F) - ) sr_t ( + .DATAW (3 * 32), + .DEPTH (F) + ) sr_wpz2 ( .clk (clk), .reset (reset), .enable (enable), - .data_in (tvec), - .data_out (t_3f) + .data_in ({w_f1[2], pz_f1[2], w_f1[1]}), + .data_out ({w2_f2, pz2_f2, w1_f2}) ); - // ── stage dot (@6F): det, and the un-scaled u/v/t numerators ────── - wire [31:0] det, u_num, v_num, t_num; - VX_rtu_fdot3 #( - .LATENCY_FMA (F) - ) dot_det ( - .clk (clk), - .reset (reset), - .enable (enable), - .a (e1_3f), - .b (pvec), - .result (det) + wire [31:0] det01, det_g, tp0, tp01, t_num; + VX_fma_unit #(.LATENCY (F), .USE_DSP (`VX_CFG_RTU_USE_DSP), .SUBNORM_ENABLE (0), .EXCEPT_ENABLE (1)) fadd_det01 ( + .clk (clk), .reset (reset), .enable (enable), .mask (1'b1), + .op_type (INST_FPU_ADD), .fmt (FMT_ADD), .frm (INST_FRM_RNE), + .dataa (w_f[0]), .datab (w_f[1]), .datac ('0), + .result (det01), `UNUSED_PIN (fflags) + ); + VX_fma_unit #(.LATENCY (F), .USE_DSP (`VX_CFG_RTU_USE_DSP), .SUBNORM_ENABLE (0), .EXCEPT_ENABLE (1)) fadd_det ( + .clk (clk), .reset (reset), .enable (enable), .mask (1'b1), + .op_type (INST_FPU_ADD), .fmt (FMT_ADD), .frm (INST_FRM_RNE), + .dataa (det01), .datab (w_f1[2]), .datac ('0), + .result (det_g), `UNUSED_PIN (fflags) ); - VX_rtu_fdot3 #( - .LATENCY_FMA (F) - ) dot_u ( - .clk (clk), - .reset (reset), - .enable (enable), - .a (t_3f), - .b (pvec), - .result (u_num) + VX_fma_unit #(.LATENCY (F), .USE_DSP (`VX_CFG_RTU_USE_DSP), .SUBNORM_ENABLE (0), .EXCEPT_ENABLE (1)) fmul_tp0 ( + .clk (clk), .reset (reset), .enable (enable), .mask (1'b1), + .op_type (INST_FPU_MUL), .fmt (FMT_ADD), .frm (INST_FRM_RNE), + .dataa (w_f[0]), .datab (pz_f[0]), .datac ('0), + .result (tp0), `UNUSED_PIN (fflags) ); - VX_rtu_fdot3 #( - .LATENCY_FMA (F) - ) dot_v ( - .clk (clk), - .reset (reset), - .enable (enable), - .a (dir_3f), - .b (qvec), - .result (v_num) + VX_fma_unit #(.LATENCY (F), .USE_DSP (`VX_CFG_RTU_USE_DSP), .SUBNORM_ENABLE (0), .EXCEPT_ENABLE (1)) fma_tp01 ( + .clk (clk), .reset (reset), .enable (enable), .mask (1'b1), + .op_type (INST_FPU_MADD), .fmt (FMT_ADD), .frm (INST_FRM_RNE), + .dataa (w_f1[1]), .datab (pz_f1[1]), .datac (tp0), + .result (tp01), `UNUSED_PIN (fflags) ); - VX_rtu_fdot3 #( - .LATENCY_FMA (F) - ) dot_t ( - .clk (clk), - .reset (reset), - .enable (enable), - .a (e2_3f), - .b (qvec), - .result (t_num) + VX_fma_unit #(.LATENCY (F), .USE_DSP (`VX_CFG_RTU_USE_DSP), .SUBNORM_ENABLE (0), .EXCEPT_ENABLE (1)) fma_t ( + .clk (clk), .reset (reset), .enable (enable), .mask (1'b1), + .op_type (INST_FPU_MADD), .fmt (FMT_ADD), .frm (INST_FRM_RNE), + .dataa (w2_f2), .datab (pz2_f2), .datac (tp01), + .result (t_num), `UNUSED_PIN (fflags) ); - // ── stage recip (@6F+V): invDet = 1/det ─────────────────────────── - wire [31:0] inv_det; + // ── 1/det (@T_F+2F -> @T_G) ─────────────────────────────────────── + wire [31:0] rcp_g; VX_fdiv_unit #( - .USE_DSP (`VX_CFG_RTU_USE_DSP), // vendor xil_fdiv on Vivado (soft in sim), like the FPU .LATENCY (V), + .FLEN (32), + .USE_DSP (`VX_CFG_RTU_USE_DSP), .SUBNORM_ENABLE (0), - .EXCEPT_ENABLE (0) - ) recip ( + .EXCEPT_ENABLE (1) + ) fdiv_rcp ( .clk (clk), .reset (reset), .enable (enable), .mask (1'b1), .fmt ('0), .frm (INST_FRM_RNE), - .dataa (FP_ONE), - .datab (det), - .result (inv_det), + .dataa (F32_ONE), + .datab (det_g), + .result (rcp_g), `UNUSED_PIN (fflags) ); - // numerators aligned from @6F to @6F+V - wire [31:0] u_num_d, v_num_d, t_num_d; + // T from @T_F+3F, w1/w2 from @T_F+2F, to @T_G + wire [31:0] t_num_g, w1_g, w2_g; VX_shift_register #( - .DATAW (96), + .DATAW (32), + .DEPTH (T_G - (T_F + 3 * F)) + ) sr_tnum ( + .clk (clk), + .reset (reset), + .enable (enable), + .data_in (t_num), + .data_out (t_num_g) + ); + VX_shift_register #( + .DATAW (2 * 32), .DEPTH (V) - ) sr_num ( + ) sr_wuv ( .clk (clk), .reset (reset), .enable (enable), - .data_in ({u_num, v_num, t_num}), - .data_out ({u_num_d, v_num_d, t_num_d}) + .data_in ({w2_f2, w1_f2}), + .data_out ({w2_g, w1_g}) ); + `UNUSED_VAR ({w_f1[0], pz_f1[0]}) - // ── stage scale (@7F+V): u/v/t = numerator * invDet ─────────────── - wire [31:0] u_w, v_w, t_w; - VX_fma_unit #( - .USE_DSP (`VX_CFG_RTU_USE_DSP), // vendor xil_fma on Vivado (soft in sim), like the FPU - .LATENCY (F), - .SUBNORM_ENABLE (0) - ) fma_u ( - .clk (clk), - .reset (reset), - .enable (enable), - .mask (1'b1), - .op_type (INST_FPU_MADD), - .fmt (FMT_ADD), - .frm (INST_FRM_RNE), - .dataa (u_num_d), - .datab (inv_det), - .datac (FP_ZERO), - .result (u_w), - `UNUSED_PIN (fflags) - ); - VX_fma_unit #( - .USE_DSP (`VX_CFG_RTU_USE_DSP), // vendor xil_fma on Vivado (soft in sim), like the FPU - .LATENCY (F), - .SUBNORM_ENABLE (0) - ) fma_v ( - .clk (clk), - .reset (reset), - .enable (enable), - .mask (1'b1), - .op_type (INST_FPU_MADD), - .fmt (FMT_ADD), - .frm (INST_FRM_RNE), - .dataa (v_num_d), - .datab (inv_det), - .datac (FP_ZERO), - .result (v_w), - `UNUSED_PIN (fflags) - ); - VX_fma_unit #( - .USE_DSP (`VX_CFG_RTU_USE_DSP), // vendor xil_fma on Vivado (soft in sim), like the FPU - .LATENCY (F), - .SUBNORM_ENABLE (0) - ) fma_t2 ( - .clk (clk), - .reset (reset), - .enable (enable), - .mask (1'b1), - .op_type (INST_FPU_MADD), - .fmt (FMT_ADD), - .frm (INST_FRM_RNE), - .dataa (t_num_d), - .datab (inv_det), - .datac (FP_ZERO), - .result (t_w), - `UNUSED_PIN (fflags) - ); + // ── stage G (@T_G): t = T*rcp, u = w1*rcp, v = w2*rcp ───────────── + wire [2:0][31:0] tuv_h; + wire [2:0][31:0] tuv_num = {w2_g, w1_g, t_num_g}; + for (genvar k = 0; k < 3; ++k) begin : g_scale + VX_fma_unit #( + .LATENCY (F), + .USE_DSP (`VX_CFG_RTU_USE_DSP), + .SUBNORM_ENABLE (0), + .EXCEPT_ENABLE (1) + ) fmul_tuv ( + .clk (clk), + .reset (reset), + .enable (enable), + .mask (1'b1), + .op_type (INST_FPU_MUL), + .fmt (FMT_ADD), + .frm (INST_FRM_RNE), + .dataa (tuv_num[k]), + .datab (rcp_g), + .datac ('0), + .result (tuv_h[k]), + `UNUSED_PIN (fflags) + ); + end - // ── stage sum (@8F+V): uv = u + v ───────────────────────────────── - wire [31:0] uv_w; - VX_fma_unit #( - .USE_DSP (`VX_CFG_RTU_USE_DSP), // vendor xil_fma on Vivado (soft in sim), like the FPU - .LATENCY (F), - .SUBNORM_ENABLE (0) - ) fma_uv ( - .clk (clk), - .reset (reset), - .enable (enable), - .mask (1'b1), - .op_type (INST_FPU_MADD), - .fmt (FMT_ADD), - .frm (INST_FRM_RNE), - .dataa (u_w), - .datab (FP_ONE), - .datac (v_w), - .result (uv_w), - `UNUSED_PIN (fflags) - ); + // ── verdict flags: edge signs (@T_F), det tests (@T_F+2F) ───────── + reg edge_ok_f; + always @(*) begin + logic any_neg, any_pos; + any_neg = 1'b0; + any_pos = 1'b0; + for (integer i = 0; i < 3; ++i) begin + if (w_f[i][30:0] != 31'd0 + && !((w_f[i][30:23] == 8'hff) && (w_f[i][22:0] != 23'd0))) begin + any_neg = any_neg | w_f[i][31]; + any_pos = any_pos | ~w_f[i][31]; + end + end + edge_ok_f = !(any_neg && any_pos); + end - // u/v/t aligned from @7F+V to @8F+V - wire [31:0] u_c, v_c, t_c; + wire edge_ok_d; VX_shift_register #( - .DATAW (96), - .DEPTH (F) - ) sr_uvt ( + .DATAW (1), + .DEPTH (2 * F) + ) sr_edge ( .clk (clk), .reset (reset), .enable (enable), - .data_in ({u_w, v_w, t_w}), - .data_out ({u_c, v_c, t_c}) + .data_in (edge_ok_f), + .data_out (edge_ok_d) ); - // det aligned from @6F to @8F+V - wire [31:0] det_c; + + wire det_nan = (det_g[30:23] == 8'hff) && (det_g[22:0] != 23'd0); + wire det_ok_d = (det_g[30:0] != 31'd0) && !det_nan; + wire back_d = det_g[31]; + + wire [2:0] flags_h; VX_shift_register #( - .DATAW (32), - .DEPTH (2*F + V) - ) sr_det ( + .DATAW (3), + .DEPTH (T_H - (T_F + 2 * F)) + ) sr_flags ( .clk (clk), .reset (reset), .enable (enable), - .data_in (det), - .data_out (det_c) + .data_in ({edge_ok_d, det_ok_d, back_d}), + .data_out (flags_h) ); - // t_min/t_max aligned from @0 to @8F+V - wire [31:0] tmin_c, tmax_c; + + wire [63:0] tmm_h; VX_shift_register #( .DATAW (64), - .DEPTH (8*F + V) + .DEPTH (T_H - T_B) ) sr_tmm ( .clk (clk), .reset (reset), .enable (enable), - .data_in ({t_min, t_max}), - .data_out ({tmin_c, tmax_c}) + .data_in ({tmin_a, tmax_a}), + .data_out (tmm_h) ); - // ── stage compare (@8F+V+1): bound and determinant tests ────────── - wire [`VX_CFG_XLEN-1:0] cu0, cu1, cv0, cuv, ct0, ct1, cdp, cdn, bfc; - `UNUSED_VAR ({cu0, cu1, cv0, cuv, ct0, ct1, cdp, cdn, bfc}) - VX_fncp_unit #( - .LATENCY (FNCP_SIZE) - ) cmp_u0 ( - .clk (clk), - .reset (reset), - .enable (enable), - .mask (1'b1), - .op_type (INST_FPU_CMP), - .fmt ('0), - .frm (3'd0 /*LE*/), - .dataa (FP_ZERO), - .datab (u_c), - .result (cu0), - `UNUSED_PIN (fflags) - ); - VX_fncp_unit #( - .LATENCY (FNCP_SIZE) - ) cmp_u1 ( - .clk (clk), - .reset (reset), - .enable (enable), - .mask (1'b1), - .op_type (INST_FPU_CMP), - .fmt ('0), - .frm (3'd0), - .dataa (u_c), - .datab (FP_ONE), - .result (cu1), - `UNUSED_PIN (fflags) - ); - VX_fncp_unit #( - .LATENCY (FNCP_SIZE) - ) cmp_v0 ( - .clk (clk), - .reset (reset), - .enable (enable), - .mask (1'b1), - .op_type (INST_FPU_CMP), - .fmt ('0), - .frm (3'd0), - .dataa (FP_ZERO), - .datab (v_c), - .result (cv0), - `UNUSED_PIN (fflags) - ); - VX_fncp_unit #( - .LATENCY (FNCP_SIZE) - ) cmp_uv ( - .clk (clk), - .reset (reset), - .enable (enable), - .mask (1'b1), - .op_type (INST_FPU_CMP), - .fmt ('0), - .frm (3'd0), - .dataa (uv_w), - .datab (FP_ONE), - .result (cuv), - `UNUSED_PIN (fflags) - ); - VX_fncp_unit #( - .LATENCY (FNCP_SIZE) - ) cmp_t0 ( - .clk (clk), - .reset (reset), - .enable (enable), - .mask (1'b1), - .op_type (INST_FPU_CMP), - .fmt ('0), - .frm (3'd0), - .dataa (tmin_c), - .datab (t_c), - .result (ct0), - `UNUSED_PIN (fflags) - ); - VX_fncp_unit #( - .LATENCY (FNCP_SIZE) - ) cmp_t1 ( - .clk (clk), - .reset (reset), - .enable (enable), - .mask (1'b1), - .op_type (INST_FPU_CMP), - .fmt ('0), - .frm (3'd0), - .dataa (t_c), - .datab (tmax_c), - .result (ct1), - `UNUSED_PIN (fflags) - ); - VX_fncp_unit #( - .LATENCY (FNCP_SIZE) - ) cmp_dp ( - .clk (clk), - .reset (reset), - .enable (enable), - .mask (1'b1), - .op_type (INST_FPU_CMP), - .fmt ('0), - .frm (3'd0), - .dataa (FP_EPS), - .datab (det_c), - .result (cdp), - `UNUSED_PIN (fflags) - ); - VX_fncp_unit #( - .LATENCY (FNCP_SIZE) - ) cmp_dn ( - .clk (clk), - .reset (reset), - .enable (enable), - .mask (1'b1), - .op_type (INST_FPU_CMP), - .fmt ('0), - .frm (3'd0), - .dataa (det_c), - .datab (FP_NEG_EPS), - .result (cdn), - `UNUSED_PIN (fflags) - ); - VX_fncp_unit #( - .LATENCY (FNCP_SIZE) - ) cmp_bf ( - .clk (clk), - .reset (reset), - .enable (enable), - .mask (1'b1), - .op_type (INST_FPU_CMP), - .fmt ('0), - .frm (3'd1 /*LT*/), - .dataa (det_c), - .datab (FP_ZERO), - .result (bfc), - `UNUSED_PIN (fflags) - ); - - wire pass_w = cu0[0] & cu1[0] & cv0[0] & cuv[0] - & ct0[0] & ct1[0] & (cdp[0] | cdn[0]); - - // u/v/t aligned from @8F+V to @8F+V+1 (one compare stage) - wire [31:0] u_a, v_a, t_a; - VX_shift_register #( - .DATAW (96), - .DEPTH (FNCP_LAT) - ) sr_uvt2 ( - .clk (clk), - .reset (reset), - .enable (enable), - .data_in ({u_c, v_c, t_c}), - .data_out ({u_a, v_a, t_a}) - ); + // ── stage H (@T_H): t_min < t < t_max ───────────────────────────── + wire range_ok = f32_lt(tmm_h[63:32], tuv_h[0]) && f32_lt(tuv_h[0], tmm_h[31:0]); - // ── stage commit (@8F+V+2): register the verdict and attributes ─── reg hit_r, bf_r; reg [31:0] u_r, v_r, t_r; always_ff @(posedge clk) begin if (enable) begin - hit_r <= pass_w; - bf_r <= bfc[0]; - u_r <= u_a; - v_r <= v_a; - t_r <= t_a; + hit_r <= flags_h[2] && flags_h[1] && range_ok; + bf_r <= flags_h[0]; + t_r <= tuv_h[0]; + u_r <= tuv_h[1]; + v_r <= tuv_h[2]; end end @@ -584,8 +547,6 @@ module VX_rtu_tri_pe import VX_gpu_pkg::*, VX_fpu_pkg::*, VX_rtu_pkg::*; #( end end - // carry the caller's tag alongside the datapath so streamed results can be - // routed back to their originating context. wire [TAG_WIDTH-1:0] tag_out_w; VX_shift_register #( .DATAW (TAG_WIDTH), diff --git a/hw/rtl/rtu/VX_rtu_xform.sv b/hw/rtl/rtu/VX_rtu_xform.sv index 5e175b03d9..fb9b0528bf 100644 --- a/hw/rtl/rtu/VX_rtu_xform.sv +++ b/hw/rtl/rtu/VX_rtu_xform.sv @@ -12,30 +12,16 @@ // limitations under the License. // VX_rtu_xform — world→object ray transform for a TLAS instance. Streams one -// instance's 3x4 affine transform + world ray and emits the object-space ray -// after a fixed latency. +// instance's world→object 3x4 matrix + world ray and emits the object-space ray +// after a fixed latency: three dependent FMAs per component, // -// obj_ro = R^T * (ro - t) obj_rd = R^T * rd +// obj_ro[i] = fma(ro.z, m[i][2], fma(ro.y, m[i][1], fma(ro.x, m[i][0], m[i][3]))) +// obj_rd[i] = fma(rd.z, m[i][2], fma(rd.y, m[i][1], rd.x * m[i][0])) // -// The instance transform is object→world; its inverse brings the world ray into -// object space. For the orthonormal rotation+translation transforms a TLAS -// carries (every instance in a valid scene), R is orthonormal so R^(-1) = R^T, -// which needs no determinant or division — a pure FMA pipeline. This is bit- -// equivalent to the SimX oracle's explicit cofactor inverse for any orthonormal -// R (the only kind the tests and a valid Vulkan TLAS produce); SimX's singular- -// matrix passthrough is moot here as there is no divide to guard. -// -// Layout of the 3x4 row-major transform (matches the shared host/SimX format): -// xform[0..2] = R row 0 xform[3] = t.x -// xform[4..6] = R row 1 xform[7] = t.y -// xform[8..10] = R row 2 xform[11] = t.z -// obj_ro[i] = (column i of R) . (ro - t); column i of R = row i of R^T: -// col0 = {xform[0], xform[4], xform[8]}, etc. -// -// The (ro - t) subtract reuses VX_fma_unit (a*1 - c); the matrix-vector products -// reuse VX_rtu_fdot3. Side-band operands are delayed through shift registers so -// every stage consumes time-aligned inputs at a fixed latency the scheduler -// tracks via valid_out — same structure as VX_rtu_tri_pe / VX_rtu_box_pe. +// The instance record carries the world→object matrix itself, so no inverse +// is taken anywhere; the direction is not renormalised, so t is the same in +// both spaces. Layout: m[i][j] = xform[4*i + j], row-major, translation in +// column 3. Mirrors SimX rtu::world_to_object_ray. `include "VX_define.vh" @@ -49,7 +35,7 @@ module VX_rtu_xform import VX_gpu_pkg::*, VX_fpu_pkg::*, VX_rtu_pkg::*; #( input wire valid_in, input wire [TAG_WIDTH-1:0] tag_in, // caller side-band (e.g. context id) - input wire [11:0][31:0] xform, // 3x4 row-major affine (object→world) + input wire [11:0][31:0] xform, // 3x4 row-major affine (world→object) input wire [2:0][31:0] ro, // world ray origin input wire [2:0][31:0] rd, // world ray direction @@ -59,95 +45,68 @@ module VX_rtu_xform import VX_gpu_pkg::*, VX_fpu_pkg::*, VX_rtu_pkg::*; #( output wire [2:0][31:0] obj_rd // object-space ray direction ); localparam F = LATENCY_FMA; - localparam LATENCY = 4 * F; // (ro-t) subtract @F, then dot @3F + localparam LATENCY = 3 * F; // three dependent FMAs - localparam [INST_FMT_BITS-1:0] FMT_SUB = 2'b10; // F32, a*b - c - localparam [31:0] FP_ONE = 32'h3F800000; + localparam [INST_FMT_BITS-1:0] FMT_ADD = 2'b00; - // translation vector t = {xform[3], xform[7], xform[11]}. - wire [2:0][31:0] tvec; - assign tvec[0] = xform[3]; - assign tvec[1] = xform[7]; - assign tvec[2] = xform[11]; - - // R columns (rows of R^T): col_i[j] = xform[4*j + i]. - wire [2:0][2:0][31:0] col; - for (genvar i = 0; i < 3; ++i) begin : g_col - for (genvar j = 0; j < 3; ++j) begin : g_col_e - assign col[i][j] = xform[4*j + i]; + // column j of the matrix and the ray's j component, held until step j + wire [2:0][2:0][31:0] col; // [j][row] + for (genvar j = 0; j < 3; ++j) begin : g_col + for (genvar i = 0; i < 3; ++i) begin : g_row + assign col[j][i] = xform[4*i + j]; end end - - // ── stage 1 (@F): d = ro - t (per axis), reusing the FMA as a*1 - c ── - wire [2:0][31:0] d; - for (genvar a = 0; a < 3; ++a) begin : g_sub - VX_fma_unit #( - .USE_DSP (`VX_CFG_RTU_USE_DSP), // vendor xil_fma on Vivado (soft in sim), like the FPU - .SUBNORM_ENABLE (0), - .LATENCY (F) - ) fma_d ( - .clk (clk), - .reset (reset), - .enable (enable), - .mask (1'b1), - .op_type (INST_FPU_MADD), - .fmt (FMT_SUB), - .frm (INST_FRM_RNE), - .dataa (ro[a]), - .datab (FP_ONE), - .datac (tvec[a]), - .result (d[a]), - `UNUSED_PIN (fflags) - ); - end - - // R columns aligned from @0 to @F to feed the dot products. - wire [2:0][2:0][31:0] col_d; + wire [2:0][31:0] col1_d, col2_d; + wire [1:0][31:0] ray1_d, ray2_d; // {rd, ro} component y, then z VX_shift_register #( - .DATAW (9*32), + .DATAW (3*32 + 2*32), .DEPTH (F) - ) sr_col ( + ) sr_step1 ( .clk (clk), .reset (reset), .enable (enable), - .data_in (col), - .data_out (col_d) + .data_in ({col[1], rd[1], ro[1]}), + .data_out ({col1_d, ray1_d}) ); - // rd aligned from @0 to @F so the direction dot starts in lock-step with d. - wire [2:0][31:0] rd_d; VX_shift_register #( - .DATAW (96), - .DEPTH (F) - ) sr_rd ( + .DATAW (3*32 + 2*32), + .DEPTH (2 * F) + ) sr_step2 ( .clk (clk), .reset (reset), .enable (enable), - .data_in (rd), - .data_out (rd_d) + .data_in ({col[2], rd[2], ro[2]}), + .data_out ({col2_d, ray2_d}) ); - // ── stage 2 (@F+3F = @4F): obj_ro[i] = col_i . d, obj_rd[i] = col_i . rd ── - for (genvar i = 0; i < 3; ++i) begin : g_dot - VX_rtu_fdot3 #( - .LATENCY_FMA (F) - ) dot_ro ( - .clk (clk), - .reset (reset), - .enable (enable), - .a (col_d[i]), - .b (d), - .result (obj_ro[i]) - ); - VX_rtu_fdot3 #( - .LATENCY_FMA (F) - ) dot_rd ( - .clk (clk), - .reset (reset), - .enable (enable), - .a (col_d[i]), - .b (rd_d), - .result (obj_rd[i]) - ); + // [row][0: origin, 1: direction] accumulators after each step + wire [2:0][1:0][31:0] acc0, acc1, acc2; + for (genvar i = 0; i < 3; ++i) begin : g_row + for (genvar k = 0; k < 2; ++k) begin : g_ray + // step 0: origin seeds with the translation, direction with -0 + // (so the product keeps its own zero sign) + VX_fma_unit #(.USE_DSP (`VX_CFG_RTU_USE_DSP), .LATENCY (F), .SUBNORM_ENABLE (0), .EXCEPT_ENABLE (1)) fma_s0 ( + .clk (clk), .reset (reset), .enable (enable), .mask (1'b1), + .op_type (INST_FPU_MADD), .fmt (FMT_ADD), .frm (INST_FRM_RNE), + .dataa ((k == 0) ? ro[0] : rd[0]), .datab (col[0][i]), + .datac ((k == 0) ? xform[4*i + 3] : 32'h80000000), + .result (acc0[i][k]), `UNUSED_PIN (fflags) + ); + VX_fma_unit #(.USE_DSP (`VX_CFG_RTU_USE_DSP), .LATENCY (F), .SUBNORM_ENABLE (0), .EXCEPT_ENABLE (1)) fma_s1 ( + .clk (clk), .reset (reset), .enable (enable), .mask (1'b1), + .op_type (INST_FPU_MADD), .fmt (FMT_ADD), .frm (INST_FRM_RNE), + .dataa (ray1_d[k]), .datab (col1_d[i]), .datac (acc0[i][k]), + .result (acc1[i][k]), `UNUSED_PIN (fflags) + ); + VX_fma_unit #(.USE_DSP (`VX_CFG_RTU_USE_DSP), .LATENCY (F), .SUBNORM_ENABLE (0), .EXCEPT_ENABLE (1)) fma_s2 ( + .clk (clk), .reset (reset), .enable (enable), .mask (1'b1), + .op_type (INST_FPU_MADD), .fmt (FMT_ADD), .frm (INST_FRM_RNE), + .dataa (ray2_d[k]), .datab (col2_d[i]), .datac (acc1[i][k]), + .result (acc2[i][k]), `UNUSED_PIN (fflags) + ); + end + assign obj_ro[i] = acc2[i][0]; + assign obj_rd[i] = acc2[i][1]; end // ── valid + tag pipe, sized to the whole datapath latency ───────── diff --git a/hw/rtl/tex/VX_tex_lerp.sv b/hw/rtl/tex/VX_tex_lerp.sv index 33c4c5ed67..906b26ec4f 100644 --- a/hw/rtl/tex/VX_tex_lerp.sv +++ b/hw/rtl/tex/VX_tex_lerp.sv @@ -17,12 +17,10 @@ module VX_tex_lerp #( parameter LATENCY = 3, - // Denominator the weight is normalized against. A bilinear tap weight is a - // fraction of 255, matching the packed-colour blend the software sampler - // uses for the same taps; a mip-level weight is a fraction of 256. The two - // round differently by up to one count, so a sample filtered on this unit - // only reproduces the software sampler if each weight keeps its own form. - parameter FRAC_SCALE = 255 + // The weight is frac/256 (8 subtexel fraction bits). A bilinear tap blend + // rounds to nearest; the mip-level blend truncates, as the software + // sampler's level blend does, so both paths keep producing the same texel. + parameter ROUND = 1 ) ( input wire clk, input wire reset, @@ -34,45 +32,28 @@ module VX_tex_lerp #( ); `UNUSED_VAR (reset) `STATIC_ASSERT(LATENCY == 3, ("invalid value")) - `STATIC_ASSERT(FRAC_SCALE == 255 || FRAC_SCALE == 256, ("invalid value")) + `STATIC_ASSERT(ROUND == 0 || ROUND == 1, ("invalid value")) - if (FRAC_SCALE == 256) begin : g_scale_256 - reg [15:0] p1, p2; - reg [15:0] sum; - reg [7:0] res; - // The blend is a fraction of 256, so the result is the high byte and the - // low one is the remainder this form truncates rather than rounds. - `UNUSED_VAR (sum[7:0]) + localparam [15:0] BIAS = ROUND ? 16'h80 : 16'h0; - wire [8:0] sub = (9'h100 - 9'(frac)); + reg [15:0] p1, p2; + reg [15:0] sum; + reg [7:0] res; + // The result is the high byte; the low one is the discarded remainder. + `UNUSED_VAR (sum[7:0]) - always @(posedge clk) begin - if (enable) begin - p1 <= 16'(in1 * sub); - p2 <= 16'(in2 * frac); - sum <= p1 + p2; - res <= sum[15:8]; - end - end - - assign out = res; - end else begin : g_scale_255 - reg [15:0] p1, p2; - reg [15:0] sum; - reg [7:0] res; - - wire [7:0] sub = (8'hff - frac); + wire [8:0] sub = (9'h100 - 9'(frac)); - always @(posedge clk) begin - if (enable) begin - p1 <= in1 * sub; - p2 <= in2 * frac; - sum <= p1 + p2 + 16'h80; - res <= 8'((sum + (sum >> 8)) >> 8); - end + // 255*256 + 128 < 2^16: the 16-bit accumulator cannot overflow. + always @(posedge clk) begin + if (enable) begin + p1 <= 16'(in1 * sub); + p2 <= 16'(in2 * frac); + sum <= p1 + p2 + BIAS; + res <= sum[15:8]; end - - assign out = res; end + assign out = res; + endmodule diff --git a/hw/rtl/tex/VX_tex_sampler.sv b/hw/rtl/tex/VX_tex_sampler.sv index 4808363136..9009033135 100644 --- a/hw/rtl/tex/VX_tex_sampler.sv +++ b/hw/rtl/tex/VX_tex_sampler.sv @@ -161,14 +161,14 @@ module VX_tex_sampler import VX_gpu_pkg::*, VX_tex_pkg::*; #( .data_out ({valid_s2, req_tag_s2, lodfrac_s2}) ); - // Blend the two levels. The weight is a fraction of 256 rather than of 255, - // which is the form the software sampler blends levels in; a sample that - // moves between the two paths then keeps the same value. + // Blend the two levels. The level weight truncates rather than rounds, + // the form the software sampler blends levels in; a sample that moves + // between the two paths then keeps the same value. for (genvar i = 0; i < NUM_LANES; ++i) begin : g_tex_lerp_LOD for (genvar j = 0; j < 4; ++j) begin : g_j VX_tex_lerp #( - .LATENCY (3), - .FRAC_SCALE (256) + .LATENCY (3), + .ROUND (0) ) tex_lerp_lod ( .clk (clk), .reset(reset), diff --git a/hw/syn/xilinx/aved/Makefile b/hw/syn/xilinx/aved/Makefile index c9b7c0cc96..f1471e46d8 100644 --- a/hw/syn/xilinx/aved/Makefile +++ b/hw/syn/xilinx/aved/Makefile @@ -128,7 +128,16 @@ HOST_TAG ?= HOST # is a full synthesis run to discover, and it has already cost one, so refuse # the tag at parse time rather than emitting the config. See platforms.mk. ifeq ($(HOST_TAG),HOST) -$(error HOST_TAG=HOST routes m_axi_host to the QDMA slave bridge, whose reads never complete on the compute shell; the CP would hang on its first ring fetch. Use HOST_TAG=HBM1 (the platforms.mk default)) +$(error HOST_TAG=HOST routes m_axi_host to the QDMA slave bridge, whose reads never complete on the compute shell; the CP would hang on its first ring fetch. Use HOST_TAG=HBM8 (the platforms.mk default)) +endif + +# MEM covers the 32-bit device space from the HBM base, i.e. HBM0..HBM7; a CP +# staging aperture there aliases device memory and large uploads overwrite the +# heap. See platforms.mk. +ifneq ($(filter HBM0 HBM1 HBM2 HBM3 HBM4 HBM5 HBM6 HBM7,$(HOST_TAG)),) +ifneq ($(filter MEM HBM0,$(MEM_TAG)),) +$(error HOST_TAG=$(HOST_TAG) lies inside the device-memory window of MEM_TAG=$(MEM_TAG) (HBM0..HBM7); CP staging would alias the Vortex heap. Use HBM8 or above) +endif endif # platforms.mk states the kernel clock in MHz; the linker config wants Hz. diff --git a/hw/syn/xilinx/aved/platforms.mk b/hw/syn/xilinx/aved/platforms.mk index 1a6e44b2ab..9a6b5b2a90 100644 --- a/hw/syn/xilinx/aved/platforms.mk +++ b/hw/syn/xilinx/aved/platforms.mk @@ -60,7 +60,14 @@ MEM_TAG = MEM # next, which cost a bitstream and several hours of hardware debugging. It # belongs here next to MEM_TAG, for the same reason and by the same mechanism: # stated before the Makefile's `HOST_TAG ?= HOST`, so plain `=` suffices. -HOST_TAG = HBM1 +# The aperture must lie OUTSIDE the device-memory window. Each HBM_AXI port +# addresses its own 512 MB slice of the stack, HBM at 0x40_0000_0000 + +# k*512 MB, and the MEM tag above exposes Vortex's whole 32-bit space from that +# same base -- so HBM0..HBM7 ARE device memory. With HBM1, VRT placed the CP's +# staging buffers at +512 MB, inside the heap: any scene whose buffers grew past +# 512 MB was overwritten by its own upload (wrong images; corrupted BVHs that +# traversal never finished). HBM8 starts at +4 GB, past the last device byte. +HOST_TAG = HBM8 # Kernel clock target (MHz). The linker also accepts a frequency request; # the runtime can retune within the platform's supported range. diff --git a/hw/unittest/Makefile b/hw/unittest/Makefile index bca7effb13..0387f2a267 100644 --- a/hw/unittest/Makefile +++ b/hw/unittest/Makefile @@ -27,6 +27,7 @@ all: $(MAKE) -C fsqrt_unit $(MAKE) -C fcvt_unit $(MAKE) -C fdivsqrt_unit + $(MAKE) -C tex_lerp run: $(MAKE) -C generic_queue run @@ -57,6 +58,7 @@ run: $(MAKE) -C fsqrt_unit run $(MAKE) -C fcvt_unit run $(MAKE) -C fdivsqrt_unit run + $(MAKE) -C tex_lerp run # FPU arithmetic units only — executed (not just built) so CI gates correctness. run-fpu: @@ -151,3 +153,4 @@ clean: $(MAKE) -C fsqrt_unit clean $(MAKE) -C fcvt_unit clean $(MAKE) -C fdivsqrt_unit clean + $(MAKE) -C tex_lerp clean diff --git a/hw/unittest/rtu_box_pe/Makefile b/hw/unittest/rtu_box_pe/Makefile new file mode 100644 index 0000000000..102b190d2c --- /dev/null +++ b/hw/unittest/rtu_box_pe/Makefile @@ -0,0 +1,31 @@ +ROOT_DIR := $(realpath ../../..) +include $(ROOT_DIR)/config.mk + +PROJECT := rtu_box_pe + +RTL_DIR := $(VORTEX_HOME)/hw/rtl +SRC_DIR := $(VORTEX_HOME)/hw/unittest/$(PROJECT) +SIMX_DIR := $(VORTEX_HOME)/sim/simx + +# The reference is the SimX model itself, so the RTU config must match both sides. +CONFIGS += -DSIMULATION -DVX_CFG_EXT_RTU_ENABLE + +CXXFLAGS := -I$(SRC_DIR) -I$(VORTEX_HOME)/hw/unittest/common -I$(SW_COMMON_DIR) +CXXFLAGS += -I$(SIMX_DIR) -I$(SIMX_DIR)/rtu -I$(SIM_COMMON_DIR) +CXXFLAGS += -I$(ROOT_DIR)/sw -I$(ROOT_DIR)/hw +CXXFLAGS += -I$(THIRD_PARTY_DIR)/softfloat/source/include + +SRCS += $(SIMX_DIR)/rtu/rtu_isect.cpp +SRCS += $(SRC_DIR)/main.cpp + +PARAMS := -GTAG_WIDTH=32 + +RTL_PKGS += $(RTL_DIR)/VX_gpu_pkg.sv $(RTL_DIR)/fpu/VX_fpu_pkg.sv $(RTL_DIR)/rtu/VX_rtu_pkg.sv +RTL_INCLUDE := -I$(ROOT_DIR)/sw -I$(RTL_DIR) -I$(RTL_DIR)/libs -I$(RTL_DIR)/interfaces +RTL_INCLUDE += -I$(RTL_DIR)/fpu -I$(RTL_DIR)/rtu -I$(SRC_DIR) + +VL_FLAGS += -I$(ROOT_DIR)/hw + +TOP := VX_rtu_box_pe_tb + +include ../common.mk diff --git a/hw/unittest/rtu_box_pe/VX_rtu_box_pe_tb.sv b/hw/unittest/rtu_box_pe/VX_rtu_box_pe_tb.sv new file mode 100644 index 0000000000..f89563d573 --- /dev/null +++ b/hw/unittest/rtu_box_pe/VX_rtu_box_pe_tb.sv @@ -0,0 +1,111 @@ +// Copyright © 2019-2023 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Ray setup + box PE as the scheduler composes them: inv_d comes from +// VX_rtu_recip on the ray direction, the rest of the box request waits for it. + +`include "VX_define.vh" + +module VX_rtu_box_pe_tb import VX_gpu_pkg::*, VX_fpu_pkg::*, VX_rtu_pkg::*; #( + parameter TAG_WIDTH = 32 +) ( + input wire clk, + input wire reset, + input wire valid_in, + input wire [TAG_WIDTH-1:0] tag_in, + input wire [2:0][31:0] origin, + input wire [2:0][7:0] exp, + input wire [2:0][7:0] qmin, + input wire [2:0][7:0] qmax, + input wire raw, + input wire [2:0][31:0] raw_min, + input wire [2:0][31:0] raw_max, + input wire [2:0][31:0] ro, + input wire [2:0][31:0] dir, + input wire [31:0] t_min, + input wire [31:0] t_max, + + // the reciprocal stage, observable on its own + output wire inv_valid, + output wire [TAG_WIDTH-1:0] inv_tag, + output wire [2:0][31:0] inv_d, + + output wire valid_out, + output wire [TAG_WIDTH-1:0] tag_out, + output wire hit, + output wire [31:0] t_near +); + localparam LAT = RTU_FDIV_LAT; + localparam REQW = 1 + TAG_WIDTH + 3*32 + 3*8*3 + 1 + 3*32*3 + 2*32; + + for (genvar a = 0; a < 3; ++a) begin : g_recip + VX_rtu_recip #( + .LATENCY (LAT), + .DSP_SEED (`VX_CFG_RTU_RECIP_DSP_SEED) + ) recip ( + .clk (clk), + .reset (reset), + .enable (1'b1), + .mask (1'b1), + .x (dir[a]), + .result (inv_d[a]) + ); + end + + wire valid_d, raw_d; + wire [TAG_WIDTH-1:0] tag_d; + wire [2:0][31:0] origin_d, raw_min_d, raw_max_d, ro_d; + wire [2:0][7:0] exp_d, qmin_d, qmax_d; + wire [31:0] t_min_d, t_max_d; + VX_shift_register #( + .DATAW (REQW), + .RESETW (1), + .DEPTH (LAT) + ) sr_req ( + .clk (clk), + .reset (reset), + .enable (1'b1), + .data_in ({valid_in, tag_in, origin, exp, qmin, qmax, raw, raw_min, raw_max, ro, t_min, t_max}), + .data_out ({valid_d, tag_d, origin_d, exp_d, qmin_d, qmax_d, raw_d, raw_min_d, raw_max_d, ro_d, t_min_d, t_max_d}) + ); + + assign inv_valid = valid_d; + assign inv_tag = tag_d; + + VX_rtu_box_pe #( + .TAG_WIDTH (TAG_WIDTH) + ) box_pe ( + .clk (clk), + .reset (reset), + .enable (1'b1), + .valid_in (valid_d), + .tag_in (tag_d), + .origin (origin_d), + .exp (exp_d), + .qmin (qmin_d), + .qmax (qmax_d), + .raw (raw_d), + .raw_min (raw_min_d), + .raw_max (raw_max_d), + .ro (ro_d), + .inv_d (inv_d), + .t_min (t_min_d), + .t_max (t_max_d), + .valid_out (valid_out), + .tag_out (tag_out), + `UNUSED_PIN (tag_out_pre), + .hit (hit), + .t_near (t_near) + ); + +endmodule diff --git a/hw/unittest/rtu_box_pe/main.cpp b/hw/unittest/rtu_box_pe/main.cpp new file mode 100644 index 0000000000..73f8916470 --- /dev/null +++ b/hw/unittest/rtu_box_pe/main.cpp @@ -0,0 +1,497 @@ +// VX_rtu_recip + VX_rtu_box_pe against SimX's box test (rtu::ray_recip, +// quant_corner, box_rel, ray_box): inv_d bit for bit, every accept decision, +// t_near by value, and the order the scheduler's insertion collector gives a +// node's accepted children against SimX's nearest-first sort. +// +// The PE's FP units flush subnormals (FTZ/DAZ) while SimX runs IEEE. A case +// whose RTL result differs from SimX but equals SimX evaluated under the host's +// FTZ/DAZ mode, and only there, is counted as a subnormal flush, not a mismatch. + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "VVX_rtu_box_pe_tb.h" +#include "verilated.h" +#include "rtu_isect.h" + +namespace { + +constexpr int kMaxChildren = 6; + +struct Ray { + float o[3], d[3], tmin, tmax; +}; + +struct Child { + uint8_t qmin[3], qmax[3]; + float rmin[3], rmax[3]; +}; + +struct Node { + Ray ray; + float origin[3]; + int8_t exp[3]; + bool raw = false; + int n = 0; + Child ch[kMaxChildren]; + int cat = 0; +}; + +const char* const kCatNames[] = { + "random", "random raw", "zero dir on slab", "zero dir off slab", "flat box", + "tmin past exit", "tmax == entry", "tmax < entry", "exit == 0", "-0 operand", + "special origin", "special ro", "special dir", "special raw", "special tmin", + "special tmax", "exp extremes", +}; +constexpr int kNumCats = int(sizeof(kCatNames) / sizeof(kCatNames[0])); + +uint32_t bits(float f) { uint32_t b; std::memcpy(&b, &f, 4); return b; } +float fbits(uint32_t b) { float f; std::memcpy(&f, &b, 4); return f; } + +bool is_nan_bits(uint32_t b) { return ((b >> 23) & 0xff) == 0xff && (b & 0x7fffff) != 0; } + +// bit-exact, except that any NaN matches any NaN (the payload is not specified) +bool same_bits(uint32_t rtl, float ref) { + return std::isnan(ref) ? is_nan_bits(rtl) : rtl == bits(ref); +} + +// IEEE value equality (+0 == -0), any NaN matches any NaN +bool same_value(uint32_t rtl, float ref) { + return std::isnan(ref) ? is_nan_bits(rtl) : fbits(rtl) == ref; +} + +// the child box in world space, to aim rays at +void reconstruct_child_aabb(const float origin[3], const int8_t exp[3], + const uint8_t qmin[3], const uint8_t qmax[3], + float out_mn[3], float out_mx[3]) { + for (int i = 0; i < 3; ++i) { + float scale = std::ldexp(1.0f, exp[i]); + out_mn[i] = origin[i] + static_cast(qmin[i]) * scale; + out_mx[i] = origin[i] + static_cast(qmax[i]) * scale; + } +} + +struct BoxResult { + bool hit = false; + float t_near = 0; +}; + +struct Ref { + float inv[3]; + BoxResult box; +}; + +Ref reference(const Node& nd, int c, bool ftz) { + const unsigned csr = _mm_getcsr(); + if (ftz) _mm_setcsr(csr | 0x8040); // FTZ | DAZ + namespace rtu = vortex::rtu; + Ref r; + for (int i = 0; i < 3; ++i) + r.inv[i] = rtu::ray_recip(nd.ray.d[i]); + static const float kRawBase[3] = { 0.f, 0.f, 0.f }; + float mn[3], mx[3], rel_mn[3], rel_mx[3]; + if (nd.raw) { + std::memcpy(mn, nd.ch[c].rmin, sizeof mn); + std::memcpy(mx, nd.ch[c].rmax, sizeof mx); + } else { + for (int i = 0; i < 3; ++i) { + mn[i] = rtu::quant_corner(nd.ch[c].qmin[i], nd.exp[i]); + mx[i] = rtu::quant_corner(nd.ch[c].qmax[i], nd.exp[i]); + } + } + rtu::box_rel(nd.raw ? kRawBase : nd.origin, mn, mx, nd.ray.o, rel_mn, rel_mx); + r.box.hit = rtu::ray_box(rel_mn, rel_mx, r.inv, nd.ray.tmin, nd.ray.tmax, r.box.t_near); + _mm_setcsr(csr); + return r; +} + +bool same_box(bool hit, uint32_t t_near, const BoxResult& r) { + return hit == r.hit && (!r.hit || same_value(t_near, r.t_near)); +} + +bool same_ref(const Ref& a, const Ref& b) { + for (int i = 0; i < 3; ++i) + if (!same_bits(bits(a.inv[i]), b.inv[i])) return false; + return same_box(a.box.hit, bits(a.box.t_near), b.box); +} + +// SimX walker: accepted children, insertion-sorted by t_near (stable). +std::vector simx_order(const std::vector& r) { + std::vector o; + for (int i = 0; i < int(r.size()); ++i) { + if (!r[i].hit) continue; + size_t j = o.size(); + o.push_back(i); + while (j > 0 && r[o[j - 1]].t_near > r[i].t_near) { o[j] = o[j - 1]; --j; } + o[j] = i; + } + return o; +} + +// RTL scheduler collector: results arrive in child order; each goes after every +// collected entry whose t_near is <= its own as unsigned bits. +std::vector rtl_order(const std::vector& hit, const std::vector& t) { + std::vector o; + for (int i = 0; i < int(hit.size()); ++i) { + if (!hit[i]) continue; + size_t j = 0; + while (j < o.size() && t[o[j]] <= t[i]) ++j; + o.insert(o.begin() + j, i); + } + return o; +} + +std::mt19937 rng(11); +float uni(float lo, float hi) { return std::uniform_real_distribution(lo, hi)(rng); } +int irand(int lo, int hi) { return std::uniform_int_distribution(lo, hi)(rng); } +bool chance(int n) { return irand(0, n - 1) == 0; } + +void random_children(Node& nd, int n) { + nd.n = n; + for (int c = 0; c < n; ++c) { + for (int a = 0; a < 3; ++a) { + int lo = irand(0, 255), hi = chance(10) ? lo : irand(lo, 255); + nd.ch[c].qmin[a] = uint8_t(lo); + nd.ch[c].qmax[a] = uint8_t(hi); + float r0 = uni(-100, 100), r1 = chance(10) ? r0 : r0 + uni(0, 50); + nd.ch[c].rmin[a] = r0; + nd.ch[c].rmax[a] = r1; + } + } +} + +// node box corner along axis a, at quantized coordinate q +float node_coord(const Node& nd, int a, float q) { + return nd.origin[a] + q * std::ldexp(1.0f, nd.exp[a]); +} + +Node make_random(uint32_t i) { + Node nd; + nd.raw = (i % 10 == 0); + nd.cat = nd.raw ? 1 : 0; + const float S = (i % 3 == 0) ? 1.f : ((i % 3 == 1) ? 100.f : 1e4f); + for (int a = 0; a < 3; ++a) { + nd.origin[a] = uni(-S, S); + nd.exp[a] = int8_t(chance(50) ? irand(-128, 127) : irand(-15, 5)); + } + random_children(nd, irand(1, kMaxChildren)); + if (chance(3)) { // overlapping siblings, so several are accepted together + for (int c = 0; c < nd.n; ++c) + for (int a = 0; a < 3; ++a) { + nd.ch[c].qmin[a] = uint8_t(irand(0, 120)); + nd.ch[c].qmax[a] = uint8_t(irand(130, 255)); + nd.ch[c].rmin[a] = uni(-100, -10); + nd.ch[c].rmax[a] = uni(10, 100); + } + } + Ray& r = nd.ray; + // half the rays aim into one child, the rest anywhere in the node + float target[3]; + const int aim = chance(2) ? irand(0, nd.n - 1) : -1; + for (int a = 0; a < 3; ++a) { + const Child& ch = nd.ch[aim < 0 ? 0 : aim]; + const float f = uni(0, 1); + if (aim < 0) + target[a] = nd.raw ? uni(-100, 150) : node_coord(nd, a, uni(0, 255)); + else + target[a] = nd.raw ? ch.rmin[a] + f * (ch.rmax[a] - ch.rmin[a]) + : node_coord(nd, a, ch.qmin[a] + f * float(ch.qmax[a] - ch.qmin[a])); + const float ext = nd.raw ? 150.f : 255.f * std::ldexp(1.0f, nd.exp[a]); + r.o[a] = chance(3) ? target[a] + uni(-0.3f, 0.3f) * ext // inside / near + : target[a] + uni(-3.f, 3.f) * ext; + } + for (int a = 0; a < 3; ++a) { + r.d[a] = target[a] - r.o[a]; + if (chance(4)) r.d[a] = uni(-1, 1); + if (chance(8)) r.d[a] = chance(2) ? 0.f : -0.f; + } + r.tmin = chance(4) ? uni(0, 2) : 0.f; + const int tm = irand(0, 3); + r.tmax = (tm == 0) ? INFINITY : (tm == 1) ? 1e30f : uni(0, 3); + return nd; +} + +std::vector directed_nodes() { + std::vector out; + const float inf = INFINITY, nan = NAN, sub = 1e-40f; + auto one = [](Node nd, int cat) { nd.n = 1; nd.cat = cat; return nd; }; + for (uint32_t i = 0; i < 3000; ++i) { + Node base = make_random(i * 7 + 1); + base.raw = (i % 5 == 0); + random_children(base, 1); + const Child& ch = base.ch[0]; + float mn[3], mx[3]; + if (base.raw) { + std::memcpy(mn, ch.rmin, sizeof mn); std::memcpy(mx, ch.rmax, sizeof mx); + } else { + reconstruct_child_aabb(base.origin, base.exp, ch.qmin, ch.qmax, mn, mx); + } + const int a = int(i % 3); + // zero direction component with the origin exactly on / off a slab plane + { + Node nd = base; + nd.ray.d[a] = (i & 8) ? -0.f : 0.f; + nd.ray.o[a] = (i & 16) ? mx[a] : mn[a]; + out.push_back(one(nd, 2)); + nd.ray.o[a] = (i & 16) ? std::nextafter(mx[a], inf) : std::nextafter(mn[a], -inf); + out.push_back(one(nd, 3)); + } + // flat box crossed by the ray + { + Node nd = base; + nd.ch[0].qmax[a] = nd.ch[0].qmin[a]; + nd.ch[0].rmax[a] = nd.ch[0].rmin[a]; + for (int k = 0; k < 3; ++k) + nd.ray.d[k] = 0.5f * ((k == a ? mn[k] : 0.5f * (mn[k] + mx[k])) - nd.ray.o[k]); + out.push_back(one(nd, 4)); + } + // aimed at the box centre; tmin / tmax placed on the slab interval ends + { + Node nd = base; + for (int k = 0; k < 3; ++k) nd.ray.d[k] = 0.5f * (mn[k] + mx[k]) - nd.ray.o[k]; + nd.ray.tmin = 0.f; nd.ray.tmax = inf; + Ref r = reference(nd, 0, false); + if (r.box.hit) { + // the slab exit: shrink tmax until the box drops out + Node e = nd; + e.ray.tmin = 1e30f; // past the exit + out.push_back(one(e, 5)); + e = nd; e.ray.tmax = r.box.t_near; // entry, tmin = 0 + out.push_back(one(e, 6)); + e.ray.tmax = std::nextafter(r.box.t_near, -inf); + out.push_back(one(e, 7)); + } + // ray leaving the box through the face it starts on: exit at t == 0 + Node x = nd; + x.ray.o[a] = mx[a]; + for (int k = 0; k < 3; ++k) if (k != a) x.ray.o[k] = 0.5f * (mn[k] + mx[k]); + x.ray.d[a] = 1.f; + out.push_back(one(x, 8)); + x.ray.d[a] = -1.f; + out.push_back(one(x, 8)); + } + // -0 in each operand + for (int slot = 0; slot < 6; ++slot) { + Node nd = base; + if (slot == 0) nd.origin[a] = -0.f; + if (slot == 1) nd.ray.o[a] = -0.f; + if (slot == 2) nd.ray.d[a] = -0.f; + if (slot == 3) { nd.ch[0].rmin[a] = -0.f; nd.ch[0].rmax[a] = 0.f; } + if (slot == 4) nd.ray.tmin = -0.f; + if (slot == 5) nd.ray.tmax = -0.f; + out.push_back(one(nd, 9)); + } + } + // special operands in every slot + const float specials[] = { nan, -nan, inf, -inf, 0.f, -0.f, sub, -sub, FLT_MAX, -FLT_MAX }; + for (uint32_t i = 0; i < 300; ++i) { + Node base = make_random(i * 13 + 2); + random_children(base, 1); + for (float sp : specials) { + for (int a = 0; a < 3; ++a) { + Node nd = base; + nd.raw = false; nd.origin[a] = sp; out.push_back(one(nd, 10)); + nd = base; nd.ray.o[a] = sp; out.push_back(one(nd, 11)); + nd = base; nd.ray.d[a] = sp; out.push_back(one(nd, 12)); + nd = base; nd.raw = true; nd.ch[0].rmin[a] = sp; out.push_back(one(nd, 13)); + nd = base; nd.raw = true; nd.ch[0].rmax[a] = sp; out.push_back(one(nd, 13)); + } + Node nd = base; + nd.ray.tmin = sp; out.push_back(one(nd, 14)); + nd = base; nd.ray.tmax = sp; out.push_back(one(nd, 15)); + } + } + // dequant exponents at and past the F32 range + const int exps[] = { -128, -127, -126, -125, -120, 100, 120, 126, 127 }; + const int qs[] = { 0, 1, 2, 3, 128, 255 }; + for (uint32_t i = 0; i < 200; ++i) { + Node base = make_random(i * 17 + 3); + base.raw = false; + random_children(base, 1); + for (int e : exps) { + for (int q : qs) { + Node nd = base; + const int a = int(i % 3); + nd.exp[a] = int8_t(e); + nd.ch[0].qmin[a] = uint8_t(i & 1 ? 0 : q); + nd.ch[0].qmax[a] = uint8_t(q); + if (i & 2) nd.origin[a] = (i & 4) ? -FLT_MAX : 0.f; + out.push_back(one(nd, 16)); + } + } + } + return out; +} + +} // namespace + +int main(int argc, char** argv) { + Verilated::commandArgs(argc, argv); + const uint32_t N = (argc > 1) ? uint32_t(std::atoi(argv[1])) : 1000000; + VVX_rtu_box_pe_tb dut; + dut.clk = 0; dut.reset = 1; dut.valid_in = 0; + auto tick = [&] { dut.clk = 0; dut.eval(); dut.clk = 1; dut.eval(); }; + for (int i = 0; i < 4; ++i) tick(); + dut.reset = 0; + + std::vector nodes = directed_nodes(); + const uint32_t ND = uint32_t(nodes.size()); + uint32_t random_boxes = 0; + for (uint32_t i = 0; random_boxes < N; ++i) { + nodes.push_back(make_random(i)); + random_boxes += uint32_t(nodes.back().n); + } + + struct Item { uint32_t node; int child; Ref ieee, ftz; }; + std::vector items; + for (uint32_t k = 0; k < nodes.size(); ++k) + for (int c = 0; c < nodes[k].n; ++c) + items.push_back({k, c, reference(nodes[k], c, false), reference(nodes[k], c, true)}); + + // per node: RTL results, and whether each child matched only the FTZ reference + std::vector> rtl_hit(nodes.size()); + std::vector> rtl_t(nodes.size()); + std::vector node_ftz(nodes.size(), false); + for (uint32_t k = 0; k < nodes.size(); ++k) { + rtl_hit[k].assign(size_t(nodes[k].n), false); + rtl_t[k].assign(size_t(nodes[k].n), 0); + } + + uint32_t cat_cases[kNumCats] = {}, cat_err[kNumCats] = {}, cat_ftz[kNumCats] = {}; + uint32_t errors = 0, flushed = 0, inv_errors = 0, hits = 0; + uint32_t order_multi = 0, order_checked = 0, order_errors = 0, order_ftz = 0; + std::deque inv_q, out_q; + std::vector inv_ok(items.size(), 0); + uint32_t sent = 0, checked = 0; + const uint32_t total = uint32_t(items.size()); + + auto report_case = [&](const Item& it, const char* what) { + const Node& nd = nodes[it.node]; + const Child& ch = nd.ch[it.child]; + std::printf(" [%s] %s raw=%d origin=(%a %a %a) exp=(%d %d %d) q=(%u %u %u)-(%u %u %u) " + "rmin=(%a %a %a) rmax=(%a %a %a) o=(%a %a %a) d=(%a %a %a) tmin=%a tmax=%a\n", + kCatNames[nd.cat], what, nd.raw, nd.origin[0], nd.origin[1], nd.origin[2], + nd.exp[0], nd.exp[1], nd.exp[2], ch.qmin[0], ch.qmin[1], ch.qmin[2], + ch.qmax[0], ch.qmax[1], ch.qmax[2], ch.rmin[0], ch.rmin[1], ch.rmin[2], + ch.rmax[0], ch.rmax[1], ch.rmax[2], nd.ray.o[0], nd.ray.o[1], nd.ray.o[2], + nd.ray.d[0], nd.ray.d[1], nd.ray.d[2], nd.ray.tmin, nd.ray.tmax); + }; + + while (checked < total) { + if (sent < total) { + const Item& it = items[sent]; + const Node& nd = nodes[it.node]; + const Child& ch = nd.ch[it.child]; + for (int a = 0; a < 3; ++a) { + dut.origin[a] = bits(nd.origin[a]); + dut.raw_min[a] = bits(ch.rmin[a]); + dut.raw_max[a] = bits(ch.rmax[a]); + dut.ro[a] = bits(nd.ray.o[a]); + dut.dir[a] = bits(nd.ray.d[a]); + } + dut.exp = uint32_t(uint8_t(nd.exp[0])) | uint32_t(uint8_t(nd.exp[1])) << 8 + | uint32_t(uint8_t(nd.exp[2])) << 16; + dut.qmin = uint32_t(ch.qmin[0]) | uint32_t(ch.qmin[1]) << 8 | uint32_t(ch.qmin[2]) << 16; + dut.qmax = uint32_t(ch.qmax[0]) | uint32_t(ch.qmax[1]) << 8 | uint32_t(ch.qmax[2]) << 16; + dut.raw = nd.raw; + dut.t_min = bits(nd.ray.tmin); + dut.t_max = bits(nd.ray.tmax); + dut.tag_in = sent; + dut.valid_in = 1; + inv_q.push_back(sent); + out_q.push_back(sent); + ++sent; + } else { + dut.valid_in = 0; + } + tick(); + if (dut.inv_valid) { + const uint32_t id = inv_q.front(); + inv_q.pop_front(); + const Item& it = items[id]; + bool ok = (dut.inv_tag == id), ftz_ok = ok; + for (int a = 0; a < 3; ++a) { + ok = ok && same_bits(dut.inv_d[a], it.ieee.inv[a]); + ftz_ok = ftz_ok && same_bits(dut.inv_d[a], it.ftz.inv[a]); + } + inv_ok[id] = ok ? 1 : (ftz_ok ? 2 : 0); + } + if (dut.valid_out) { + const uint32_t id = out_q.front(); + out_q.pop_front(); + const Item& it = items[id]; + const int cat = nodes[it.node].cat; + ++cat_cases[cat]; + bool ok = (dut.tag_out == id) && inv_ok[id] == 1 + && same_box(dut.hit, dut.t_near, it.ieee.box); + if (it.ieee.box.hit) ++hits; + if (!ok && dut.tag_out == id && inv_ok[id] != 0 + && same_box(dut.hit, dut.t_near, it.ftz.box) && !same_ref(it.ieee, it.ftz)) { + ok = true; + ++cat_ftz[cat]; + ++flushed; + node_ftz[it.node] = true; + } + if (!ok) { + ++errors; + if (inv_ok[id] == 0) ++inv_errors; + if (cat_err[cat]++ < 3) { + report_case(it, "MISMATCH"); + std::printf(" ref hit=%d t_near=%08x inv=(%08x %08x %08x) | rtl tag=%u hit=%d t_near=%08x inv_ok=%d\n", + it.ieee.box.hit, bits(it.ieee.box.t_near), bits(it.ieee.inv[0]), + bits(it.ieee.inv[1]), bits(it.ieee.inv[2]), dut.tag_out, dut.hit, + dut.t_near, inv_ok[id]); + } + } + rtl_hit[it.node][size_t(it.child)] = dut.hit; + rtl_t[it.node][size_t(it.child)] = dut.t_near; + ++checked; + } + } + + // child visit order per node + { + size_t base = 0; + for (uint32_t k = 0; k < nodes.size(); ++k) { + const int n = nodes[k].n; + if (n >= 2) { + std::vector ieee, ftz; + for (int c = 0; c < n; ++c) { + ieee.push_back(items[base + size_t(c)].ieee.box); + ftz.push_back(items[base + size_t(c)].ftz.box); + } + ++order_checked; + int acc = 0; + for (const BoxResult& b : ieee) acc += b.hit; + if (acc >= 2) ++order_multi; + const std::vector got = rtl_order(rtl_hit[k], rtl_t[k]); + if (got != simx_order(ieee)) { + if (node_ftz[k] && got == simx_order(ftz)) { + ++order_ftz; + } else { + ++order_errors; + } + } + } + base += size_t(n); + } + } + + std::printf("rtu_box_pe: %u boxes (%u directed), %u accepted, %u mismatches " + "(%u in inv_d), %u subnormal flushes\n", + checked, ND, hits, errors, inv_errors, flushed); + std::printf(" child order: %u nodes (%u with 2+ accepted), %u mismatches, %u subnormal flushes\n", + order_checked, order_multi, order_errors, order_ftz); + for (int k = 0; k < kNumCats; ++k) + std::printf(" %-20s %8u boxes %6u mismatches %6u subnormal flushes\n", + kCatNames[k], cat_cases[k], cat_err[k], cat_ftz[k]); + const bool fail = errors || order_errors; + std::printf(fail ? "FAILED!\n" : "PASSED!\n"); + return fail ? 1 : 0; +} diff --git a/hw/unittest/rtu_tri_pe/Makefile b/hw/unittest/rtu_tri_pe/Makefile new file mode 100644 index 0000000000..3d196e3e5d --- /dev/null +++ b/hw/unittest/rtu_tri_pe/Makefile @@ -0,0 +1,34 @@ +ROOT_DIR := $(realpath ../../..) +include $(ROOT_DIR)/config.mk + +PROJECT := rtu_tri_pe + +RTL_DIR := $(VORTEX_HOME)/hw/rtl +SRC_DIR := $(VORTEX_HOME)/hw/unittest/$(PROJECT) +SIMX_DIR := $(VORTEX_HOME)/sim/simx + +# The reference is the SimX model itself, so the RTU config must match both sides. +CONFIGS += -DSIMULATION -DVX_CFG_EXT_RTU_ENABLE + +CXXFLAGS := -I$(SRC_DIR) -I$(VORTEX_HOME)/hw/unittest/common -I$(SW_COMMON_DIR) +CXXFLAGS += -I$(SIMX_DIR) -I$(SIMX_DIR)/rtu -I$(SIM_COMMON_DIR) +CXXFLAGS += -I$(ROOT_DIR)/sw -I$(ROOT_DIR)/hw +CXXFLAGS += -I$(THIRD_PARTY_DIR)/softfloat/source/include + +SRCS += $(SIMX_DIR)/rtu/rtu_isect.cpp +SRCS += $(SRC_DIR)/main.cpp + +PARAMS := -GTAG_WIDTH=32 + +RTL_PKGS += $(RTL_DIR)/VX_gpu_pkg.sv $(RTL_DIR)/fpu/VX_fpu_pkg.sv $(RTL_DIR)/rtu/VX_rtu_pkg.sv +RTL_INCLUDE := -I$(ROOT_DIR)/sw -I$(RTL_DIR) -I$(RTL_DIR)/libs -I$(RTL_DIR)/interfaces +RTL_INCLUDE += -I$(RTL_DIR)/fpu -I$(RTL_DIR)/rtu -I$(SRC_DIR) + +VL_FLAGS += -I$(ROOT_DIR)/hw + +TOP := VX_rtu_tri_pe + +include ../common.mk + +# the soft FMA's function locals shadow this module's v port once inlined +VL_FLAGS += -Wno-VARHIDDEN diff --git a/hw/unittest/rtu_tri_pe/main.cpp b/hw/unittest/rtu_tri_pe/main.cpp new file mode 100644 index 0000000000..e33687ada1 --- /dev/null +++ b/hw/unittest/rtu_tri_pe/main.cpp @@ -0,0 +1,313 @@ +// VX_rtu_tri_pe against SimX's rtu::ray_triangle: every verdict and, for a hit, +// t / u / v / back_facing must match bit for bit (any NaN matches any NaN). +// +// The PE's FP units flush subnormals (FTZ/DAZ) while SimX runs IEEE. A case +// whose RTL result differs from SimX but equals SimX evaluated under the host's +// FTZ/DAZ mode, and only there, is counted as a subnormal flush, not a mismatch. + +#include +#include +#include +#include +#include +#include +#include +#include +#include "VVX_rtu_tri_pe.h" +#include "verilated.h" +#include "rtu_isect.h" + +namespace { + +struct Case { + float o[3], d[3], v[3][3], tmin, tmax; + int cat = 0; // directed-case family (0: random) +}; + +const char* const kCatNames[] = { + "random", "t==tmin", "t==tmax", "t==tmin==tmax", "t in (t-,t+)", + "tmin=t+", "tmax=t-", "origin on triangle", "zero dir", + "special origin", "special dir", "special vertex", "special tmin", "special tmax", + "shared edge", +}; +constexpr int kNumCats = int(sizeof(kCatNames) / sizeof(kCatNames[0])); + +struct Result { + bool hit = false; + float t = 0, u = 0, v = 0; + bool back = false; +}; + +struct Expect { + uint32_t id; + int cat; + Result ieee, ftz; +}; + +uint32_t bits(float f) { uint32_t b; std::memcpy(&b, &f, 4); return b; } + +// bit-exact, except that any NaN matches any NaN (the payload is not specified) +bool same(uint32_t rtl, float ref) { + const bool rtl_nan = ((rtl >> 23) & 0xff) == 0xff && (rtl & 0x7fffff) != 0; + return std::isnan(ref) ? rtl_nan : rtl == bits(ref); +} + +Result reference(const Case& c, bool ftz) { + const unsigned csr = _mm_getcsr(); + if (ftz) _mm_setcsr(csr | 0x8040); // FTZ | DAZ + Result r; + r.hit = vortex::rtu::ray_triangle(c.o, c.d, c.v[0], c.v[1], c.v[2], + c.tmin, c.tmax, r.t, r.u, r.v, r.back); + _mm_setcsr(csr); + return r; +} + +template +bool matches(const Dut& dut, const Result& r) { + if (bool(dut.hit) != r.hit) return false; + return !r.hit || (same(dut.t, r.t) && same(dut.u, r.u) && same(dut.v, r.v) + && bool(dut.back_facing) == r.back); +} + +bool same_result(const Result& a, const Result& b) { + return a.hit == b.hit + && (!a.hit || (same(bits(a.t), b.t) && same(bits(a.u), b.u) + && same(bits(a.v), b.v) && a.back == b.back)); +} + +std::mt19937 rng(7); +float uni(float lo, float hi) { return std::uniform_real_distribution(lo, hi)(rng); } + +// A ray aimed at a point inside (or just outside) the triangle, so hits, +// near-edge misses and shared-edge cases all get exercised. +Case make_case(uint32_t i, const Case* twin_of) { + Case c; + if (twin_of) { + c = *twin_of; + // coincident twin: rotate (even) or mirror (odd) the vertex order + float v[3][3]; + std::memcpy(v, c.v, sizeof v); + if (i & 1) { std::memcpy(c.v[1], v[2], 12); std::memcpy(c.v[2], v[1], 12); } + else { std::memcpy(c.v[0], v[1], 12); std::memcpy(c.v[1], v[2], 12); std::memcpy(c.v[2], v[0], 12); } + return c; + } + const float s = (i % 7 == 0) ? 1e3f : ((i % 5 == 0) ? 1e-2f : 10.f); + for (auto& vv : c.v) for (float& x : vv) x = uni(-s, s); + float a = uni(-0.2f, 1.2f), b = uni(-0.2f, 1.2f); + if ((i % 3) == 0) b = 1.f - a; // on / near an edge + float p[3]; + for (int k = 0; k < 3; ++k) + p[k] = c.v[0][k] + a * (c.v[1][k] - c.v[0][k]) + b * (c.v[2][k] - c.v[0][k]); + for (int k = 0; k < 3; ++k) c.o[k] = uni(-3 * s, 3 * s); + for (int k = 0; k < 3; ++k) c.d[k] = p[k] - c.o[k]; + if ((i % 11) == 0) c.d[i % 3] = 0.f; // axis-aligned component + c.tmin = (i % 13 == 0) ? uni(0.f, 1.f) : 0.f; + c.tmax = (i % 17 == 0) ? uni(0.f, 2.f) : 1e30f; + return c; +} + +// Quad (p0, p1, p2, p3) split along p0-p2 into (p0, p1, p2) and (p0, p2, p3), +// both wound alike; the ray aims at a point on the shared edge p0-p2. +float quad_pts[20000][4][3]; + +Case shared_edge_case(uint32_t i) { + std::mt19937 g(1000 + i); + auto u = [&](float lo, float hi) { return std::uniform_real_distribution(lo, hi)(g); }; + float (&p)[4][3] = quad_pts[i]; + const float s = (i % 3 == 0) ? 1e3f : ((i % 3 == 1) ? 1.f : 1e-2f); + for (int k = 0; k < 3; ++k) { p[0][k] = u(-s, s); p[2][k] = u(-s, s); } + float m[3], n[3]; + for (int k = 0; k < 3; ++k) { m[k] = 0.5f * (p[0][k] + p[2][k]); n[k] = u(-s, s); } + for (int k = 0; k < 3; ++k) { p[1][k] = m[k] + n[k]; p[3][k] = m[k] - n[k]; } + Case c; + c.cat = 14; + std::memcpy(c.v[0], p[0], 12); + std::memcpy(c.v[1], p[1], 12); + std::memcpy(c.v[2], p[2], 12); + const float a = u(0.f, 1.f); + float e[3]; + for (int k = 0; k < 3; ++k) e[k] = p[0][k] + a * (p[2][k] - p[0][k]); + for (int k = 0; k < 3; ++k) c.o[k] = u(-3 * s, 3 * s); + for (int k = 0; k < 3; ++k) c.d[k] = e[k] - c.o[k]; + c.tmin = 0.f; + c.tmax = INFINITY; + return c; +} + +const float* shared_edge_far(uint32_t i) { return quad_pts[i][3]; } + +// Directed cases: t landing exactly on tmin / tmax, a zero t from an origin on +// the triangle, and NaN / inf / -0 / subnormal operands. +std::vector directed_cases() { + std::vector out; + const float inf = INFINITY, nan = NAN; + const float sub = 1e-40f; + for (uint32_t i = 0; i < 4000; ++i) { + Case c = make_case(i * 3 + 1, nullptr); // edge-biased family excluded + c.tmin = 0.f; c.tmax = 1e30f; + float t, u, v; bool bf; + if (!vortex::rtu::ray_triangle(c.o, c.d, c.v[0], c.v[1], c.v[2], + -inf, inf, t, u, v, bf)) + continue; + Case e = c; + e.cat = 1; e.tmin = t; e.tmax = inf; out.push_back(e); + e.cat = 2; e.tmin = -inf; e.tmax = t; out.push_back(e); + e.cat = 3; e.tmin = t; e.tmax = t; out.push_back(e); + e.cat = 4; e.tmin = std::nextafter(t, -inf); e.tmax = std::nextafter(t, inf); + out.push_back(e); + e.cat = 5; e.tmin = std::nextafter(t, inf); e.tmax = inf; out.push_back(e); + e.cat = 6; e.tmin = -inf; e.tmax = std::nextafter(t, -inf); out.push_back(e); + } + // origin on the triangle's plane: every rz is 0, so t is exactly +-0 + for (uint32_t i = 0; i < 2000; ++i) { + Case c; + c.cat = 7; + for (auto& vv : c.v) { vv[0] = uni(-5, 5); vv[1] = uni(-5, 5); vv[2] = 0.f; } + const float a = uni(0.f, 0.5f), b = uni(0.f, 0.5f); + for (int k = 0; k < 2; ++k) + c.o[k] = c.v[0][k] + a * (c.v[1][k] - c.v[0][k]) + b * (c.v[2][k] - c.v[0][k]); + c.o[2] = (i & 1) ? -0.f : 0.f; + c.d[0] = uni(-1, 1); c.d[1] = uni(-1, 1); c.d[2] = (i & 2) ? uni(0.5f, 2) : -uni(0.5f, 2); + const float tmins[] = { 0.f, -0.f, -1.f, sub, -sub }; + c.tmin = tmins[i % 5]; + c.tmax = (i % 7 == 0) ? 0.f : 1e30f; + out.push_back(c); + } + // special operands in every input slot + const float specials[] = { nan, -nan, inf, -inf, 0.f, -0.f, sub, -sub, 3.4e38f }; + for (uint32_t i = 0; i < 400; ++i) { + const Case base = make_case(i * 5 + 2, nullptr); + for (float sp : specials) { + for (int slot = 0; slot < 17; ++slot) { + Case c = base; + c.cat = (slot < 3) ? 9 : (slot < 6) ? 10 : (slot < 15) ? 11 : (slot == 15) ? 12 : 13; + if (slot < 3) c.o[slot] = sp; + else if (slot < 6) c.d[slot - 3] = sp; + else if (slot < 15) c.v[(slot - 6) / 3][(slot - 6) % 3] = sp; + else if (slot == 15) c.tmin = sp; + else c.tmax = sp; + out.push_back(c); + } + } + Case z = base; // zero direction + z.cat = 8; + z.d[0] = z.d[1] = z.d[2] = (i & 1) ? -0.f : 0.f; + out.push_back(z); + } + // rays at the shared edge of two triangles (a quad split along its diagonal), + // both triangles traced + for (uint32_t i = 0; i < 20000; ++i) { + Case c = shared_edge_case(i); + out.push_back(c); + std::memcpy(c.v[1], c.v[2], 12); + std::memcpy(c.v[2], shared_edge_far(i), 12); + out.push_back(c); + } + return out; +} + +// Watertightness: a ray through a shared edge hits at least one of the two +// triangles (SimX alone; the RTL then matches it case by case). +uint32_t shared_edge_leaks() { + uint32_t leaks = 0; + for (uint32_t i = 0; i < 20000; ++i) { + Case a = shared_edge_case(i), b = a; + std::memcpy(b.v[1], b.v[2], 12); + std::memcpy(b.v[2], shared_edge_far(i), 12); + float t, u, v; bool bf; + const bool ha = vortex::rtu::ray_triangle(a.o, a.d, a.v[0], a.v[1], a.v[2], + a.tmin, a.tmax, t, u, v, bf); + const bool hb = vortex::rtu::ray_triangle(b.o, b.d, b.v[0], b.v[1], b.v[2], + b.tmin, b.tmax, t, u, v, bf); + leaks += !(ha || hb); + } + return leaks; +} + +} // namespace + +int main(int argc, char** argv) { + Verilated::commandArgs(argc, argv); + const uint32_t N = (argc > 1) ? uint32_t(std::atoi(argv[1])) : 1000000; + VVX_rtu_tri_pe dut; + dut.clk = 0; dut.reset = 1; dut.enable = 1; dut.valid_in = 0; + auto tick = [&] { dut.clk = 0; dut.eval(); dut.clk = 1; dut.eval(); }; + for (int i = 0; i < 4; ++i) tick(); + dut.reset = 0; + + std::deque exp; + uint32_t sent = 0, checked = 0, errors = 0, hits = 0, twins = 0; + uint32_t cat_cases[kNumCats] = {}, cat_errors[kNumCats] = {}, cat_ftz[kNumCats] = {}; + uint32_t flushed = 0; + std::deque sent_cases; + Case prev{}; + const std::vector directed = directed_cases(); + const uint32_t leaks = shared_edge_leaks(); + const uint32_t ND = uint32_t(directed.size()); + const uint32_t total = N + ND; + while (checked < total) { + if (sent < total) { + Case c; + if (sent < ND) { + c = directed[sent]; + } else { + const uint32_t r = sent - ND; + const bool twin = (r > 0) && (r % 4 == 1); + c = make_case(r, twin ? &prev : nullptr); + if (!twin) prev = c; else ++twins; + } + for (int k = 0; k < 3; ++k) { + dut.origin[k] = bits(c.o[k]); + dut.dir[k] = bits(c.d[k]); + dut.v0[k] = bits(c.v[0][k]); + dut.v1[k] = bits(c.v[1][k]); + dut.v2[k] = bits(c.v[2][k]); + } + dut.t_min = bits(c.tmin); + dut.t_max = bits(c.tmax); + dut.tag_in = sent; + dut.valid_in = 1; + Expect e{sent, c.cat, reference(c, false), reference(c, true)}; + exp.push_back(e); + sent_cases.push_back(c); + ++sent; + } else { + dut.valid_in = 0; + } + tick(); + if (dut.valid_out) { + const Expect e = exp.front(); + exp.pop_front(); + const Case c = sent_cases.front(); + sent_cases.pop_front(); + ++cat_cases[e.cat]; + bool ok = (dut.tag_out == e.id) && matches(dut, e.ieee); + if (e.ieee.hit) ++hits; + if (!ok && dut.tag_out == e.id && matches(dut, e.ftz) && !same_result(e.ieee, e.ftz)) { + ++cat_ftz[e.cat]; + ++flushed; + ok = true; + } + if (!ok) { ++cat_errors[e.cat]; ++errors; } + if (!ok && cat_errors[e.cat] <= 3) { + std::printf(" [%s] o=(%a %a %a) d=(%a %a %a) v0=(%a %a %a) v1=(%a %a %a) v2=(%a %a %a) tmin=%a tmax=%a\n", + kCatNames[e.cat], c.o[0], c.o[1], c.o[2], c.d[0], c.d[1], c.d[2], + c.v[0][0], c.v[0][1], c.v[0][2], c.v[1][0], c.v[1][1], c.v[1][2], + c.v[2][0], c.v[2][1], c.v[2][2], c.tmin, c.tmax); + std::printf("MISMATCH #%u: ref hit=%d t=%08x u=%08x v=%08x bf=%d | rtl tag=%u hit=%d t=%08x u=%08x v=%08x bf=%d\n", + e.id, e.ieee.hit, bits(e.ieee.t), bits(e.ieee.u), bits(e.ieee.v), e.ieee.back, + dut.tag_out, dut.hit, dut.t, dut.u, dut.v, dut.back_facing); + } + ++checked; + } + } + std::printf("rtu_tri_pe: %u cases (%u directed, %u hits, %u twins), %u mismatches, %u subnormal flushes\n", + checked, ND, hits, twins, errors, flushed); + for (int k = 0; k < kNumCats; ++k) + std::printf(" %-20s %8u cases %6u mismatches %6u subnormal flushes\n", + kCatNames[k], cat_cases[k], cat_errors[k], cat_ftz[k]); + std::printf(" shared-edge rays through neither triangle: %u of 20000\n", leaks); + const bool fail = errors || leaks; + std::printf(fail ? "FAILED!\n" : "PASSED!\n"); + return fail ? 1 : 0; +} diff --git a/hw/unittest/rtu_xform/Makefile b/hw/unittest/rtu_xform/Makefile new file mode 100644 index 0000000000..12eccb6bad --- /dev/null +++ b/hw/unittest/rtu_xform/Makefile @@ -0,0 +1,31 @@ +ROOT_DIR := $(realpath ../../..) +include $(ROOT_DIR)/config.mk + +PROJECT := rtu_xform + +RTL_DIR := $(VORTEX_HOME)/hw/rtl +SRC_DIR := $(VORTEX_HOME)/hw/unittest/$(PROJECT) +SIMX_DIR := $(VORTEX_HOME)/sim/simx + +# The reference is the SimX model itself, so the RTU config must match both sides. +CONFIGS += -DSIMULATION -DVX_CFG_EXT_RTU_ENABLE + +CXXFLAGS := -I$(SRC_DIR) -I$(VORTEX_HOME)/hw/unittest/common -I$(SW_COMMON_DIR) +CXXFLAGS += -I$(SIMX_DIR) -I$(SIMX_DIR)/rtu -I$(SIM_COMMON_DIR) +CXXFLAGS += -I$(ROOT_DIR)/sw -I$(ROOT_DIR)/hw +CXXFLAGS += -I$(THIRD_PARTY_DIR)/softfloat/source/include + +SRCS += $(SIMX_DIR)/rtu/rtu_isect.cpp +SRCS += $(SRC_DIR)/main.cpp + +PARAMS := -GTAG_WIDTH=32 + +RTL_PKGS += $(RTL_DIR)/VX_gpu_pkg.sv $(RTL_DIR)/fpu/VX_fpu_pkg.sv $(RTL_DIR)/rtu/VX_rtu_pkg.sv +RTL_INCLUDE := -I$(ROOT_DIR)/sw -I$(RTL_DIR) -I$(RTL_DIR)/libs -I$(RTL_DIR)/interfaces +RTL_INCLUDE += -I$(RTL_DIR)/fpu -I$(RTL_DIR)/rtu -I$(SRC_DIR) + +VL_FLAGS += -I$(ROOT_DIR)/hw + +TOP := VX_rtu_xform + +include ../common.mk diff --git a/hw/unittest/rtu_xform/main.cpp b/hw/unittest/rtu_xform/main.cpp new file mode 100644 index 0000000000..1dd8f2fb9f --- /dev/null +++ b/hw/unittest/rtu_xform/main.cpp @@ -0,0 +1,183 @@ +// VX_rtu_xform against SimX's rtu::world_to_object_ray: the object-space origin +// and direction must match bit for bit (any NaN matches any NaN) for random +// affine instances -- rotation, non-uniform scale, shear, translation -- and +// special operands. +// +// The xform's FP units flush subnormals (FTZ/DAZ) while SimX runs IEEE. A case +// whose RTL result differs from SimX but equals SimX evaluated under the host's +// FTZ/DAZ mode, and only there, is counted as a subnormal flush, not a mismatch. + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "VVX_rtu_xform.h" +#include "verilated.h" +#include "rtu_isect.h" + +namespace { + +struct Case { + float m[12], o[3], d[3]; + int cat = 0; +}; + +const char* const kCatNames[] = { + "random affine", "rigid", "identity", "special matrix", "special ray", +}; +constexpr int kNumCats = int(sizeof(kCatNames) / sizeof(kCatNames[0])); + +struct Result { float o[3], d[3]; }; + +uint32_t bits(float f) { uint32_t b; std::memcpy(&b, &f, 4); return b; } + +bool same(uint32_t rtl, float ref) { + const bool rtl_nan = ((rtl >> 23) & 0xff) == 0xff && (rtl & 0x7fffff) != 0; + return std::isnan(ref) ? rtl_nan : rtl == bits(ref); +} + +Result reference(const Case& c, bool ftz) { + const unsigned csr = _mm_getcsr(); + if (ftz) _mm_setcsr(csr | 0x8040); // FTZ | DAZ + Result r; + vortex::rtu::world_to_object_ray(c.m, c.o, c.d, r.o, r.d); + _mm_setcsr(csr); + return r; +} + +template +bool matches(const Dut& dut, const Result& r) { + for (int a = 0; a < 3; ++a) + if (!same(dut.obj_ro[a], r.o[a]) || !same(dut.obj_rd[a], r.d[a])) return false; + return true; +} + +bool same_result(const Result& a, const Result& b) { + for (int k = 0; k < 3; ++k) + if (!same(bits(a.o[k]), b.o[k]) || !same(bits(a.d[k]), b.d[k])) return false; + return true; +} + +std::mt19937 rng(5); +float uni(float lo, float hi) { return std::uniform_real_distribution(lo, hi)(rng); } + +// A random world->object matrix: rotation x scale x shear (or rigid), plus a +// translation, at assorted magnitudes. +Case make_case(uint32_t i) { + Case c; + const float ax = uni(-1, 1), ay = uni(-1, 1), az = uni(-1, 1); + const float n = std::sqrt(ax * ax + ay * ay + az * az) + 1e-6f; + const float x = ax / n, y = ay / n, z = az / n, th = uni(-3.2f, 3.2f); + const float cs = std::cos(th), sn = std::sin(th), t = 1 - cs; + float R[9] = { t * x * x + cs, t * x * y - sn * z, t * x * z + sn * y, + t * x * y + sn * z, t * y * y + cs, t * y * z - sn * x, + t * x * z - sn * y, t * y * z + sn * x, t * z * z + cs }; + c.cat = (i % 5 == 0) ? 1 : 0; + float S[3] = { 1, 1, 1 }, H[3] = { 0, 0, 0 }; + if (c.cat == 0) { + for (float& s : S) s = (i % 3 == 0) ? uni(1e-3f, 1e3f) : uni(0.1f, 10.f); + for (float& h : H) h = uni(-2, 2); + } + const float M[9] = { S[0], H[0], H[1], 0, S[1], H[2], 0, 0, S[2] }; + for (int r = 0; r < 3; ++r) + for (int k = 0; k < 3; ++k) + c.m[r * 4 + k] = R[r * 3 + 0] * M[0 * 3 + k] + R[r * 3 + 1] * M[1 * 3 + k] + + R[r * 3 + 2] * M[2 * 3 + k]; + const float span = (i % 7 == 0) ? 1e4f : 50.f; + for (int r = 0; r < 3; ++r) c.m[r * 4 + 3] = uni(-span, span); + for (int a = 0; a < 3; ++a) { c.o[a] = uni(-span, span); c.d[a] = uni(-1, 1); } + if (i % 11 == 0) c.d[i % 3] = (i & 1) ? -0.f : 0.f; + return c; +} + +std::vector directed_cases() { + std::vector out; + const float specials[] = { NAN, INFINITY, -INFINITY, 0.f, -0.f, 1e-40f, -1e-40f, + FLT_MAX, -FLT_MAX }; + for (uint32_t i = 0; i < 2000; ++i) { + Case c = make_case(i * 3 + 1); + Case id = c; + id.cat = 2; + for (int k = 0; k < 12; ++k) id.m[k] = (k % 5 == 0) ? 1.f : 0.f; + out.push_back(id); + for (float sp : specials) { + Case s = c; + s.cat = 3; s.m[i % 12] = sp; out.push_back(s); + s = c; s.cat = 4; + if (i & 1) s.o[i % 3] = sp; else s.d[i % 3] = sp; + out.push_back(s); + } + } + return out; +} + +} // namespace + +int main(int argc, char** argv) { + Verilated::commandArgs(argc, argv); + const uint32_t N = (argc > 1) ? uint32_t(std::atoi(argv[1])) : 1000000; + VVX_rtu_xform dut; + dut.clk = 0; dut.reset = 1; dut.enable = 1; dut.valid_in = 0; + auto tick = [&] { dut.clk = 0; dut.eval(); dut.clk = 1; dut.eval(); }; + for (int i = 0; i < 4; ++i) tick(); + dut.reset = 0; + + const std::vector directed = directed_cases(); + const uint32_t ND = uint32_t(directed.size()); + const uint32_t total = N + ND; + struct Expect { uint32_t id; int cat; Result ieee, ftz; }; + std::deque exp; + std::deque cases; + uint32_t sent = 0, checked = 0, errors = 0, flushed = 0; + uint32_t cat_cases[kNumCats] = {}, cat_err[kNumCats] = {}, cat_ftz[kNumCats] = {}; + while (checked < total) { + if (sent < total) { + const Case c = (sent < ND) ? directed[sent] : make_case(sent - ND); + for (int k = 0; k < 12; ++k) dut.xform[k] = bits(c.m[k]); + for (int a = 0; a < 3; ++a) { dut.ro[a] = bits(c.o[a]); dut.rd[a] = bits(c.d[a]); } + dut.tag_in = sent; + dut.valid_in = 1; + exp.push_back({ sent, c.cat, reference(c, false), reference(c, true) }); + cases.push_back(c); + ++sent; + } else { + dut.valid_in = 0; + } + tick(); + if (dut.valid_out) { + const Expect e = exp.front(); exp.pop_front(); + const Case c = cases.front(); cases.pop_front(); + ++cat_cases[e.cat]; + bool ok = (dut.tag_out == e.id) && matches(dut, e.ieee); + if (!ok && dut.tag_out == e.id && matches(dut, e.ftz) && !same_result(e.ieee, e.ftz)) { + ok = true; ++flushed; ++cat_ftz[e.cat]; + } + if (!ok) { + ++errors; + if (cat_err[e.cat]++ < 3) { + std::printf("MISMATCH #%u [%s] m=(", e.id, kCatNames[e.cat]); + for (float v : c.m) std::printf("%a ", v); + std::printf(") o=(%a %a %a) d=(%a %a %a)\n", c.o[0], c.o[1], c.o[2], c.d[0], c.d[1], c.d[2]); + std::printf(" ref o=(%08x %08x %08x) d=(%08x %08x %08x) | rtl o=(%08x %08x %08x) d=(%08x %08x %08x)\n", + bits(e.ieee.o[0]), bits(e.ieee.o[1]), bits(e.ieee.o[2]), + bits(e.ieee.d[0]), bits(e.ieee.d[1]), bits(e.ieee.d[2]), + dut.obj_ro[0], dut.obj_ro[1], dut.obj_ro[2], + dut.obj_rd[0], dut.obj_rd[1], dut.obj_rd[2]); + } + } + ++checked; + } + } + std::printf("rtu_xform: %u cases (%u directed), %u mismatches, %u subnormal flushes\n", + checked, ND, errors, flushed); + for (int k = 0; k < kNumCats; ++k) + std::printf(" %-16s %8u cases %6u mismatches %6u subnormal flushes\n", + kCatNames[k], cat_cases[k], cat_err[k], cat_ftz[k]); + std::printf(errors ? "FAILED!\n" : "PASSED!\n"); + return errors ? 1 : 0; +} diff --git a/hw/unittest/tex_lerp/Makefile b/hw/unittest/tex_lerp/Makefile new file mode 100644 index 0000000000..6e90f797ae --- /dev/null +++ b/hw/unittest/tex_lerp/Makefile @@ -0,0 +1,19 @@ +ROOT_DIR := $(realpath ../../..) +include $(ROOT_DIR)/config.mk + +PROJECT := tex_lerp + +RTL_DIR := $(VORTEX_HOME)/hw/rtl +SRC_DIR := $(VORTEX_HOME)/hw/unittest/$(PROJECT) + +CXXFLAGS := -I$(SRC_DIR) -I$(VORTEX_HOME)/hw/unittest/common -I$(SW_COMMON_DIR) +CXXFLAGS += -I$(ROOT_DIR)/sw -I$(THIRD_PARTY_DIR) + +SRCS := $(SRC_DIR)/main.cpp + +RTL_INCLUDE := -I$(ROOT_DIR)/sw -I$(RTL_DIR) -I$(RTL_DIR)/libs -I$(RTL_DIR)/tex -I$(SRC_DIR) +VL_FLAGS += -I$(ROOT_DIR)/hw + +TOP := VX_tex_lerp_top + +include ../common.mk diff --git a/hw/unittest/tex_lerp/VX_tex_lerp_top.sv b/hw/unittest/tex_lerp/VX_tex_lerp_top.sv new file mode 100644 index 0000000000..d6570a7173 --- /dev/null +++ b/hw/unittest/tex_lerp/VX_tex_lerp_top.sv @@ -0,0 +1,53 @@ +// Copyright © 2019-2023 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +`include "VX_platform.vh" + +// Both VX_tex_lerp forms the sampler instantiates, on shared inputs. +module VX_tex_lerp_top ( + input wire clk, + input wire reset, + input wire enable, + input wire [7:0] in1, + input wire [7:0] in2, + input wire [7:0] frac, + output wire [7:0] out_round, + output wire [7:0] out_trunc +); + VX_tex_lerp #( + .LATENCY (3), + .ROUND (1) + ) lerp_round ( + .clk (clk), + .reset (reset), + .enable (enable), + .in1 (in1), + .in2 (in2), + .frac (frac), + .out (out_round) + ); + + VX_tex_lerp #( + .LATENCY (3), + .ROUND (0) + ) lerp_trunc ( + .clk (clk), + .reset (reset), + .enable (enable), + .in1 (in1), + .in2 (in2), + .frac (frac), + .out (out_trunc) + ); + +endmodule diff --git a/hw/unittest/tex_lerp/main.cpp b/hw/unittest/tex_lerp/main.cpp new file mode 100644 index 0000000000..c264cb961e --- /dev/null +++ b/hw/unittest/tex_lerp/main.cpp @@ -0,0 +1,99 @@ +// VX_tex_lerp against the sampler's C model (vx_gfx_abi.h Lerp8888 for the +// bilinear taps, gfx_frag_tex.h TexLodLerp for the level blend) and against the +// exact Vulkan blend a*(1-f/256) + b*f/256, over every (in1, in2, frac). + +#include +#include +#include +#include +#include "VVX_tex_lerp_top.h" +#include "verilated.h" +#include + +static VVX_tex_lerp_top* dut; + +static void clock_cycle() { + dut->clk = 0; dut->eval(); + dut->clk = 1; dut->eval(); +} + +struct Vec { uint32_t a, b, f; }; + +static int errors = 0; + +static void check(const Vec& v, uint32_t rnd, uint32_t trn) { + using vortex::graphics::Lerp8888; + // the packed model, in both lanes it blends + uint32_t m_lo = Lerp8888(v.a, v.b, v.f) & 0xff; + uint32_t m_hi = (Lerp8888(v.a << 16, v.b << 16, v.f) >> 16) & 0xff; + uint32_t m_lod = gfx_tex::TexLodLerp(v.a, v.b, v.f) & 0xff; + // exact blend scaled by 256 + int32_t e256 = (int32_t)(v.a * (256 - v.f) + v.b * v.f); + bool rnd_exact = 2 * std::abs((int32_t)rnd * 256 - e256) <= 256; + bool trn_exact = (int32_t)trn * 256 <= e256 && e256 < ((int32_t)trn + 1) * 256; + if (rnd != m_lo || rnd != m_hi || trn != m_lod || !rnd_exact || !trn_exact) { + if (errors < 16) + std::printf("MISMATCH a=%u b=%u f=%u: rtl round=%u trunc=%u, model lo=%u hi=%u lod=%u, exact=%.4f\n", + v.a, v.b, v.f, rnd, trn, m_lo, m_hi, m_lod, e256 / 256.0); + ++errors; + } +} + +int main(int argc, char** argv) { + Verilated::commandArgs(argc, argv); + dut = new VVX_tex_lerp_top; + dut->reset = 1; + dut->enable = 1; + dut->in1 = dut->in2 = dut->frac = 0; + for (int i = 0; i < 4; ++i) clock_cycle(); + dut->reset = 0; + + const uint32_t LATENCY = 3; + std::deque inflight; + uint64_t checked = 0; + + // A stall cycle (enable low) must hold every stage; the results then still + // pair with their inputs after LATENCY enabled cycles. + srand(1); + auto step = [&](const Vec& v) { + if ((rand() % 16) == 0) { + dut->enable = 0; + clock_cycle(); + } + dut->enable = 1; + dut->in1 = v.a; dut->in2 = v.b; dut->frac = v.f; + inflight.push_back(v); + clock_cycle(); + if (inflight.size() == LATENCY) { + check(inflight.front(), dut->out_round, dut->out_trunc); + inflight.pop_front(); + ++checked; + } + }; + + // directed: pass-through at f=0, full-weight end of the range, the half + // point where round and truncate split + const Vec directed[] = { + {0, 255, 0}, {255, 0, 0}, {0, 255, 255}, {255, 0, 255}, {255, 255, 255}, + {8, 10, 128}, {0, 1, 128}, {1, 0, 128}, {200, 100, 1}, {100, 200, 254}, + }; + for (auto& v : directed) step(v); + + // exhaustive + for (uint32_t f = 0; f < 256; ++f) + for (uint32_t a = 0; a < 256; ++a) + for (uint32_t b = 0; b < 256; ++b) { + step({a, b, f}); + } + + for (uint32_t i = 1; i < LATENCY; ++i) step({0, 0, 0}); // drain + dut->final(); + delete dut; + + if (errors) { + std::printf("FAILED: %d mismatches over %llu vectors\n", errors, (unsigned long long)checked); + return 1; + } + std::printf("PASSED: %llu vectors\n", (unsigned long long)checked); + return 0; +} diff --git a/sim/simx/Makefile b/sim/simx/Makefile index 1198844b29..63581b4b8f 100644 --- a/sim/simx/Makefile +++ b/sim/simx/Makefile @@ -100,6 +100,7 @@ ifneq ($(filter -DVX_CFG_EXT_RTU_ENABLE, $(XCONFIGS)),) SRCS += $(SRC_DIR)/rtu/rtu_classifier.cpp SRCS += $(SRC_DIR)/rtu/rtu_walker.cpp SRCS += $(SRC_DIR)/rtu/rtu_memory.cpp + SRCS += $(SRC_DIR)/rtu/rtu_raylog.cpp endif # SST integration: build libvortex.so as the SST element library and diff --git a/sim/simx/alu_unit.cpp b/sim/simx/alu_unit.cpp index a866e2f1ca..a48fc3ebe5 100644 --- a/sim/simx/alu_unit.cpp +++ b/sim/simx/alu_unit.cpp @@ -345,14 +345,16 @@ void AluUnit::execute(instr_trace_t* trace) { // (which `tmask` aliases) to suppress source lanes, so source-lane liveness // must be judged against the pre-suppression mask, not the live one. auto active = tmask; - uint32_t last_tid = thread_start; - for (uint32_t t = thread_start; t < num_threads; ++t) - if (active.test(t)) last_tid = t; + uint32_t last_tid = (thread_last >= 0) ? uint32_t(thread_last) : 0; // WGATHER writes the FULL nibble (every non-source lane) regardless of // the active mask, so the gathered value is materialised even in masked // lanes; source lanes stay suppressed (keep their self value). Reads fall - // back to the last active lane when the nominal source is masked. - for (uint32_t t = thread_start; t < num_threads; ++t) { + // back to the last active lane when the nominal source is masked. Every + // lane of the warp is visited -- not just those from the first active + // one on: a warp whose low lanes are masked still gets its nibbles, which + // is what a consumer reading fixed lanes (the RTU's config in lanes 1-3) + // relies on. + for (uint32_t t = 0; t < num_threads; ++t) { if ((t & 0x3u) == src_offset) { trace->tmask.reset(t); // suppress writeback for source lane continue; diff --git a/sim/simx/processor.cpp b/sim/simx/processor.cpp index c160d1f829..0aa304656f 100644 --- a/sim/simx/processor.cpp +++ b/sim/simx/processor.cpp @@ -16,6 +16,9 @@ #include "core.h" #include "scheduler.h" #include +#ifdef VX_CFG_EXT_RTU_ENABLE +#include "rtu_raylog.h" +#endif #include #include @@ -199,6 +202,9 @@ ProcessorImpl::~ProcessorImpl() { void ProcessorImpl::attach_ram(RAM* ram) { ram_ = ram; memsim_->attach_ram(ram); +#ifdef VX_CFG_EXT_RTU_ENABLE + rtu::raylog::attach_ram(ram); +#endif } void ProcessorImpl::flush_caches() { diff --git a/sim/simx/rtu/rtu_bvh.h b/sim/simx/rtu/rtu_bvh.h index 6b0b7a5e9c..e684b98fc8 100644 --- a/sim/simx/rtu/rtu_bvh.h +++ b/sim/simx/rtu/rtu_bvh.h @@ -31,7 +31,7 @@ // from one decode point. // - Triangle stride (40 B) and TLAS instance stride (64 B) match the // flat-list constants in rtu_core.cpp so the existing intersection -// helpers (`ray_triangle`, `affine_inverse_transform_ray`) drop in +// helpers (`ray_triangle`, `world_to_object_ray`) drop in // unchanged. #ifndef _VX_RTU_BVH_H_ @@ -198,8 +198,11 @@ inline void decode_bvh6_node(const VxBvh6InternalNode* n, uint32_t count, // uint32 kind : bits 0..7 = kVxBvhKindLeafTri/Inst/Proc // bits 8..15 = prim_count // uint32 geometry_index : Vulkan gl_GeometryIndexEXT for this leaf -// uint32 flags : bit 0 = OPAQUE (all prims), bit 1 = forced -// non-opaque, bits 8..15 = SBT_IDX +// uint32 flags : LeafProc: bit 0 = OPAQUE (all prims), bit 1 = +// forced non-opaque, bits 8..15 = SBT_IDX. +// LeafTri / LeafInst: reserved, ignored (a +// triangle carries its own flag word). LeafInst +// also ignores geometry_index and prim_base. // uint32 prim_base : gl_PrimitiveID of this leaf's first // primitive; the walker reports // prim_base + within-leaf index so a @@ -254,7 +257,8 @@ constexpr uint32_t kVxBvhTriStride = 40; // from the SCENE root, not from a private BLAS base — gives us a // single base address for the whole TLAS+BLAS bundle). // -// floats 0..11 : 48 B object→world affine (3x4, row-major) +// floats 0..11 : 48 B world→object affine (3x4, row-major), +// the inverse of the instance transform // uint32 blas_root_off : 4 B byte offset to this instance's BLAS // root node from the scene-buffer base // uint32 custom_id : 4 B VK_INSTANCE_CUSTOM_INDEX_KHR diff --git a/sim/simx/rtu/rtu_core.cpp b/sim/simx/rtu/rtu_core.cpp index 75492a860c..8c2872fdb9 100644 --- a/sim/simx/rtu/rtu_core.cpp +++ b/sim/simx/rtu/rtu_core.cpp @@ -26,6 +26,7 @@ #include "rtu_isect.h" // BoxPe / TriPe pipeline depths #include "rtu_walker.h" // FlatWalker / Bvh4Walker #include "rtu_memory.h" // MemoryEngine +#include "rtu_raylog.h" #include "socket.h" #include "constants.h" #include "debug.h" @@ -340,9 +341,11 @@ class RtuCore::Impl { s.req = req; s.state = SlotState::READY; uint32_t first_active = uint32_t(-1); + s.lanes = {}; for (uint32_t t = 0; t < VX_CFG_NUM_THREADS; ++t) { if (s.req.tmask_bits & (1u << t)) { s.lanes[t].active = true; + s.lanes[t].walk_needed = true; if (first_active == uint32_t(-1)) first_active = t; } } @@ -354,6 +357,7 @@ class RtuCore::Impl { if (s.req.dir_z[first_active] < 0.f) sig |= 0x4; s.coh_signature = sig; } + if (raylog::enabled()) raylog::on_accept(this, idx, s.req); ch.pop(); ++perf_stats_.rays_issued; DT(3, "rtu-core accept: tag=" << s.req.tag << ", slot=" << idx); @@ -375,7 +379,16 @@ class RtuCore::Impl { LaneState& l = s.lanes[t]; if (!l.cb_pending) continue; uint32_t action = req.cb_action[t]; - if (action == VX_RT_CB_ACCEPT || action == VX_RT_CB_TERMINATE) { + const bool decides = (l.cb_type == VX_RT_CB_TYPE_ANYHIT + || l.cb_type == VX_RT_CB_TYPE_PROC); + const bool accept = (action == VX_RT_CB_ACCEPT || action == VX_RT_CB_TERMINATE); + // An intersection shader reports its own t, which need not be nearer than + // what the walk already committed: only a nearer hit inside the ray's + // interval replaces it. + const float new_t = (l.cb_type == VX_RT_CB_TYPE_PROC) ? req.cb_hit_t[t] : l.cand_t; + const bool commits = accept && (!decides + || (new_t >= s.req.tmin[t] && new_t < (l.hit ? l.hit_t : s.req.tmax[t]))); + if (commits) { l.hit = true; // A procedural (IS) accept commits the shader's own hit_t; a triangle // AHS keeps the geometric candidate t. Either way the hitAttribute the @@ -398,15 +411,25 @@ class RtuCore::Impl { } } // IGNORE leaves the committed hit alone; DONE means the CHS dispatcher has - // finished shading an already-committed hit. Traversal is - // single-yield-per-lane, so either way the lane is resolved and the slot - // drops to its terminal record once every yielding lane has answered. + // finished shading an already-committed hit. An any-hit / intersection + // verdict that does not end the ray resumes its walk above the decided + // candidate, so every candidate along the ray is offered in (t, key) + // order; TERMINATE, or an accept under TERMINATE_ON_FIRST_HIT, ends it. l.cb_pending = false; - bool any_pending = false; + const bool ends = action == VX_RT_CB_TERMINATE + || (commits && (s.req.flags[t] & VX_RT_FLAG_TERMINATE_ON_FIRST_HIT)); + if (decides && !ends) { + l.has_floor = true; + l.floor_t = l.cand_t; + l.floor_key = l.cand_key; + l.walk_needed = true; + } + bool any_pending = false, any_walk = false; for (auto const& ll : s.lanes) { - if (ll.cb_pending) { any_pending = true; break; } + any_pending |= ll.cb_pending; + any_walk |= ll.walk_needed; } - if (!any_pending) s.state = SlotState::RESP; + if (!any_pending) s.state = any_walk ? SlotState::READY : SlotState::RESP; } // Clear this warp's callback-in-flight gate so the next queued CB_YIELD for // the same warp (e.g. the second SBT group) can be emitted. @@ -436,7 +459,7 @@ class RtuCore::Impl { Slot& s = pool_.at(best); uint32_t need = 0; for (uint32_t t = 0; t < VX_CFG_NUM_THREADS; ++t) { - if (s.lanes[t].active) ++need; + if (s.lanes[t].active && s.lanes[t].walk_needed) ++need; } uint32_t avail = 0; for (const auto& cx : contexts_) { @@ -452,7 +475,8 @@ class RtuCore::Impl { uint32_t next_free = 0; for (uint32_t t = 0; t < VX_CFG_NUM_THREADS; ++t) { - if (!s.lanes[t].active) continue; + if (!s.lanes[t].active || !s.lanes[t].walk_needed) continue; + s.lanes[t].walk_needed = false; while (contexts_[next_free].valid) ++next_free; bind_context(next_free, best, t, s.req.scene_root[t]); ++next_free; @@ -561,6 +585,7 @@ class RtuCore::Impl { cx.next_state = CtxState::REQ; } else { s.lanes[cx.lane] = result; // carries cb_pending if the ray yielded + if (raylog::enabled()) raylog::on_walk_done(cx.lines); cx.next_state = CtxState::DONE; } cx.state = (cx.fsm_states || lat) ? CtxState::PE : cx.next_state; @@ -633,6 +658,7 @@ class RtuCore::Impl { const LaneState& l = s.lanes[t]; if (!l.active || !l.cb_pending) continue; any_cb = true; + if (raylog::enabled()) raylog::on_callback(this, i, t, l.cb_type); QueueEntry e{i, s.req.warp_id, uint8_t(t), l.sbt_idx, l.cb_type, l.cand_t, l.cand_u, l.cand_v, l.cand_prim, @@ -688,6 +714,7 @@ class RtuCore::Impl { // so it stays live until the WAIT that consumes the record calls // free_slot(). Until then it sits in EMITTED and is not re-sent. rsp.slot_idx = i; + if (raylog::enabled()) raylog::on_terminal(this, i, s.req, s.lanes); port.send(rsp); DT(3, "rtu-core complete: tag=" << s.req.tag << ", slot=" << i); s.state = SlotState::EMITTED; diff --git a/sim/simx/rtu/rtu_isect.cpp b/sim/simx/rtu/rtu_isect.cpp index d87ab078db..8f4d475954 100644 --- a/sim/simx/rtu/rtu_isect.cpp +++ b/sim/simx/rtu/rtu_isect.cpp @@ -12,7 +12,9 @@ // limitations under the License. #include "rtu_isect.h" +#include #include +#include namespace vortex { namespace rtu { @@ -21,90 +23,111 @@ bool ray_triangle(const float ro[3], const float rd[3], float tmin, float tmax, float& out_t, float& out_u, float& out_v, bool& out_back_facing) { - Vec3 O = { ro[0], ro[1], ro[2] }; - Vec3 D = { rd[0], rd[1], rd[2] }; - Vec3 V0 = { v0[0], v0[1], v0[2] }; - Vec3 V1 = { v1[0], v1[1], v1[2] }; - Vec3 V2 = { v2[0], v2[1], v2[2] }; - - Vec3 e1 = V1 - V0; - Vec3 e2 = V2 - V0; - Vec3 P = cross(D, e2); - float det = dot(e1, P); - constexpr float EPS = 1e-6f; - if (det > -EPS && det < EPS) return false; - float invDet = 1.0f / det; - Vec3 T = O - V0; - float u = dot(T, P) * invDet; - if (u < 0.f || u > 1.f) return false; - Vec3 Q = cross(T, e1); - float v = dot(D, Q) * invDet; - if (v < 0.f || u + v > 1.f) return false; - float t = dot(e2, Q) * invDet; - if (t < tmin || t > tmax) return false; + const float* vin[3] = { v0, v1, v2 }; + + // Watertight ray/triangle test (Woop, Benthin, Wald, JCGT 2013), F32 only: + // shear the triangle into the ray's frame so the ray runs along +z, then + // test the 2D edge functions. Each edge function is two rounded products and + // a rounded difference of the sheared vertices alone, so an edge shared by + // two triangles evaluates to exactly negated weights in both and no ray + // slips between them. Mirrors VX_rtu_tri_pe op for op. + const float ad[3] = { std::fabs(rd[0]), std::fabs(rd[1]), std::fabs(rd[2]) }; + int kz = (ad[0] >= ad[1]) ? ((ad[0] >= ad[2]) ? 0 : 2) + : ((ad[1] >= ad[2]) ? 1 : 2); + int kx = (kz + 1) % 3; + int ky = (kx + 1) % 3; + if (rd[kz] < 0.f) std::swap(kx, ky); // keep the winding + + const float sz = 1.0f / rd[kz]; + const float sx = rd[kx] * sz; + const float sy = rd[ky] * sz; + + float px[3], py[3], pz[3]; + for (int i = 0; i < 3; ++i) { + const float* q = vin[i]; + const float rx = q[kx] - ro[kx]; + const float ry = q[ky] - ro[ky]; + const float rz = q[kz] - ro[kz]; + px[i] = std::fma(-sx, rz, rx); + py[i] = std::fma(-sy, rz, ry); + pz[i] = sz * rz; + } + + // Edge functions: w[i] is the weight of vertex i. + const float w0 = px[2] * py[1] - py[2] * px[1]; + const float w1 = px[0] * py[2] - py[0] * px[2]; + const float w2 = px[1] * py[0] - py[1] * px[0]; + if ((w0 < 0.f || w1 < 0.f || w2 < 0.f) && (w0 > 0.f || w1 > 0.f || w2 > 0.f)) + return false; + + const float det = (w0 + w1) + w2; + // Reject only an edge-on or zero-area triangle: |det| scales with the + // triangle's area, so any epsilon would drop small triangles. + if (!(det != 0.f)) return false; + + const float T = std::fma(w2, pz[2], std::fma(w1, pz[1], w0 * pz[0])); + const float rcp = 1.0f / det; + const float t = T * rcp; + // Vulkan's ray interval for a triangle is open at both ends: an intersection + // candidate needs t_min < t < t_max (Ray Intersection Candidate + // Determination). + if (!(tmin < t && t < tmax)) return false; + out_t = t; - out_u = u; - out_v = v; + out_u = w1 * rcp; + out_v = w2 * rcp; + // det > 0: (v0, v1, v2) winds counter-clockwise as seen by the ray. out_back_facing = (det < 0.f); return true; } -bool ray_aabb_intersect(const float ro[3], const float rd[3], - const float mn[3], const float mx[3], - float tmin, float tmax, float& t_near) { - float tn = tmin, tf = tmax; +float ray_recip(float d) { + // A zero (or subnormal, which the PEs flush) component has no reciprocal; + // FLT_MAX keeps every slab product finite, so no slab is 0 * inf = NaN. + if (std::fabs(d) < FLT_MIN) return FLT_MAX; + return 1.0f / d; +} + +float quant_corner(uint8_t q, int8_t e) { + // Exact product; the PE flushes a subnormal result, inf past the range. + const float c = std::ldexp(float(q), e); + return (c < FLT_MIN) ? 0.f : c; +} + +void box_rel(const float base[3], const float mn[3], const float mx[3], + const float ro[3], float rel_mn[3], float rel_mx[3]) { for (int i = 0; i < 3; ++i) { - float inv = 1.0f / rd[i]; - float t0 = (mn[i] - ro[i]) * inv; - float t1 = (mx[i] - ro[i]) * inv; - if (t0 > t1) { float tmp = t0; t0 = t1; t1 = tmp; } - if (t0 > tn) tn = t0; - if (t1 < tf) tf = t1; - if (tn > tf) return false; + const float c = base[i] - ro[i]; + rel_mn[i] = mn[i] + c; + rel_mx[i] = mx[i] + c; } - t_near = tn; +} + +bool ray_box(const float rel_mn[3], const float rel_mx[3], const float inv[3], + float tmin, float tmax, float& t_near) { + // fmin/fmax drop a NaN operand, so a NaN slab (a NaN box or ray) drops out + // of the fold rather than poisoning it. + float lo = std::fmax(-INFINITY, tmin); + float hi = std::fmin(INFINITY, tmax); + for (int i = 0; i < 3; ++i) { + const float t0 = rel_mn[i] * inv[i]; + const float t1 = rel_mx[i] * inv[i]; + lo = std::fmax(lo, std::fmin(t0, t1)); + hi = std::fmin(hi, std::fmax(t0, t1)); + } + if (!(lo <= hi)) return false; + t_near = (lo == 0.f) ? 0.f : lo; return true; } -void affine_inverse_transform_ray(const float xform[12], - const float ro[3], const float rd[3], - float ro_out[3], float rd_out[3]) { - const float r00 = xform[0], r01 = xform[1], r02 = xform[2], tx = xform[3]; - const float r10 = xform[4], r11 = xform[5], r12 = xform[6], ty = xform[7]; - const float r20 = xform[8], r21 = xform[9], r22 = xform[10], tz = xform[11]; - - // det(R) by cofactor expansion along row 0. - float det = r00 * (r11 * r22 - r12 * r21) - - r01 * (r10 * r22 - r12 * r20) - + r02 * (r10 * r21 - r11 * r20); - if (det > -1e-9f && det < 1e-9f) { - // Singular — pass through (treat as identity). - for (int i = 0; i < 3; ++i) { ro_out[i] = ro[i]; rd_out[i] = rd[i]; } - return; +void world_to_object_ray(const float wto[12], + const float ro[3], const float rd[3], + float ro_out[3], float rd_out[3]) { + for (int i = 0; i < 3; ++i) { + const float* m = wto + 4 * i; + ro_out[i] = std::fma(ro[2], m[2], std::fma(ro[1], m[1], std::fma(ro[0], m[0], m[3]))); + rd_out[i] = std::fma(rd[2], m[2], std::fma(rd[1], m[1], rd[0] * m[0])); } - float inv_det = 1.f / det; - - // R^(-1) = (1/det) * adj(R). - float i00 = (r11 * r22 - r12 * r21) * inv_det; - float i01 = -(r01 * r22 - r02 * r21) * inv_det; - float i02 = (r01 * r12 - r02 * r11) * inv_det; - float i10 = -(r10 * r22 - r12 * r20) * inv_det; - float i11 = (r00 * r22 - r02 * r20) * inv_det; - float i12 = -(r00 * r12 - r02 * r10) * inv_det; - float i20 = (r10 * r21 - r11 * r20) * inv_det; - float i21 = -(r00 * r21 - r01 * r20) * inv_det; - float i22 = (r00 * r11 - r01 * r10) * inv_det; - - // ro_obj = R^(-1) * (ro - t). - float dx = ro[0] - tx, dy = ro[1] - ty, dz = ro[2] - tz; - ro_out[0] = i00 * dx + i01 * dy + i02 * dz; - ro_out[1] = i10 * dx + i11 * dy + i12 * dz; - ro_out[2] = i20 * dx + i21 * dy + i22 * dz; - - // rd_obj = R^(-1) * rd. - rd_out[0] = i00 * rd[0] + i01 * rd[1] + i02 * rd[2]; - rd_out[1] = i10 * rd[0] + i11 * rd[1] + i12 * rd[2]; - rd_out[2] = i20 * rd[0] + i21 * rd[1] + i22 * rd[2]; } // PE cost model. There is ONE box PE and ONE tri PE per RtuCore, each streaming @@ -119,8 +142,9 @@ uint32_t BoxPe::pipe_depth() { } uint32_t TriPe::pipe_depth() { - // 8 FMA stages + 1 reciprocal + 2 = 91. - return 8 * kRtuLatencyFma + kRtuFdivLat + 2; + // input select + 1/dir[kz] + shear scale + shear + 2 edge stages + det + + // 1/det + t/u/v scale + verdict (VX_rtu_tri_pe). + return 2 + 2 * kRtuFdivLat + 7 * kRtuLatencyFma; } }} // namespace vortex::rtu diff --git a/sim/simx/rtu/rtu_isect.h b/sim/simx/rtu/rtu_isect.h index 4d2de405a2..d03be7727a 100644 --- a/sim/simx/rtu/rtu_isect.h +++ b/sim/simx/rtu/rtu_isect.h @@ -30,12 +30,14 @@ namespace vortex { namespace rtu { // ──────────────────────────────────────────────────────────────────── -// Möller-Trumbore ray-triangle intersection. +// Watertight ray-triangle intersection (Woop, Benthin, Wald, JCGT 2013), F32. // // out_back_facing reports whether the ray hit the back side of the // triangle's geometric normal (ray-flag face culling). Convention: // triangle front face is the side from which (v0, v1, v2) appear CCW. // Equivalently, det > 0 ↔ ray hits the front face. +// A hit needs tmin < t < tmax (Vulkan's open triangle interval); the walker +// passes the committed hit's t as tmax. // ──────────────────────────────────────────────────────────────────── bool ray_triangle(const float ro[3], const float rd[3], const float v0[3], const float v1[3], const float v2[3], @@ -44,48 +46,48 @@ bool ray_triangle(const float ro[3], const float rd[3], bool& out_back_facing); // ──────────────────────────────────────────────────────────────────── -// Ray-vs-AABB slab test. Returns true if the ray's [tmin, tmax] -// interval overlaps the AABB; t_near is the entry parameter (clamped -// to tmin) used by the BVH4 walker to prune descent order. +// Ray-vs-AABB slab test, as VX_rtu_recip + VX_rtu_box_pe compute it. // -// Assumes well-conditioned rays (no axis-aligned ray with zero -// direction component). A robust branchless ±inf variant is a later -// refinement. +// ray_recip 1/d per direction component, FLT_MAX for a zero one +// quant_corner a quantized child corner q * 2^e (exact; FTZ) +// box_rel box corners relative to the ray origin: m + (base - ro), +// base = the node origin (+0 for a raw procedural box) +// ray_box slabs rel * inv, culled against [tmin, tmax]: hit iff +// max(tmin, entry) <= min(tmax, exit); t_near = that max, +// the entry distance the walker orders children by // ──────────────────────────────────────────────────────────────────── -bool ray_aabb_intersect(const float ro[3], const float rd[3], - const float mn[3], const float mx[3], - float tmin, float tmax, float& t_near); +float ray_recip(float d); +float quant_corner(uint8_t q, int8_t e); +void box_rel(const float base[3], const float mn[3], const float mx[3], + const float ro[3], float rel_mn[3], float rel_mx[3]); +bool ray_box(const float rel_mn[3], const float rel_mx[3], const float inv[3], + float tmin, float tmax, float& t_near); // ──────────────────────────────────────────────────────────────────── -// Apply the inverse of a 3x4 row-major affine to a ray, producing the -// object-space ray. Used by the BVH4 walker on LeafInst descent to -// convert world→object space. Mirrors the hardware XFORM unit -// (latency = 3 cycles). +// Bring a world ray into an instance's object space with the instance +// record's world→object 3x4 row-major matrix m, as FMA chains: // -// xform = [r00 r01 r02 tx | r10 r11 r12 ty | r20 r21 r22 tz] -// ro_obj = R^(-1) * (ro_world - t) -// rd_obj = R^(-1) * rd_world +// ro_obj[i] = fma(ro.z, m[i][2], fma(ro.y, m[i][1], fma(ro.x, m[i][0], m[i][3]))) +// rd_obj[i] = fma(rd.z, m[i][2], fma(rd.y, m[i][1], rd.x * m[i][0])) // -// For pure rotation+translation (det(R) == ±1) the t parameter is -// preserved across spaces, so the BLAS-reported hit_t is also the -// world hit_t. Non-uniform scale would require renormalising hit_t; -// out of scope. +// The direction is not renormalised, so t is the same in both spaces. +// Mirrors VX_rtu_xform bit for bit. // ──────────────────────────────────────────────────────────────────── -void affine_inverse_transform_ray(const float xform[12], - const float ro[3], const float rd[3], - float ro_out[3], float rd_out[3]); +void world_to_object_ray(const float wto[12], + const float ro[3], const float rd[3], + float ro_out[3], float rd_out[3]); // ════════════════════════════════════════════════════════════════════ // The intersection coprocessors — pipelined BoxPe / TriPe. // ════════════════════════════════════════════════════════════════════ // // BoxPe (ray-vs-AABB): ONE PE, 1 box/cycle, 31-cycle pipeline depth. -// TriPe (ray-vs-tri): ONE PE, 1 tri/cycle, 91-cycle pipeline depth. +// TriPe (ray-vs-tri): ONE PE, 1 tri/cycle, 99-cycle pipeline depth. // // Both are shared across the whole context array, so the issue slots are handed // out by the orchestrator one per cycle and the contention is modelled, not // assumed away. The math itself is done synchronously by the scalar -// ray_triangle / ray_aabb_intersect helpers above; these classes contribute only +// ray_triangle / ray_box helpers above; these classes contribute only // the drain behind the last test entered. class BoxPe { public: diff --git a/sim/simx/rtu/rtu_raylog.cpp b/sim/simx/rtu/rtu_raylog.cpp new file mode 100644 index 0000000000..e0a6b5e0d5 --- /dev/null +++ b/sim/simx/rtu/rtu_raylog.cpp @@ -0,0 +1,237 @@ +// Copyright © 2019-2023 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "rtu_raylog.h" +#include +#include +#include +#include +#include +#include +#include +#include "mem.h" + +namespace vortex { namespace rtu { namespace raylog { + +namespace { + +template uint32_t bits(T v) { + static_assert(sizeof(T) == 4, "32-bit field"); + uint32_t u; + std::memcpy(&u, &v, 4); + return u; +} + +class Logger { +public: + Logger() { + const char* path = std::getenv("VX_RTU_RAYLOG"); + if (path == nullptr || path[0] == '\0') return; + fp_ = std::fopen(path, "wb"); + if (fp_ == nullptr) { + std::fprintf(stderr, "[rtu-raylog] cannot open %s\n", path); + return; + } + static char buf[1 << 20]; + std::setvbuf(fp_, buf, _IOFBF, sizeof(buf)); + if (const char* s = std::getenv("VX_RTU_RAYLOG_MAX")) max_rays_ = std::strtoull(s, nullptr, 0); + if (const char* s = std::getenv("VX_RTU_RAYLOG_EVERY")) every_ = std::max(1ull, std::strtoull(s, nullptr, 0)); + RaylogHeader h{}; + h.magic = kMagic; + h.version = kVersion; + h.num_threads = VX_CFG_NUM_THREADS; + h.line_bytes = VX_CFG_MEM_BLOCK_SIZE; + h.bvh_width = VX_CFG_RTU_BVH_WIDTH; + h.ray_bytes = sizeof(RaylogRay); + std::fwrite(&h, sizeof(h), 1, fp_); + } + + ~Logger() { + if (fp_ == nullptr) return; + std::fclose(fp_); + std::fprintf(stderr, "[rtu-raylog] rays=%llu logged=%llu replayable=%llu lines=%zu epochs=%u\n", + (unsigned long long)seen_, (unsigned long long)logged_, + (unsigned long long)replayable_, image_.size(), epoch_ + 1); + } + + bool on() const { return fp_ != nullptr; } + + void set_ram(const RAM* ram) { + std::lock_guard g(mu_); + ram_ = ram; + } + + void accept(const void* owner, uint32_t slot, const RtuReq& req) { + std::lock_guard g(mu_); + cb_[{owner, slot}].fill(0); + for (uint32_t t = 0; t < VX_CFG_NUM_THREADS; ++t) { + if (req.tmask_bits & (1u << t)) snapshot_scene(req.scene_root[t]); + } + } + + void callback(const void* owner, uint32_t slot, uint32_t lane, uint32_t cb_type) { + std::lock_guard g(mu_); + cb_[{owner, slot}].at(lane) |= 1u << (cb_type & 7); + } + + void walk_done(const std::unordered_map& lines) { + std::lock_guard g(mu_); + if (full()) return; + for (const auto& kv : lines) { + auto it = image_.find(kv.first); + if (it != image_.end()) { + if (it->second == kv.second) continue; + // The device rewrote a line the scene image already holds: the rays + // from here on see a different memory image. + ++epoch_; + it->second = kv.second; + } else { + image_.emplace(kv.first, kv.second); + } + RaylogLine r{}; + r.type = REC_LINE; + r.epoch = epoch_; + r.addr = kv.first; + std::memcpy(r.data, kv.second.data(), sizeof(r.data)); + std::fwrite(&r, sizeof(r), 1, fp_); + } + } + + void terminal(const void* owner, uint32_t slot, const RtuReq& req, + const std::array& lanes) { + std::lock_guard g(mu_); + auto& cbm = cb_[{owner, slot}]; + for (uint32_t t = 0; t < VX_CFG_NUM_THREADS; ++t) { + const LaneState& l = lanes[t]; + if (!l.active) continue; + uint64_t seq = seen_++; + if (full() || (seq % every_) != 0) continue; + RaylogRay r{}; + r.type = REC_RAY; + const uint32_t cbs = cbm[t]; + const bool decided_by_shader = cbs & ((1u << VX_RT_CB_TYPE_ANYHIT) | (1u << VX_RT_CB_TYPE_PROC)); + r.info = (cbs << kInfoCbShift) | (decided_by_shader ? 0u : kInfoReplayable); + r.seq = uint32_t(seq); + r.epoch = epoch_; + r.scene_root = req.scene_root[t]; + r.ray_flags = req.flags[t]; + r.cull_mask = req.cull_mask[t]; + r.warp_lane = t | (req.warp_id << 8) | (slot << 16); + r.origin[0] = bits(req.origin_x[t]); r.origin[1] = bits(req.origin_y[t]); r.origin[2] = bits(req.origin_z[t]); + r.dir[0] = bits(req.dir_x[t]); r.dir[1] = bits(req.dir_y[t]); r.dir[2] = bits(req.dir_z[t]); + r.tmin = bits(req.tmin[t]); + r.tmax = bits(req.tmax[t]); + if (l.hit) { + r.status = VX_RT_STS_DONE_HIT; + r.hit_t = bits(l.hit_t); + r.hit_u = bits(l.hit_u); + r.hit_v = bits(l.hit_v); + r.prim = l.hit_prim; + r.geom = l.hit_geometry; + r.inst_id = l.hit_instance_id; + r.inst_custom = l.hit_instance_custom; + r.hit_attr = l.hit_attr; + } else { + r.status = VX_RT_STS_DONE_MISS; + } + std::fwrite(&r, sizeof(r), 1, fp_); + ++logged_; + if (!decided_by_shader) ++replayable_; + } + } + +private: + bool full() const { return logged_ >= max_rays_; } + + // Instance records sit below the scene root, one stride per instance id. + static constexpr uint64_t kInstTableSpan = 64 * 1024; + static constexpr uint64_t kMaxSceneBytes = 1ull << 30; + + void snapshot_scene(uint32_t root) { + if (ram_ == nullptr || full() || !snapped_.insert(root).second) return; + const RAM& ram = *ram_; + uint32_t scene_bytes = 0; + for (int i = 0; i < 4; ++i) scene_bytes |= uint32_t(ram[root + 8 + i]) << (8 * i); + const uint64_t lo = (root > kInstTableSpan ? root - kInstTableSpan : 0) & kRtuLineMask; + const uint64_t hi = uint64_t(root) + std::min(scene_bytes, kMaxSceneBytes); + uint64_t added = 0; + for (uint64_t a = lo; a < hi; a += VX_CFG_MEM_BLOCK_SIZE) { + if (image_.count(a)) continue; + LineBuf line; + bool nonzero = false; + for (uint32_t i = 0; i < VX_CFG_MEM_BLOCK_SIZE; ++i) { + line[i] = ram[a + i]; + nonzero |= line[i] != 0; + } + if (!nonzero) continue; + image_.emplace(a, line); + RaylogLine r{}; + r.type = REC_LINE; + r.epoch = epoch_; + r.addr = a; + std::memcpy(r.data, line.data(), sizeof(r.data)); + std::fwrite(&r, sizeof(r), 1, fp_); + ++added; + } + std::fprintf(stderr, "[rtu-raylog] scene 0x%x: %u bytes, snapshot %llu lines\n", + root, scene_bytes, (unsigned long long)added); + } + + std::mutex mu_; + const RAM* ram_ = nullptr; + std::unordered_set snapped_; + FILE* fp_ = nullptr; + unsigned long long max_rays_ = 4ull << 20; + unsigned long long every_ = 1; + uint64_t seen_ = 0; + uint64_t logged_ = 0; + uint64_t replayable_ = 0; + uint32_t epoch_ = 0; + std::unordered_map image_; + std::map, std::array> cb_; +}; + +Logger& logger() { + static Logger inst; + return inst; +} + +} // namespace + +bool enabled() { + static const bool on = logger().on(); + return on; +} + +void attach_ram(const RAM* ram) { + if (enabled()) logger().set_ram(ram); +} + +void on_accept(const void* owner, uint32_t slot, const RtuReq& req) { + logger().accept(owner, slot, req); +} + +void on_callback(const void* owner, uint32_t slot, uint32_t lane, uint32_t cb_type) { + logger().callback(owner, slot, lane, cb_type); +} + +void on_walk_done(const std::unordered_map& lines) { + logger().walk_done(lines); +} + +void on_terminal(const void* owner, uint32_t slot, const RtuReq& req, + const std::array& lanes) { + logger().terminal(owner, slot, req, lanes); +} + +}}} // namespace vortex::rtu::raylog diff --git a/sim/simx/rtu/rtu_raylog.h b/sim/simx/rtu/rtu_raylog.h new file mode 100644 index 0000000000..9f64b62eb1 --- /dev/null +++ b/sim/simx/rtu/rtu_raylog.h @@ -0,0 +1,122 @@ +// Copyright © 2019-2023 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// PRISM RTU — ray log (opt-in, VX_RTU_RAYLOG=). +// +// Records every traced ray with its terminal result, plus every scene line a +// walk read, so a ray can be re-traced outside the workload that issued it +// (tests/raytracing/rt_replay) and compared bit-exactly against another model. +// +// Stream layout (little-endian): a RaylogHeader, then records tagged by their +// first word. A LINE record is emitted the first time a walk reads a line, and +// again, with the epoch bumped, if a later walk reads different bytes there; +// every RAY record names the epoch whose memory image it was traced against. +// +// Optional knobs: +// VX_RTU_RAYLOG_MAX= stop after n RAY records (default 4M) +// VX_RTU_RAYLOG_EVERY= keep one terminal ray in n (default 1) +// +// The lines a SimX walk reads are not all another model may read: one that +// culls differently descends into nodes SimX never fetched. So the first trace +// against a scene also snapshots its whole image from device memory — the +// header's scene_bytes after the root, and the instance table packed below it. + +#ifndef _VX_RTU_RAYLOG_H_ +#define _VX_RTU_RAYLOG_H_ + +#include +#include +#include +#include "rtu_types.h" + +namespace vortex { +class RAM; +namespace rtu { namespace raylog { + +constexpr uint32_t kMagic = 0x4C525856; // "VXRL" +constexpr uint32_t kVersion = 1; + +enum RecType : uint32_t { + REC_LINE = 1, + REC_RAY = 2, +}; + +// RaylogRay::info bits. +constexpr uint32_t kInfoCbShift = 0; // bit (cb_type) set per callback yielded +constexpr uint32_t kInfoReplayable = 1u << 8; // no any-hit / intersection callback decided it + +struct RaylogHeader { + uint32_t magic; + uint32_t version; + uint32_t num_threads; + uint32_t line_bytes; + uint32_t bvh_width; + uint32_t ray_bytes; + uint32_t reserved[2]; +}; + +struct RaylogLine { + uint32_t type; // REC_LINE + uint32_t epoch; + uint64_t addr; + uint8_t data[VX_CFG_MEM_BLOCK_SIZE]; +}; + +struct RaylogRay { + uint32_t type; // REC_RAY + uint32_t info; + uint32_t seq; + uint32_t epoch; + uint32_t scene_root; + uint32_t ray_flags; + uint32_t cull_mask; + uint32_t warp_lane; // lane | warp << 8 | slot << 16 + uint32_t origin[3]; // float bits + uint32_t dir[3]; + uint32_t tmin; + uint32_t tmax; + uint32_t status; // VX_RT_STS_DONE_HIT / DONE_MISS + uint32_t hit_t; // float bits (0 on a miss) + uint32_t hit_u; + uint32_t hit_v; + uint32_t prim; + uint32_t geom; // geometry index | VX_RT_HIT_BACK_FACING + uint32_t inst_id; + uint32_t inst_custom; + uint32_t hit_attr; + uint32_t reserved[3]; +}; +static_assert(sizeof(RaylogRay) == 112, "RaylogRay layout"); + +bool enabled(); + +// The device memory the scene snapshots read from. +void attach_ram(const RAM* ram); + +// A trace landed in `slot` of the RTU at `owner`: forget the callbacks the +// slot's previous trace raised, and snapshot any scene it is the first to name. +void on_accept(const void* owner, uint32_t slot, const RtuReq& req); + +// A lane of `slot` yielded a callback of `cb_type`. +void on_callback(const void* owner, uint32_t slot, uint32_t lane, uint32_t cb_type); + +// A walk completed against `lines`. +void on_walk_done(const std::unordered_map& lines); + +// `slot` emitted its terminal record. +void on_terminal(const void* owner, uint32_t slot, const RtuReq& req, + const std::array& lanes); + +}}} // namespace vortex::rtu::raylog + +#endif // _VX_RTU_RAYLOG_H_ diff --git a/sim/simx/rtu/rtu_types.h b/sim/simx/rtu/rtu_types.h index 8d8858bce2..07d0ecb695 100644 --- a/sim/simx/rtu/rtu_types.h +++ b/sim/simx/rtu/rtu_types.h @@ -163,8 +163,14 @@ struct RtuRsp { uint32_t slot_idx = 0; RtuRsp() = default; + // A lane without a hit reads back its own ray: t = t_max (the committed t a + // ray query reports with no hit, as the Vulkan reference does) and the world + // ray as its object ray. RtuRsp(const RtuReq& req) : uuid(req.uuid), tag(req.tag), + hit_t(req.tmax), + obj_o_x(req.origin_x), obj_o_y(req.origin_y), obj_o_z(req.origin_z), + obj_d_x(req.dir_x), obj_d_y(req.dir_y), obj_d_z(req.dir_z), trace(req.trace), block_id(req.block_id), warp_id(req.warp_id), slot_idx(req.slot_idx) {} @@ -298,14 +304,14 @@ constexpr uint32_t kRtuImageStatesPerRay = 1; // scene header constexpr uint32_t kRtuSetupLatency = 17; // reciprocal pipe depth constexpr uint32_t kRtuFdivLat = 17; // reciprocal pipe depth constexpr uint32_t kRtuLatencyFma = 9; // FMA pipe depth -// Per-instance transform latency = 4 * FMA pipe depth = 36: an (ro-t) subtract -// at FMA depth, then a 3-deep dot product. Charged per TLAS instance descent -// in the SimX cost model. -constexpr uint32_t kRtuXformLatency = 36; // 4 * FMA pipe depth +// Per-instance transform latency = 3 * FMA pipe depth = 27: three dependent +// FMAs (VX_rtu_xform). Charged per TLAS instance descent in the SimX cost +// model. +constexpr uint32_t kRtuXformLatency = 27; // 3 * FMA pipe depth // TLAS instance record (64 B). Lives inline after the scene header for // "TLAS + inline BLAS" layout. -// floats 0..11 = 3x4 affine transform (rows r0|r1|r2), object→world +// floats 0..11 = 3x4 affine transform (rows r0|r1|r2), world→object // uint32 [48..52) = blas_byte_offset // uint32 [52..56) = custom_id (Vulkan VK_INSTANCE_CUSTOM_INDEX_KHR) // uint32 [56..60) = cull_mask (low byte = Vulkan instance mask; @@ -436,6 +442,14 @@ struct LaneState { float cand_obj_d[3] = {0.f, 0.f, 0.f}; uint32_t hit_instance_id = 0; uint32_t hit_instance_custom = 0; + // Multi-candidate traversal. A verdict that does not end the ray resumes its + // walk above the decided candidate's (t, key); `walk_needed` marks the lanes + // the next promote binds contexts to. + uint64_t cand_key = 0; + bool has_floor = false; + float floor_t = 0.f; + uint64_t floor_key = 0; + bool walk_needed = false; }; struct Slot { diff --git a/sim/simx/rtu/rtu_walker.cpp b/sim/simx/rtu/rtu_walker.cpp index efb5724c21..3164e9f9d7 100644 --- a/sim/simx/rtu/rtu_walker.cpp +++ b/sim/simx/rtu/rtu_walker.cpp @@ -23,8 +23,8 @@ #include "rtu_types.h" // RtuReq, SceneView, LaneState, PerfStats, // scene-format constants #include "rtu_bvh.h" // CW-BVH node/leaf/instance layouts -#include "rtu_isect.h" // ray_triangle, ray_aabb_intersect, - // affine_inverse_transform_ray +#include "rtu_isect.h" // ray_triangle, ray_box, + // world_to_object_ray #include "rtu_classifier.h" // classify_tri_hit, finalise_lane namespace vortex { namespace rtu { @@ -62,18 +62,6 @@ void read_scene_bytes(SceneView& sv, uint32_t off, uint32_t len, uint8_t* out) { } } -// CW-BVH: reconstruct a child AABB from quantized representation. -// real = origin + qaabb * 2^exp (per axis) -inline void reconstruct_child_aabb(const float origin[3], const int8_t exp[3], - const uint8_t qmin[3], const uint8_t qmax[3], - float out_mn[3], float out_mx[3]) { - for (int i = 0; i < 3; ++i) { - float scale = std::ldexp(1.0f, exp[i]); - out_mn[i] = origin[i] + static_cast(qmin[i]) * scale; - out_mx[i] = origin[i] + static_cast(qmax[i]) * scale; - } -} - // Copy a 3-vector (object-space ray capture helper). inline void vcopy3(float dst[3], const float src[3]) { dst[0] = src[0]; dst[1] = src[1]; dst[2] = src[2]; @@ -105,8 +93,37 @@ struct WalkCtx { // at the top level (no instance). float best_obj_o[3], best_obj_d[3]; float yield_obj_o[3], yield_obj_d[3]; + // Candidates are offered one at a time in ascending (t, key) order, key = + // (instance_id << 32) | record offset -- unique per primitive per instance. A + // resumed walk skips everything at or below the last decided candidate's key. + uint64_t yield_key; + bool has_floor; + float floor_t; + uint64_t floor_key; }; +inline uint64_t cand_key(uint32_t instance_id, uint32_t record_off) { + return (uint64_t(instance_id) << 32) | record_off; +} + +// Whether a candidate at (t, key) replaces the pending one: nearer than the +// committed hit, above the resume floor, and first in (t, key) order. +inline bool cand_takes(const WalkCtx& ctx, float t, uint64_t key) { + if (!(t < ctx.best_t)) return false; + if (ctx.has_floor + && (t < ctx.floor_t || (t == ctx.floor_t && key <= ctx.floor_key))) + return false; + if (!ctx.yield_pending) return true; + return t < ctx.yield_t || (t == ctx.yield_t && key < ctx.yield_key); +} + +// The geometry word's back-facing bit, after the instance's FLIP_FACING: +// what gl_HitKindEXT reports. +inline uint32_t hit_facing_bit(bool back_facing, uint32_t inst_flags) { + if (inst_flags & kRtuInstanceFlagTriFlip) back_facing = !back_facing; + return back_facing ? VX_RT_HIT_BACK_FACING : 0u; +} + // Depth-first walker for one BVH sub-tree under the supplied (object-space) // ray. Recurses on LeafInst so each instance's BLAS gets walked with its // transformed ray. ctx accumulates hits/yields across the whole call tree. @@ -122,6 +139,9 @@ void walk_bvh4_subtree(SceneView& sv, uint32_t root_off, uint32_t instance_id, uint32_t custom_id, uint32_t inst_flags, WalkCtx& ctx, PerfStats& perf) { + // The ray setup's reciprocals, once per (object-space) ray. + const float inv[3] = { ray_recip(rd[0]), ray_recip(rd[1]), ray_recip(rd[2]) }; + auto visit_leaf_tri = [&](uint32_t leaf_off, uint32_t count) { uint8_t hdr_buf[kVxBvhLeafHeaderBytes]; read_scene_bytes(sv, leaf_off, sizeof(hdr_buf), hdr_buf); @@ -145,43 +165,47 @@ void walk_bvh4_subtree(SceneView& sv, float t_hit = 0.f, u = 0.f, v = 0.f; bool back_facing = false; ++perf.bvh_tri_tests; + // The interval shrinks to the committed hit: a hit is only reported + // strictly nearer than it, so on equal t the first one found stays. if (!ray_triangle(ro, rd, &tri[0], &tri[3], &tri[6], - ctx.tmin, ctx.tmax, + ctx.tmin, ctx.best_t, t_hit, u, v, back_facing)) { continue; } TriClassify cls = classify_tri_hit(ctx.ray_flags, tri_flags, inst_flags, back_facing); + const uint32_t hit_geom = (leaf_geom & VX_RT_HIT_GEOMETRY_MASK) + | hit_facing_bit(back_facing, inst_flags); if (cls.action == TriAction::Ignore) continue; if (cls.action == TriAction::Commit) { - if (t_hit < ctx.best_t) { - ctx.best_t = t_hit; ctx.best_u = u; ctx.best_v = v; - ctx.best_prim = leaf_prim_base + i; - ctx.best_instance = instance_id; - ctx.best_custom = custom_id; - ctx.best_geom = leaf_geom; - ctx.any_hit = true; - vcopy3(ctx.best_obj_o, ro); // object-space ray of this BLAS - vcopy3(ctx.best_obj_d, rd); - if (ctx.yield_pending && ctx.yield_t >= ctx.best_t) { - ctx.yield_pending = false; - ctx.yield_t = ctx.tmax; - } - if (cls.terminate_on_first_hit) { - ctx.terminated = true; - return; - } + ctx.best_t = t_hit; ctx.best_u = u; ctx.best_v = v; + ctx.best_prim = leaf_prim_base + i; + ctx.best_instance = instance_id; + ctx.best_custom = custom_id; + ctx.best_geom = hit_geom; + ctx.any_hit = true; + vcopy3(ctx.best_obj_o, ro); // object-space ray of this BLAS + vcopy3(ctx.best_obj_d, rd); + if (ctx.yield_pending && ctx.yield_t >= ctx.best_t) { + ctx.yield_pending = false; + ctx.yield_t = ctx.tmax; + } + if (cls.terminate_on_first_hit) { + ctx.terminated = true; + return; } } else { // TriAction::Yield - if (t_hit < ctx.best_t && t_hit < ctx.yield_t) { + uint64_t key = cand_key(instance_id, tris_off + i * kVxBvhTriStride); + if (cand_takes(ctx, t_hit, key)) { ctx.yield_pending = true; + ctx.yield_key = key; ctx.yield_t = t_hit; ctx.yield_u = u; ctx.yield_v = v; ctx.yield_prim = leaf_prim_base + i; ctx.yield_instance = instance_id; ctx.yield_custom = custom_id; - ctx.yield_geom = leaf_geom; + ctx.yield_geom = hit_geom; ctx.yield_sbt = cls.yield_sbt_idx; ctx.yield_cb_type = cls.yield_cb_type; vcopy3(ctx.yield_obj_o, ro); // object-space ray for AHS/IS @@ -212,21 +236,25 @@ void walk_bvh4_subtree(SceneView& sv, if (sv.miss) return; const VxBvhProcAabb* rec = reinterpret_cast(rec_buf); - float t_near = 0.f; + static const float kRawBase[3] = { 0.f, 0.f, 0.f }; + float rel_mn[3], rel_mx[3], t_near = 0.f; + box_rel(kRawBase, rec->aabb_min, rec->aabb_max, ro, rel_mn, rel_mx); ++perf.bvh_box_tests; - if (!ray_aabb_intersect(ro, rd, rec->aabb_min, rec->aabb_max, - ctx.tmin, ctx.best_t, t_near)) { + if (!ray_box(rel_mn, rel_mx, inv, ctx.tmin, ctx.best_t, t_near)) { continue; } // Procedural primitives are inherently non-opaque (the IS decides the // hit), so always stage an IS yield for the closest candidate. - if (t_near < ctx.best_t && t_near < ctx.yield_t) { + uint64_t key = cand_key(instance_id, + aabbs_off + i * uint32_t(sizeof(VxBvhProcAabb))); + if (cand_takes(ctx, t_near, key)) { ctx.yield_pending = true; + ctx.yield_key = key; ctx.yield_t = t_near; ctx.yield_u = 0.f; ctx.yield_v = 0.f; - ctx.yield_prim = i; + ctx.yield_prim = hdr->prim_base + i; // gl_PrimitiveID, as for LEAF_TRI ctx.yield_instance = instance_id; ctx.yield_custom = custom_id; - ctx.yield_geom = hdr->geometry_index; + ctx.yield_geom = hdr->geometry_index & VX_RT_HIT_GEOMETRY_MASK; ctx.yield_sbt = leaf_sbt; ctx.yield_cb_type = VX_RT_CB_TYPE_PROC; vcopy3(ctx.yield_obj_o, ro); @@ -236,6 +264,9 @@ void walk_bvh4_subtree(SceneView& sv, }; auto visit_leaf_inst = [&](uint32_t leaf_off, uint32_t count) { + uint8_t hdr_buf[kVxBvhLeafHeaderBytes]; + read_scene_bytes(sv, leaf_off, sizeof(hdr_buf), hdr_buf); + if (sv.miss) return; uint32_t insts_off = leaf_off + kVxBvhLeafHeaderBytes; for (uint32_t i = 0; i < count; ++i) { uint8_t inst_buf[kVxBvhInstanceStride]; @@ -252,7 +283,7 @@ void walk_bvh4_subtree(SceneView& sv, uint32_t inst_flags2 = (inst->cull_mask >> kRtuInstanceFlagsShift) & kRtuInstanceFlagsMask; float obj_ro[3], obj_rd[3]; - affine_inverse_transform_ray(inst->xform, ro, rd, obj_ro, obj_rd); + world_to_object_ray(inst->xform, ro, rd, obj_ro, obj_rd); ++perf.bvh_instance_descents; walk_bvh4_subtree(sv, obj_ro, obj_rd, inst->blas_root_byte_offset, @@ -335,14 +366,15 @@ void walk_bvh4_subtree(SceneView& sv, uint32_t off_word = nv.child_offsets[i]; uint32_t child_off = off_word & kVxBvhChildOffsetMask; if (off_word == kVxBvhChildEmpty) continue; - float mn[3], mx[3]; - reconstruct_child_aabb(nv.origin, nv.exp, - nv.qaabb_min[i], nv.qaabb_max[i], - mn, mx); + float mn[3], mx[3], rel_mn[3], rel_mx[3]; + for (int a = 0; a < 3; ++a) { + mn[a] = quant_corner(nv.qaabb_min[i][a], nv.exp[a]); + mx[a] = quant_corner(nv.qaabb_max[i][a], nv.exp[a]); + } + box_rel(nv.origin, mn, mx, ro, rel_mn, rel_mx); float t_near = 0.f; ++perf.bvh_box_tests; - if (!ray_aabb_intersect(ro, rd, mn, mx, - ctx.tmin, ctx.best_t, t_near)) { + if (!ray_box(rel_mn, rel_mx, inv, ctx.tmin, ctx.best_t, t_near)) { continue; } hits[hit_count++] = { child_off, t_near }; @@ -418,6 +450,7 @@ bool emit_lane_result(const RtuReq& req, LaneState& l, uint32_t t, l.cand_prim = ctx.yield_prim; l.cand_instance = ctx.yield_instance; l.cand_custom = ctx.yield_custom; + l.cand_key = ctx.yield_key; return true; case LaneAction::YieldChs: l.cb_pending = true; @@ -448,9 +481,10 @@ bool emit_lane_result(const RtuReq& req, LaneState& l, uint32_t t, return false; // unreachable } -// Common init of the traversal accumulator from the ray. +// Common init of the traversal accumulator from the ray, and -- for a walk +// resumed after a callback verdict -- from the lane's committed hit and floor. WalkCtx init_ctx(const RtuReq& req, uint32_t t, - const float ro[3], const float rd[3]) { + const float ro[3], const float rd[3], const LaneState& l) { WalkCtx ctx; ctx.tmin = req.tmin[t]; ctx.tmax = req.tmax[t]; @@ -471,6 +505,20 @@ WalkCtx init_ctx(const RtuReq& req, uint32_t t, // under an instance). vcopy3(ctx.best_obj_o, ro); vcopy3(ctx.best_obj_d, rd); vcopy3(ctx.yield_obj_o, ro); vcopy3(ctx.yield_obj_d, rd); + ctx.yield_key = 0; + ctx.has_floor = l.has_floor; + ctx.floor_t = l.floor_t; + ctx.floor_key = l.floor_key; + if (l.has_floor && l.hit) { + ctx.any_hit = true; + ctx.best_t = l.hit_t; ctx.best_u = l.hit_u; ctx.best_v = l.hit_v; + ctx.best_prim = l.hit_prim; + ctx.best_instance = l.hit_instance_id; + ctx.best_custom = l.hit_instance_custom; + ctx.best_geom = l.hit_geometry; + vcopy3(ctx.best_obj_o, l.hit_obj_o); + vcopy3(ctx.best_obj_d, l.hit_obj_d); + } return ctx; } @@ -492,10 +540,10 @@ WalkResult FlatWalker::walk_lane(const RtuReq& req, uint32_t t, SceneView& sv, const float ro[3] = { req.origin_x[t], req.origin_y[t], req.origin_z[t] }; const float rd[3] = { req.dir_x[t], req.dir_y[t], req.dir_z[t] }; - WalkCtx ctx = init_ctx(req, t, ro, rd); + WalkCtx ctx = init_ctx(req, t, ro, rd, l); // TLAS scenes walk one or more instances; each instance points at a BLAS (a - // triangle list) and (optionally) applies an object→world affine transform. + // triangle list) through its world→object affine transform. uint32_t num_instances = 1; uint32_t triangle_count = 0; #ifdef VX_CFG_RTU_TLAS_ENABLE @@ -556,10 +604,9 @@ WalkResult FlatWalker::walk_lane(const RtuReq& req, uint32_t t, SceneView& sv, std::memcpy(&cur_custom, inst_buf + kRtuInstanceCustomIdOff, sizeof(uint32_t)); - // World→object ray transform. For pure rotation + translation the t - // parameter is preserved, so the BLAS-reported hit_t is also the world - // hit_t. - affine_inverse_transform_ray(xform, ro, rd, ray_o, ray_d); + // World→object ray transform; the direction is not renormalised, so + // the BLAS-reported hit_t is also the world hit_t. + world_to_object_ray(xform, ro, rd, ray_o, ray_d); ++perf.bvh_instance_descents; uint8_t blas_hdr[4]; read_scene_bytes(sv, blas_byte_off, sizeof(blas_hdr), blas_hdr); @@ -592,48 +639,48 @@ WalkResult FlatWalker::walk_lane(const RtuReq& req, uint32_t t, SceneView& sv, float t_hit = 0.f, u = 0.f, v = 0.f; bool back_facing = false; ++perf.bvh_tri_tests; - // Test against ray.tmax (not best_t) so an opaque hit committed earlier in - // this walk doesn't pre-cull a non-opaque candidate that might survive an - // ACCEPT. if (!ray_triangle(ray_o, ray_d, &tri[0], &tri[3], &tri[6], - ctx.tmin, ctx.tmax, + ctx.tmin, ctx.best_t, t_hit, u, v, back_facing)) { continue; } TriClassify cls = classify_tri_hit(ctx.ray_flags, tri_flags, cur_inst_flags, back_facing); + const uint32_t hit_geom = hit_facing_bit(back_facing, cur_inst_flags); if (cls.action == TriAction::Ignore) continue; if (cls.action == TriAction::Commit) { - if (t_hit < ctx.best_t) { - ctx.best_t = t_hit; ctx.best_u = u; ctx.best_v = v; - ctx.best_prim = i; - ctx.best_instance = inst_idx; - ctx.best_custom = cur_custom; - ctx.any_hit = true; - vcopy3(ctx.best_obj_o, ray_o); // this instance's object ray - vcopy3(ctx.best_obj_d, ray_d); - if (ctx.yield_pending && ctx.yield_t >= ctx.best_t) { - ctx.yield_pending = false; - ctx.yield_t = ctx.tmax; - } - if (cls.terminate_on_first_hit) { - // Halt the whole walk: this hit is committed as the result and no - // later triangle or instance may replace it. - ctx.terminated = true; - break; - } + ctx.best_t = t_hit; ctx.best_u = u; ctx.best_v = v; + ctx.best_prim = i; + ctx.best_instance = inst_idx; + ctx.best_custom = cur_custom; + ctx.best_geom = hit_geom; + ctx.any_hit = true; + vcopy3(ctx.best_obj_o, ray_o); // this instance's object ray + vcopy3(ctx.best_obj_d, ray_d); + if (ctx.yield_pending && ctx.yield_t >= ctx.best_t) { + ctx.yield_pending = false; + ctx.yield_t = ctx.tmax; + } + if (cls.terminate_on_first_hit) { + // Halt the whole walk: this hit is committed as the result and no + // later triangle or instance may replace it. + ctx.terminated = true; + break; } } else { // TriAction::Yield - if (t_hit < ctx.best_t && t_hit < ctx.yield_t) { + uint64_t key = cand_key(inst_idx, blas_tri_off + i * kPhase2TriStride); + if (cand_takes(ctx, t_hit, key)) { ctx.yield_pending = true; + ctx.yield_key = key; ctx.yield_t = t_hit; ctx.yield_u = u; ctx.yield_v = v; ctx.yield_prim = i; ctx.yield_sbt = cls.yield_sbt_idx; ctx.yield_cb_type = cls.yield_cb_type; ctx.yield_instance = inst_idx; ctx.yield_custom = cur_custom; + ctx.yield_geom = hit_geom; vcopy3(ctx.yield_obj_o, ray_o); // object ray for AHS/IS vcopy3(ctx.yield_obj_d, ray_d); } @@ -661,7 +708,7 @@ WalkResult Bvh4Walker::walk_lane(const RtuReq& req, uint32_t t, SceneView& sv, const float ro[3] = { req.origin_x[t], req.origin_y[t], req.origin_z[t] }; const float rd[3] = { req.dir_x[t], req.dir_y[t], req.dir_z[t] }; - WalkCtx ctx = init_ctx(req, t, ro, rd); + WalkCtx ctx = init_ctx(req, t, ro, rd, l); // Top-level (non-instanced) triangles carry no instance flags. walk_bvh4_subtree(sv, ro, rd, root_off, 0, 0, 0, ctx, perf); diff --git a/sw/common/rtu_cfg.h b/sw/common/rtu_cfg.h index f0afdb0e58..4e92efa146 100644 --- a/sw/common/rtu_cfg.h +++ b/sw/common/rtu_cfg.h @@ -66,7 +66,8 @@ #define RTU_BVH_COUNT_SHIFT 8 // CW-BVH TLAS instance record (64 B). Emitted under a LEAF_INST leaf. -// float xform[12] @0 3x4 row-major object->world affine +// float xform[12] @0 3x4 row-major world->object affine (the inverse of +// the instance transform; the walker applies it) // uint32 blas_root @48 byte offset (from scene base) of this instance's BLAS root // uint32 custom_id @52 VK_INSTANCE_CUSTOM_INDEX_KHR // uint32 instance_id @56 HW-assigned instance ID diff --git a/sw/common/vx_gfx_abi.h b/sw/common/vx_gfx_abi.h index ae7b639844..4c1184a1d6 100644 --- a/sw/common/vx_gfx_abi.h +++ b/sw/common/vx_gfx_abi.h @@ -250,10 +250,12 @@ static inline uint32_t Pack8888(uint32_t lo, uint32_t hi) { return (hi << 8) | lo; } +// Texel blend by an 8-bit subtexel fraction: the weight is f/256 (Vulkan +// subTexelPrecisionBits = 8), rounded to nearest. A lane peaks at +// 255*256 + 128 < 2^16, so the two packed lanes never carry into each other. static inline uint32_t Lerp8888(uint32_t a, uint32_t b, uint32_t f) { - uint32_t p = a * (0xff - f) + b * f + 0x00800080; - uint32_t q = (p >> 8) & 0x00ff00ff; - return ((p + q) >> 8) & 0x00ff00ff; + uint32_t p = a * (0x100 - f) + b * f + 0x00800080; + return (p >> 8) & 0x00ff00ff; } } // namespace graphics diff --git a/sw/kernel/include/vx_dxa.h b/sw/kernel/include/vx_dxa.h index dbbc031e37..ea722b2752 100644 --- a/sw/kernel/include/vx_dxa.h +++ b/sw/kernel/include/vx_dxa.h @@ -45,6 +45,25 @@ extern "C" { // 1D and 2D: all rs2 lanes are zero, so rs2 = x0 (no second vx_wgather). // 3D–5D: rs2 carries coord2..coord4, requiring a second vx_wgather. +// The DXA reads its packed operands across the warp regardless of the thread +// mask, so it reads them from the warp-gather registers the gathers wrote +// (x31, and x30 for rs2; see __VX_WGATHER_IN). +#define __VX_DXA_ISSUE1(a0v) do { \ + register uint32_t __rs1 __asm__("x31") = (a0v); \ + __asm__ volatile (".insn r %0, 0, %1, x0, %2, x0\n\t" \ + : : "i"(VX_DXA_EXT_OPCODE), "i"(VX_DXA_FUNCT7), "r"(__rs1) \ + : "memory"); \ + } while (0) + +#define __VX_DXA_ISSUE2(a0v, a1v) do { \ + register uint32_t __rs1 __asm__("x31") = (a0v); \ + register uint32_t __rs2 __asm__("x30") = (a1v); \ + __asm__ volatile (".insn r %0, 0, %1, x0, %2, %3\n\t" \ + : : "i"(VX_DXA_EXT_OPCODE), "i"(VX_DXA_FUNCT7), "r"(__rs1), \ + "r"(__rs2) \ + : "memory"); \ + } while (0) + inline uint32_t vx_dxa_pack_meta(uint32_t desc_slot, uint32_t barrier_id) { return (barrier_id << 4) | desc_slot; } @@ -59,11 +78,7 @@ inline void vx_dxa_issue_1d_wg(uint32_t desc_slot, (size_t)meta, (size_t)coord0, (size_t)0u); - __asm__ volatile ( - ".insn r %0, 0, %1, x0, %2, x0\n\t" - : - : "i"(VX_DXA_EXT_OPCODE), "i"(VX_DXA_FUNCT7), "r"(a0) - : "memory"); + __VX_DXA_ISSUE1(a0); } // 2D: rs1 = wgather(smem_addr, meta, coord0, coord1), rs2 = x0 @@ -77,11 +92,7 @@ inline void vx_dxa_issue_2d_wg(uint32_t desc_slot, (size_t)meta, (size_t)coord0, (size_t)coord1); - __asm__ volatile ( - ".insn r %0, 0, %1, x0, %2, x0\n\t" - : - : "i"(VX_DXA_EXT_OPCODE), "i"(VX_DXA_FUNCT7), "r"(a0) - : "memory"); + __VX_DXA_ISSUE1(a0); } // 3D–5D: rs2 = wgather(coord2, coord3, coord4, 0) @@ -96,15 +107,11 @@ inline void vx_dxa_issue_3d_wg(uint32_t desc_slot, (size_t)meta, (size_t)coord0, (size_t)coord1); - const uint32_t a1 = (uint32_t)vx_wgather((size_t)coord2, + const uint32_t a1 = (uint32_t)__VX_WGATHER_IN("x30", 0, (size_t)coord2, (size_t)0u, (size_t)0u, (size_t)0u); - __asm__ volatile ( - ".insn r %0, 0, %1, x0, %2, %3\n\t" - : - : "i"(VX_DXA_EXT_OPCODE), "i"(VX_DXA_FUNCT7), "r"(a0), "r"(a1) - : "memory"); + __VX_DXA_ISSUE2(a0, a1); } inline void vx_dxa_issue_4d_wg(uint32_t desc_slot, @@ -119,15 +126,11 @@ inline void vx_dxa_issue_4d_wg(uint32_t desc_slot, (size_t)meta, (size_t)coord0, (size_t)coord1); - const uint32_t a1 = (uint32_t)vx_wgather((size_t)coord2, + const uint32_t a1 = (uint32_t)__VX_WGATHER_IN("x30", 0, (size_t)coord2, (size_t)coord3, (size_t)0u, (size_t)0u); - __asm__ volatile ( - ".insn r %0, 0, %1, x0, %2, %3\n\t" - : - : "i"(VX_DXA_EXT_OPCODE), "i"(VX_DXA_FUNCT7), "r"(a0), "r"(a1) - : "memory"); + __VX_DXA_ISSUE2(a0, a1); } inline void vx_dxa_issue_5d_wg(uint32_t desc_slot, @@ -143,15 +146,11 @@ inline void vx_dxa_issue_5d_wg(uint32_t desc_slot, (size_t)meta, (size_t)coord0, (size_t)coord1); - const uint32_t a1 = (uint32_t)vx_wgather((size_t)coord2, + const uint32_t a1 = (uint32_t)__VX_WGATHER_IN("x30", 0, (size_t)coord2, (size_t)coord3, (size_t)coord4, (size_t)0u); - __asm__ volatile ( - ".insn r %0, 0, %1, x0, %2, %3\n\t" - : - : "i"(VX_DXA_EXT_OPCODE), "i"(VX_DXA_FUNCT7), "r"(a0), "r"(a1) - : "memory"); + __VX_DXA_ISSUE2(a0, a1); } // Multicast DXA issues read GMEM once and replay SMEM writes to multiple @@ -168,15 +167,11 @@ inline void vx_dxa_issue_1d_multicast_wg(uint32_t desc_slot, (size_t)meta, (size_t)coord0, (size_t)0u); - const uint32_t a1 = (uint32_t)vx_wgather((size_t)0, + const uint32_t a1 = (uint32_t)__VX_WGATHER_IN("x30", 0, (size_t)0, (size_t)0, (size_t)0, (size_t)cta_mask); - __asm__ volatile ( - ".insn r %0, 0, %1, x0, %2, %3\n\t" - : - : "i"(VX_DXA_EXT_OPCODE), "i"(VX_DXA_FUNCT7), "r"(a0), "r"(a1) - : "memory"); + __VX_DXA_ISSUE2(a0, a1); } // 2D multicast: rs2 = wgather(0, 0, 0, cta_mask) @@ -191,15 +186,11 @@ inline void vx_dxa_issue_2d_multicast_wg(uint32_t desc_slot, (size_t)meta, (size_t)coord0, (size_t)coord1); - const uint32_t a1 = (uint32_t)vx_wgather((size_t)0, + const uint32_t a1 = (uint32_t)__VX_WGATHER_IN("x30", 0, (size_t)0, (size_t)0, (size_t)0, (size_t)cta_mask); - __asm__ volatile ( - ".insn r %0, 0, %1, x0, %2, %3\n\t" - : - : "i"(VX_DXA_EXT_OPCODE), "i"(VX_DXA_FUNCT7), "r"(a0), "r"(a1) - : "memory"); + __VX_DXA_ISSUE2(a0, a1); } // 3D multicast: rs2 = wgather(coord2, 0, 0, cta_mask) @@ -215,15 +206,11 @@ inline void vx_dxa_issue_3d_multicast_wg(uint32_t desc_slot, (size_t)meta, (size_t)coord0, (size_t)coord1); - const uint32_t a1 = (uint32_t)vx_wgather((size_t)coord2, + const uint32_t a1 = (uint32_t)__VX_WGATHER_IN("x30", 0, (size_t)coord2, (size_t)0, (size_t)0, (size_t)cta_mask); - __asm__ volatile ( - ".insn r %0, 0, %1, x0, %2, %3\n\t" - : - : "i"(VX_DXA_EXT_OPCODE), "i"(VX_DXA_FUNCT7), "r"(a0), "r"(a1) - : "memory"); + __VX_DXA_ISSUE2(a0, a1); } // 4D multicast: rs2 = wgather(coord2, coord3, 0, cta_mask) @@ -240,15 +227,11 @@ inline void vx_dxa_issue_4d_multicast_wg(uint32_t desc_slot, (size_t)meta, (size_t)coord0, (size_t)coord1); - const uint32_t a1 = (uint32_t)vx_wgather((size_t)coord2, + const uint32_t a1 = (uint32_t)__VX_WGATHER_IN("x30", 0, (size_t)coord2, (size_t)coord3, (size_t)0, (size_t)cta_mask); - __asm__ volatile ( - ".insn r %0, 0, %1, x0, %2, %3\n\t" - : - : "i"(VX_DXA_EXT_OPCODE), "i"(VX_DXA_FUNCT7), "r"(a0), "r"(a1) - : "memory"); + __VX_DXA_ISSUE2(a0, a1); } // 5D multicast: rs2 = wgather(coord2, coord3, coord4, cta_mask) @@ -266,15 +249,11 @@ inline void vx_dxa_issue_5d_multicast_wg(uint32_t desc_slot, (size_t)meta, (size_t)coord0, (size_t)coord1); - const uint32_t a1 = (uint32_t)vx_wgather((size_t)coord2, + const uint32_t a1 = (uint32_t)__VX_WGATHER_IN("x30", 0, (size_t)coord2, (size_t)coord3, (size_t)coord4, (size_t)cta_mask); - __asm__ volatile ( - ".insn r %0, 0, %1, x0, %2, %3\n\t" - : - : "i"(VX_DXA_EXT_OPCODE), "i"(VX_DXA_FUNCT7), "r"(a0), "r"(a1) - : "memory"); + __VX_DXA_ISSUE2(a0, a1); } #ifdef __cplusplus diff --git a/sw/kernel/include/vx_intrinsics.h b/sw/kernel/include/vx_intrinsics.h index 11ee4081a8..427169096e 100644 --- a/sw/kernel/include/vx_intrinsics.h +++ b/sw/kernel/include/vx_intrinsics.h @@ -507,8 +507,17 @@ inline __attribute__((const)) float vx_quad_ddy_f32(float value) { // Each lane gathers a value from the source lane's register file. // S = source lane (compile-time constant, 0-3). // The source lane retains its own rd value; lane (S+1) gets v1[S], (S+2) gets v2[S], (S+3) gets v3[S]. -#define __VX_WGATHER(src_lane, self_val, v1, v2, v3) ({ \ - size_t __ret = (self_val); \ +// +// WGATHER writes every non-source lane of rd, active or not, so a unit reading +// the packed operand across the warp (the RTU trace config, a DXA descriptor) +// sees all of it whatever the thread mask. In an allocatable register that +// would clobber an inactive lane's live value, so rd is pinned to a dedicated +// warp-gather register -- x31, or x30 for a second packed operand live at the +// same time -- which the compiler reserves in any function naming one. A +// consumer of the packed lanes must read that register directly (bind it with +// a register variable); a copy elsewhere is written on active lanes only. +#define __VX_WGATHER_IN(reg, src_lane, self_val, v1, v2, v3) ({ \ + register size_t __ret __asm__(reg) = (self_val); \ __asm__ volatile ( \ ".insn r4 %1, 0, %2, %0, %3, %4, %5" \ : "+r"(__ret) \ @@ -518,6 +527,9 @@ inline __attribute__((const)) float vx_quad_ddy_f32(float value) { __ret; \ }) +#define __VX_WGATHER(src_lane, self_val, v1, v2, v3) \ + __VX_WGATHER_IN("x31", src_lane, self_val, v1, v2, v3) + // Warp-level gather with source lane 0. inline __attribute__((const)) size_t vx_wgather(size_t self_val, size_t v1, size_t v2, size_t v3) { diff --git a/sw/kernel/include/vx_raytrace.h b/sw/kernel/include/vx_raytrace.h index 0ff732b88c..de4d87ec47 100644 --- a/sw/kernel/include/vx_raytrace.h +++ b/sw/kernel/include/vx_raytrace.h @@ -119,6 +119,7 @@ typedef struct { uint32_t geometry_index; uint32_t instance_id; uint32_t instance_custom; // gl_InstanceCustomIndexEXT (VK_INSTANCE_CUSTOM_INDEX) + uint32_t back_facing; // gl_HitKindEXT == BACK_FACING } vx_hit_t; // The struct field order (memory layout) is intentionally NOT the RTU register- @@ -170,9 +171,12 @@ uint32_t vx_rt_wtrace(uint32_t scene_ptr, uint32_t payload_ptr, // list (read by HW convention, like the tensor unit's fragment window); // the encoding itself only names rd/rs1. Named operands (not %0/%1) keep // the field references stable across the long register-binding list. + // The RTU reads the packed config lanes regardless of the thread mask, so it + // reads them from the warp-gather register itself (see __VX_WGATHER_IN). + register uint32_t cfg_reg __asm__("x31") = cfg; __asm__ volatile (".insn r %[op], 7, 0, %[hnd], %[cfg], x0" : [hnd]"=r"(handle) - : [op]"i"(RISCV_CUSTOM1), [cfg]"r"(cfg), + : [op]"i"(RISCV_CUSTOM1), [cfg]"r"(cfg_reg), "f"(r0), "f"(r1), "f"(r2), "f"(r3), "f"(r4), "f"(r5), "f"(r6), "f"(r7)); return handle; @@ -222,8 +226,9 @@ uint32_t vx_rt_wait(uint32_t handle, vx_hit_t* hit) { hit->v = hv; hit->primitive_id = hp; hit->instance_id = hi; - hit->geometry_index = hg; + hit->geometry_index = hg & VX_RT_HIT_GEOMETRY_MASK; hit->instance_custom = hc; + hit->back_facing = (hg & VX_RT_HIT_BACK_FACING) != 0; return status; } diff --git a/sw/kernel/scripts/vxbin.py b/sw/kernel/scripts/vxbin.py index dae8d82bfc..3ca24a0853 100755 --- a/sw/kernel/scripts/vxbin.py +++ b/sw/kernel/scripts/vxbin.py @@ -49,18 +49,26 @@ def get_vma_size(elf_file): print("Failed to calculate vma size due to an error: {}".format(str(e))) sys.exit(-1) +def read_symbols(elf_file): + # (value, name) for each `readelf -s -W` row. Split columns rather than + # pattern-match them: readelf prints a Size of 100000 or more in hex. + cmd = ['readelf', '-s', '-W', elf_file] + output = subprocess.check_output(cmd, universal_newlines=True) + symbols = [] + for line in output.splitlines(): + cols = line.split() + if len(cols) == 8 and cols[0][:-1].isdigit() and cols[0].endswith(':'): + symbols.append((int(cols[1], 16), cols[7])) + return symbols + def get_symbol(elf_file, name): # Read a symbol value from the ELF. We use _edata as the start of BSS and # _end as the end of BSS so runtime_size covers the full RW region (the # linker's DATA_SEGMENT_ALIGN can push _edata/_end past the end of the last # LOAD segment when the kernel has little/no data or BSS). - cmd = ['readelf', '-s', '-W', elf_file] - output = subprocess.check_output(cmd, universal_newlines=True) - regex = re.compile(r'\s*\d+:\s+([0-9a-fA-F]+)\s+\d+\s+\S+\s+\S+\s+\S+\s+\S+\s+' + re.escape(name) + r'$') - for line in output.splitlines(): - match = regex.match(line) - if match: - return int(match.group(1), 16) + for value, sym in read_symbols(elf_file): + if sym == name: + return value print("Error: {} symbol not found in {}".format(name, elf_file)) sys.exit(-1) @@ -69,23 +77,18 @@ def get_kernel_entries(elf_file): # "__vx_kentry_" alias per vortex.kernel function; the runtime's # vx_module_get_kernel() resolves to its address. The conventional # single-kernel entry "kernel_main" is exposed under the public name "main". - cmd = ['readelf', '-s', '-W', elf_file] - output = subprocess.check_output(cmd, universal_newlines=True) - regex = re.compile( - r'\s*\d+:\s+([0-9a-fA-F]+)\s+\d+\s+\S+\s+\S+\s+\S+\s+\S+\s+' - r'__vx_kentry_(\S+)$') entries = [] seen = set() - for line in output.splitlines(): - match = regex.match(line) - if match: - name = match.group(2) - if name == 'kernel_main': - name = 'main' - if name in seen: - continue - seen.add(name) - entries.append((name, int(match.group(1), 16))) + for value, sym in read_symbols(elf_file): + if not sym.startswith('__vx_kentry_'): + continue + name = sym[len('__vx_kentry_'):] + if name == 'kernel_main': + name = 'main' + if name in seen: + continue + seen.add(name) + entries.append((name, value)) return entries def build_symtab_footer(entries): diff --git a/sw/runtime/aved/vortex.cpp b/sw/runtime/aved/vortex.cpp index d07c3b0558..08d431c41a 100644 --- a/sw/runtime/aved/vortex.cpp +++ b/sw/runtime/aved/vortex.cpp @@ -267,6 +267,21 @@ class vx_device { // Only the hardware platform has a slave bridge, so anything else needs // the explicit host-memory sync in cp_reg_write/cp_reg_read. sim_mode_ = (vrtDevice_.getPlatform() != vrt::Platform::HARDWARE); + // Like XRT's xclbin load, clock the kernel at the rate the image was + // timed for: a design write leaves the user clock at whatever the + // previous image set, which can be far above or below this one's. + if (!sim_mode_) { + const uint64_t timed_hz = vrtDevice_.getMaxFrequency(); + vrtDevice_.setFrequency(timed_hz); + const uint64_t clk_hz = vrtDevice_.getFrequency(); + if (clk_hz == 0 || clk_hz > timed_hz + timed_hz / 200) { + fprintf(stderr, "[VXDRV] Error: user clock reads %lu Hz, image is timed for %lu Hz\n", + (unsigned long)clk_hz, (unsigned long)timed_hz); + return -1; + } + printf("[VXDRV] kernel clock %lu Hz (image timed for %lu Hz)\n", + (unsigned long)clk_hz, (unsigned long)timed_hz); + } vrtKernel_ = vrt::Kernel(vrtDevice_, KERNEL_NAME); VRT_CATCH(-1) diff --git a/sw/runtime/common/device.cpp b/sw/runtime/common/device.cpp index a145d98ac9..0eee737e92 100644 --- a/sw/runtime/common/device.cpp +++ b/sw/runtime/common/device.cpp @@ -14,6 +14,7 @@ #include "scope.h" // vx_scope_drain — lossless SCOPE tap-ring drainer #endif +#include #include #include #include @@ -25,6 +26,9 @@ namespace vx { +// Upper bound on one CP-visible host staging buffer for a device transfer. +static constexpr uint64_t CP_STAGING_CHUNK = uint64_t(64) << 20; + // Resolve the pinned-region size: compile-time default // VX_CFG_VM_PINNED_REGION_SIZE, optionally overridden by the // VORTEX_VM_PINNED_SIZE env var (decimal bytes). Returns 0 when VM is @@ -974,25 +978,28 @@ vx_result_t Device::cp_submit_mem_write(uint64_t dev_dst, const void* host_src, uint64_t size, bool physical) { if (size == 0) return VX_SUCCESS; if (!host_src) return VX_ERR_INVALID_VALUE; - // Stage the payload into CP-visible host memory (a plain memcpy through - // the host pointer), then have the CP DMA it to device memory. `physical` + // Stage through a bounded CP-visible host buffer, one chunk at a time: + // a backend can cap a single host allocation well below a large upload + // (an acceleration structure runs to GBs). Each chunk is a plain memcpy + // through the host pointer, then a CP DMA to device memory. `physical` // (set for page-table writes) tells the CP DMA to skip VM translation. HostMem staging; - auto r = host_alloc(size, &staging); + auto r = host_alloc(std::min(size, CP_STAGING_CHUNK), &staging); if (r != VX_SUCCESS) return r; - std::memcpy(staging.host_ptr, host_src, size); - // Make the fill visible to the CP before the command that reads it can - // be fetched. On shadowing backends this is the ONLY push of this - // region: the backend's doorbell publish deliberately does not touch - // generic regions (a blanket publish can push a half-filled or stale - // shadow over device bytes another agent owns). - r = platform()->host_mem_push(staging.cp_addr); - if (r != VX_SUCCESS) { - host_free(staging.cp_addr); - return r; + auto src = static_cast(host_src); + for (uint64_t off = 0; off < size && r == VX_SUCCESS; off += CP_STAGING_CHUNK) { + const uint64_t n = std::min(size - off, CP_STAGING_CHUNK); + std::memcpy(staging.host_ptr, src + off, n); + // Make the fill visible to the CP before the command that reads it can + // be fetched. On shadowing backends this is the ONLY push of this + // region: the backend's doorbell publish deliberately does not touch + // generic regions (a blanket publish can push a half-filled or stale + // shadow over device bytes another agent owns). + r = platform()->host_mem_push(staging.cp_addr); + if (r == VX_SUCCESS) + r = cp_submit_mem_(CP_OPCODE_MEM_WRITE, dev_dst + off, + staging.cp_addr, n, physical); } - r = cp_submit_mem_(CP_OPCODE_MEM_WRITE, dev_dst, staging.cp_addr, size, - physical); host_free(staging.cp_addr); return r; } @@ -1001,22 +1008,25 @@ vx_result_t Device::cp_submit_mem_read(void* host_dst, uint64_t dev_src, uint64_t size, bool physical) { if (size == 0) return VX_SUCCESS; if (!host_dst) return VX_ERR_INVALID_VALUE; - // Have the CP DMA device->host into a CP-visible host staging buffer, - // then memcpy it back to the caller's pointer. + // Have the CP DMA device->host into a bounded CP-visible host staging + // buffer chunk by chunk (see cp_submit_mem_write), copying each back. HostMem staging; - auto r = host_alloc(size, &staging); + auto r = host_alloc(std::min(size, CP_STAGING_CHUNK), &staging); if (r != VX_SUCCESS) return r; - r = cp_submit_mem_(CP_OPCODE_MEM_READ, staging.cp_addr, dev_src, size, - physical); - if (r == VX_SUCCESS) { + auto dst = static_cast(host_dst); + for (uint64_t off = 0; off < size && r == VX_SUCCESS; off += CP_STAGING_CHUNK) { + const uint64_t n = std::min(size - off, CP_STAGING_CHUNK); + r = cp_submit_mem_(CP_OPCODE_MEM_READ, staging.cp_addr, dev_src + off, + n, physical); // The CP wrote the staging region; on a backend that shadows CP // memory the host copy is stale until pulled. The submit's Q_SEQNUM // poll has already fenced on the completion line, so the pull reads // settled bytes. - r = platform()->host_mem_pull(staging.cp_addr); + if (r == VX_SUCCESS) + r = platform()->host_mem_pull(staging.cp_addr); + if (r == VX_SUCCESS) + std::memcpy(dst + off, staging.host_ptr, n); } - if (r == VX_SUCCESS) - std::memcpy(host_dst, staging.host_ptr, size); host_free(staging.cp_addr); return r; } diff --git a/sw/runtime/common/queue.cpp b/sw/runtime/common/queue.cpp index fd7f54e994..df63f7a9dc 100644 --- a/sw/runtime/common/queue.cpp +++ b/sw/runtime/common/queue.cpp @@ -90,9 +90,17 @@ void Queue::worker_loop() { uint64_t start_ns = submit_ns; uint64_t end_ns = submit_ns; + { + std::lock_guard g(cmd_mu_); + if (r == VX_SUCCESS && async_error_ != VX_SUCCESS) r = async_error_; + } if (r == VX_SUCCESS && cmd.work) { r = cmd.work(&start_ns, &end_ns); } + if (r != VX_SUCCESS) { + std::lock_guard g(cmd_mu_); + if (async_error_ == VX_SUCCESS) async_error_ = r; + } if (cmd.completion) { if (profiling_enabled()) { @@ -169,6 +177,14 @@ vx_result_t Queue::finish(uint64_t timeout_ns) { if (r != VX_SUCCESS) return r; r = to_event(ev)->wait(timeout_ns); to_event(ev)->release(); + if (r == VX_ERR_TIMEOUT) return r; + // Report the first failure since the last finish, then let the queue run + // again (the commands behind it completed with that error, unexecuted). + std::lock_guard g(cmd_mu_); + if (async_error_ != VX_SUCCESS) { + r = async_error_; + async_error_ = VX_SUCCESS; + } return r; } diff --git a/sw/runtime/common/vortex2_internal.h b/sw/runtime/common/vortex2_internal.h index e6c23a2cd8..ef803eefc7 100644 --- a/sw/runtime/common/vortex2_internal.h +++ b/sw/runtime/common/vortex2_internal.h @@ -765,6 +765,9 @@ class Queue : public RefCounted { std::condition_variable cmd_cv_; std::deque commands_; bool shutdown_ = false; + // First failure of an in-order command; later commands fail with it + // instead of running on incomplete state, until finish() reports it. + vx_result_t async_error_ = VX_SUCCESS; std::thread worker_; }; diff --git a/sw/runtime/include/raytrace.h b/sw/runtime/include/raytrace.h index 82860dfe3b..c01a75e7bd 100644 --- a/sw/runtime/include/raytrace.h +++ b/sw/runtime/include/raytrace.h @@ -310,7 +310,7 @@ class BvhBuilder { groups.push_back(std::move(b)); } - NodeRef ch[6]; + NodeRef ch[6] = {}; uint32_t n = 0; for (auto& g : groups) if (!g.empty()) ch[n++] = build_node(g); @@ -325,6 +325,27 @@ class BvhBuilder { bool overflow_ = false; }; +// The world->object matrix an instance record carries: the inverse of the +// host's object->world 3x4 affine, in double, rounded once. +inline void world_to_object(const float otw[12], float wto[12]) { + const double a = otw[0], b = otw[1], c = otw[2]; + const double d = otw[4], e = otw[5], f = otw[6]; + const double g = otw[8], h = otw[9], k = otw[10]; + const double det = a * (e * k - f * h) - b * (d * k - f * g) + c * (d * h - e * g); + const double inv = (det != 0.0) ? 1.0 / det : 0.0; + const double r[9] = { (e * k - f * h) * inv, (c * h - b * k) * inv, (b * f - c * e) * inv, + (f * g - d * k) * inv, (a * k - c * g) * inv, (c * d - a * f) * inv, + (d * h - e * g) * inv, (b * g - a * h) * inv, (a * e - b * d) * inv }; + for (int i = 0; i < 3; ++i) { + double t = 0.0; + for (int j = 0; j < 3; ++j) { + wto[i * 4 + j] = float(r[i * 3 + j]); + t -= r[i * 3 + j] * double(otw[j * 4 + 3]); + } + wto[i * 4 + 3] = float(t); + } +} + } // namespace detail // ── Host-side scene preparation ───────────────────────────────────────── @@ -425,7 +446,9 @@ inline bool build_tlas_scene(const host_tlas_t& src, const host_instance_t& in = src.instances[i]; if (in.blas_index >= src.blas_count) { out_scene.clear(); return false; } uint8_t* rec = out_scene.data() + insts_off + i * RTU_BVH_INSTANCE_STRIDE; - std::memcpy(rec, in.xform, sizeof(in.xform)); + float wto[12]; + detail::world_to_object(in.xform, wto); + std::memcpy(rec, wto, sizeof(wto)); uint32_t broot = blas_root[in.blas_index]; uint32_t cid = in.custom_id; uint32_t iid = in.instance_id; diff --git a/tests/raytracing/Makefile b/tests/raytracing/Makefile index 5745ed6c45..da11b9fee6 100644 --- a/tests/raytracing/Makefile +++ b/tests/raytracing/Makefile @@ -11,7 +11,8 @@ TESTS := \ rt_smoke_miss rt_smoke_is rt_smoke_sbt rt_smoke_tlas \ rt_smoke_ahs_mixed rt_smoke_recursive \ rt_smoke_bvh_basic rt_smoke_bvh_multilevel \ - rt_smoke_bvh_instanced rt_smoke_ahs_bvh \ + rt_smoke_bvh_instanced rt_smoke_ahs_bvh rt_smoke_ahs_multi \ + rt_smoke_partial_mask rt_smoke_proc_inst \ rt_smoke_shadow rt_smoke_cull_back \ rt_smoke_async_batch rt_smoke_cull_mask \ rt_smoke_proc rt_smoke_bvh6 rt_bvh_multinode rt_smoke_numctx \ diff --git a/tests/raytracing/rt_replay/Makefile b/tests/raytracing/rt_replay/Makefile new file mode 100644 index 0000000000..0250d63b17 --- /dev/null +++ b/tests/raytracing/rt_replay/Makefile @@ -0,0 +1,29 @@ +ROOT_DIR := $(realpath ../../..) +include $(ROOT_DIR)/config.mk + +# The scene format the RTU walks is a build-time choice (VX_CFG_RTU_BVH_WIDTH): +# replay against the configuration that recorded the log, so CONFIGS is used +# as given, plus the RTU itself. +CONFIGS := $(if $(findstring -DVX_CFG_EXT_RTU_ENABLE,$(CONFIGS)),$(CONFIGS),$(CONFIGS) -DVX_CFG_EXT_RTU_ENABLE) + +PROJECT := rt_replay + +SRC_DIR := $(VORTEX_HOME)/tests/raytracing/$(PROJECT) + +SRCS := $(SRC_DIR)/main.cpp +HDRS := $(SRC_DIR)/common.h + +VX_SRCS := $(SRC_DIR)/kernel.cpp +VX_HDRS := $(SRC_DIR)/common.h + +# Usage (from the build dir), e.g. for the LumiBench RTU config on rtlsim: +# CONFIGS="$(cat /CONFIGS.txt) -DVX_DBG_STALL_TIMEOUT=2000000000" \ +# make -C tests/raytracing/rt_replay run-rtlsim OPTS="-f -r 0:1024" +# The stall-timeout define is required on rtlsim: one hard ray can keep every +# warp waiting in vx_rt_wait longer than the scheduler watchdog's default. +# Logs come from SimX with VX_RTU_RAYLOG=; ./rt_replay -h lists options. +OPTS ?= + +KERNEL_LIB := vortex2 + +include ../common.mk diff --git a/tests/raytracing/rt_replay/common.h b/tests/raytracing/rt_replay/common.h new file mode 100644 index 0000000000..8f94077e7b --- /dev/null +++ b/tests/raytracing/rt_replay/common.h @@ -0,0 +1,81 @@ +// Copyright © 2019-2023 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// rt_replay — re-traces rays recorded by the SimX RTU ray log +// (VX_RTU_RAYLOG, sim/simx/rtu/rtu_raylog.h) against the scene lines the +// recording walks read, restored at their original device addresses. + +#ifndef _RT_REPLAY_COMMON_H_ +#define _RT_REPLAY_COMMON_H_ + +#include + +// One ray as the kernel consumes it. Each warp's rays share scene / flags / +// cull: those ride the warp-uniform half of vx_rt_wtrace. +typedef struct { + uint32_t scene; + uint32_t flags; + uint32_t cull; + uint32_t pad; + float origin[3]; + float dir[3]; + float tmin; + float tmax; +} replay_ray_t; + +// Status bit set when the trace yielded a candidate (the recorded ray did not). +#define REPLAY_STS_YIELDED 0x100u + +typedef struct { + uint32_t status; + uint32_t t; // float bits + uint32_t u; + uint32_t v; + uint32_t prim; + uint32_t geom; // geometry index | VX_RT_HIT_BACK_FACING + uint32_t inst_id; + uint32_t inst_custom; +} replay_result_t; + +typedef struct { + uint64_t rays_addr; + uint64_t results_addr; + uint32_t count; + uint32_t pad; +} kernel_arg_t; + +#ifdef __cplusplus +// Ray-log stream layout (mirror of sim/simx/rtu/rtu_raylog.h). +#define RAYLOG_MAGIC 0x4C525856u +#define RAYLOG_VERSION 1u +#define RAYLOG_REC_LINE 1u +#define RAYLOG_REC_RAY 2u +#define RAYLOG_INFO_REPLAYABLE (1u << 8) + +struct RaylogHeader { + uint32_t magic, version, num_threads, line_bytes, bvh_width, ray_bytes; + uint32_t reserved[2]; +}; + +struct RaylogRay { + uint32_t type, info, seq, epoch; + uint32_t scene_root, ray_flags, cull_mask, warp_lane; + uint32_t origin[3], dir[3]; + uint32_t tmin, tmax; + uint32_t status, hit_t, hit_u, hit_v, prim, geom, inst_id, inst_custom, hit_attr; + uint32_t reserved[3]; +}; +static_assert(sizeof(RaylogRay) == 112, "RaylogRay layout"); +#endif + +#endif // _RT_REPLAY_COMMON_H_ diff --git a/tests/raytracing/rt_replay/kernel.cpp b/tests/raytracing/rt_replay/kernel.cpp new file mode 100644 index 0000000000..a6cf6922c5 --- /dev/null +++ b/tests/raytracing/rt_replay/kernel.cpp @@ -0,0 +1,50 @@ +// Copyright © 2019-2023 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// rt_replay kernel — one recorded ray per thread. The host pads every +// warp-sized group to one (scene, flags, cull), so the trace config is uniform. + +#include +#include +#include "common.h" + +__kernel void kernel_main(kernel_arg_t* arg) { + uint32_t i = blockIdx.x * blockDim.x + threadIdx.x; + if (i >= arg->count) return; + + const replay_ray_t* rays = (const replay_ray_t*)((uintptr_t)arg->rays_addr); + const replay_ray_t& r = rays[i]; + vx_ray_t ray = { {r.origin[0], r.origin[1], r.origin[2]}, + {r.dir[0], r.dir[1], r.dir[2]}, + r.tmin, r.tmax }; + uint32_t h = vx_rt_wtrace(r.scene, 0u, r.flags, r.cull, &ray); + vx_hit_t hit; + uint32_t sts = vx_rt_wait(h, &hit); + uint32_t yielded = 0; + // A replayed ray was never decided by a shader; a candidate here is itself a + // divergence. Resolve it as a closest-hit/miss dispatch would, and flag it. + while (vx_rt_sts_is_yield(sts)) { + yielded = REPLAY_STS_YIELDED; + sts = vx_rt_continue(h, VX_RT_CB_IGNORE, hit.t, 0u, &hit); + } + + replay_result_t* res = (replay_result_t*)((uintptr_t)arg->results_addr) + i; + res->status = sts | yielded; + res->t = __builtin_bit_cast(uint32_t, hit.t); + res->u = __builtin_bit_cast(uint32_t, hit.u); + res->v = __builtin_bit_cast(uint32_t, hit.v); + res->prim = hit.primitive_id; + res->geom = hit.geometry_index | (hit.back_facing ? VX_RT_HIT_BACK_FACING : 0u); + res->inst_id = hit.instance_id; + res->inst_custom = hit.instance_custom; +} diff --git a/tests/raytracing/rt_replay/main.cpp b/tests/raytracing/rt_replay/main.cpp new file mode 100644 index 0000000000..e326bcecfe --- /dev/null +++ b/tests/raytracing/rt_replay/main.cpp @@ -0,0 +1,438 @@ +// Copyright © 2019-2023 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// rt_replay — host driver. +// +// Loads a SimX RTU ray log (VX_RTU_RAYLOG), restores every scene line the +// recorded walks read at its original device address (absolute pointers inside +// the scene stay valid), re-traces the recorded rays in batches and compares +// each result bit-exactly with the recorded one. + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include "common.h" + +#define RT_CHECK(_expr) \ + do { \ + int _ret = _expr; \ + if (0 == _ret) break; \ + printf("Error: '%s' returned %d!\n", #_expr, (int)_ret); \ + cleanup(); \ + exit(-1); \ + } while (false) + +namespace { + +const char* kernel_file = "kernel.vxbin"; + +vx_device_h device = nullptr; +vx_queue_h queue = nullptr; +vx_module_h module_ = nullptr; +vx_kernel_h kernel = nullptr; +vx_buffer_h rays_buf = nullptr; +vx_buffer_h res_buf = nullptr; +std::vector scene_bufs; + +void cleanup() { + if (!device) return; + for (auto b : scene_bufs) vx_buffer_release(b); + if (rays_buf) vx_buffer_release(rays_buf); + if (res_buf) vx_buffer_release(res_buf); + if (kernel) vx_kernel_release(kernel); + if (module_) vx_module_release(module_); + if (queue) vx_queue_release(queue); + vx_device_release(device); + device = nullptr; +} + +struct Line { + uint64_t addr; + uint32_t epoch; + std::vector data; +}; + +struct Range { + uint64_t base; + uint64_t size; + vx_buffer_h buf; +}; + +const char* log_path = nullptr; +const char* list_path = nullptr; +const char* out_path = nullptr; +uint32_t batch = 4096; +uint64_t range_lo = 0; +uint64_t range_hi = UINT64_MAX; +uint64_t stride = 1; +int64_t only_epoch = -1; +uint32_t max_print = 20; +bool all_rays = false; +bool verbose = false; + +void show_usage() { + printf("Usage: rt_replay -f [-b batch] [-r lo:hi] [-l listfile] [-s stride]\n" + " [-e epoch] [-m maxprint] [-o mismatch_file] [-a] [-v]\n" + " -r lo:hi replay ray indices [lo, hi) (index = RAY record order in the log)\n" + " -l file replay only the ray indices listed (one per line, '#' comments;\n" + " a mismatch file from -o is accepted as is)\n" + " -s n replay every n-th selected ray\n" + " -a also replay rays a shader decided (any-hit / intersection)\n" + "Build with the CONFIGS that recorded the log. For rtlsim also pass\n" + "-DVX_DBG_STALL_TIMEOUT=2000000000: one hard ray can hold every warp in\n" + "vx_rt_wait past the scheduler's default all-warps-stalled watchdog.\n"); +} + +void parse_args(int argc, char** argv) { + int c; + while ((c = getopt(argc, argv, "f:b:r:l:s:e:m:o:avh")) != -1) { + switch (c) { + case 'f': log_path = optarg; break; + case 'b': batch = std::max(1u, (uint32_t)strtoul(optarg, nullptr, 0)); break; + case 'r': { + const char* colon = strchr(optarg, ':'); + range_lo = strtoull(optarg, nullptr, 0); + if (colon && colon[1]) range_hi = strtoull(colon + 1, nullptr, 0); + else if (!colon) range_hi = range_lo + 1; + break; + } + case 'l': list_path = optarg; break; + case 's': stride = std::max(1, strtoull(optarg, nullptr, 0)); break; + case 'e': only_epoch = strtoll(optarg, nullptr, 0); break; + case 'm': max_print = strtoul(optarg, nullptr, 0); break; + case 'o': out_path = optarg; break; + case 'a': all_rays = true; break; + case 'v': verbose = true; break; + default: show_usage(); exit(c == 'h' ? 0 : -1); + } + } + if (!log_path) { show_usage(); exit(-1); } +} + +float fbits(uint32_t u) { float f; memcpy(&f, &u, 4); return f; } + +float near_t(float t) { return t + std::fabs(t) * 0x1p-19f; } + +// Why a replayed ray disagrees with its record. +const char* classify(const RaylogRay& ref, const replay_result_t& got) { + if (got.status & REPLAY_STS_YIELDED) return "yielded"; + const bool rh = ref.status == VX_RT_STS_DONE_HIT; + const bool gh = got.status == VX_RT_STS_DONE_HIT; + if (rh && !gh) return "ref_hit_got_miss"; + if (!rh && gh) return "ref_miss_got_hit"; + if (!rh) return "status"; + const bool same_prim = ref.prim == got.prim && ref.inst_id == got.inst_id + && (ref.geom & VX_RT_HIT_GEOMETRY_MASK) == (got.geom & VX_RT_HIT_GEOMETRY_MASK); + if (same_prim) { + if (ref.hit_t != got.t || ref.hit_u != got.u || ref.hit_v != got.v) return "same_prim_bits"; + return "attr"; + } + if (ref.ray_flags & VX_RT_FLAG_TERMINATE_ON_FIRST_HIT) return "tofh_other_hit"; + const float a = fbits(ref.hit_t), b = fbits(got.t); + if (a == b) return "tie_exact"; + if ((a < b && near_t(a) >= b) || (b < a && near_t(b) >= a)) return "near_tie"; + return b < a ? "got_nearer_hit" : "got_farther_hit"; +} + +bool matches(const RaylogRay& ref, const replay_result_t& got) { + if (got.status != ref.status) return false; + if (ref.status != VX_RT_STS_DONE_HIT) return true; + return got.t == ref.hit_t && got.u == ref.hit_u && got.v == ref.hit_v + && got.prim == ref.prim && got.geom == ref.geom + && got.inst_id == ref.inst_id && got.inst_custom == ref.inst_custom; +} + +void print_ray(FILE* fp, uint64_t idx, const RaylogRay& r, const replay_result_t& g, const char* cat) { + fprintf(fp, "%llu seq=%u cat=%s epoch=%u scene=0x%x flags=0x%x cull=0x%x" + " o=(%a,%a,%a) d=(%a,%a,%a) tmin=%a tmax=%a\n", + (unsigned long long)idx, r.seq, cat, r.epoch, r.scene_root, r.ray_flags, r.cull_mask, + fbits(r.origin[0]), fbits(r.origin[1]), fbits(r.origin[2]), + fbits(r.dir[0]), fbits(r.dir[1]), fbits(r.dir[2]), + fbits(r.tmin), fbits(r.tmax)); + fprintf(fp, "# ref: sts=%u t=%a (0x%08x) u=0x%08x v=0x%08x prim=%u geom=0x%08x inst=%u custom=%u\n", + r.status, fbits(r.hit_t), r.hit_t, r.hit_u, r.hit_v, r.prim, r.geom, r.inst_id, r.inst_custom); + fprintf(fp, "# got: sts=%u t=%a (0x%08x) u=0x%08x v=0x%08x prim=%u geom=0x%08x inst=%u custom=%u\n", + g.status, fbits(g.t), g.t, g.u, g.v, g.prim, g.geom, g.inst_id, g.inst_custom); +} + +} // namespace + +int main(int argc, char** argv) { + parse_args(argc, argv); + + // ── load the log ────────────────────────────────────────────────── + FILE* fp = fopen(log_path, "rb"); + if (!fp) { printf("Error: cannot open %s\n", log_path); return -1; } + RaylogHeader hdr{}; + if (fread(&hdr, sizeof(hdr), 1, fp) != 1 || hdr.magic != RAYLOG_MAGIC + || hdr.version != RAYLOG_VERSION || hdr.ray_bytes != sizeof(RaylogRay)) { + printf("Error: %s is not a version-%u ray log\n", log_path, RAYLOG_VERSION); + return -1; + } +#ifdef VX_CFG_RTU_BVH_WIDTH + if (hdr.bvh_width != VX_CFG_RTU_BVH_WIDTH) { + printf("Error: log recorded with RTU_BVH_WIDTH=%u, replay built with %u\n", + hdr.bvh_width, (uint32_t)VX_CFG_RTU_BVH_WIDTH); + return -1; + } +#endif + const uint32_t lb = hdr.line_bytes; + std::vector lines; + std::vector rays; + { + std::vector rec(std::max(sizeof(RaylogRay), 16 + lb)); + uint32_t type; + while (fread(&type, 4, 1, fp) == 1) { + if (type == RAYLOG_REC_LINE) { + if (fread(rec.data() + 4, 12 + lb, 1, fp) != 1) break; + Line l; + memcpy(&l.epoch, rec.data() + 4, 4); + memcpy(&l.addr, rec.data() + 8, 8); + l.data.assign(rec.data() + 16, rec.data() + 16 + lb); + lines.push_back(std::move(l)); + } else if (type == RAYLOG_REC_RAY) { + RaylogRay r; + r.type = type; + if (fread(reinterpret_cast(&r) + 4, sizeof(r) - 4, 1, fp) != 1) break; + rays.push_back(r); + } else { + printf("Error: corrupt record type %u in %s\n", type, log_path); + return -1; + } + } + } + fclose(fp); + uint32_t num_epochs = 1; + for (auto& l : lines) num_epochs = std::max(num_epochs, l.epoch + 1); + printf("raylog: %zu rays, %zu lines (%u B), %u epoch(s), bvh_width=%u, recorded NT=%u\n", + rays.size(), lines.size(), lb, num_epochs, hdr.bvh_width, hdr.num_threads); + + // ── select rays ─────────────────────────────────────────────────── + std::vector sel; + { + std::vector cand; + if (list_path) { + FILE* lf = fopen(list_path, "r"); + if (!lf) { printf("Error: cannot open %s\n", list_path); return -1; } + char buf[4096]; + while (fgets(buf, sizeof(buf), lf)) { + if (buf[0] == '#' || buf[0] == '\n') continue; + cand.push_back(strtoull(buf, nullptr, 0)); + } + fclose(lf); + } else { + for (uint64_t i = range_lo; i < std::min(range_hi, rays.size()); ++i) cand.push_back(i); + } + uint64_t skipped_cb = 0, n = 0; + for (uint64_t i : cand) { + if (i >= rays.size()) continue; + if (range_lo > i || i >= range_hi) continue; + const RaylogRay& r = rays[i]; + if (only_epoch >= 0 && r.epoch != (uint32_t)only_epoch) continue; + if (!all_rays && !(r.info & RAYLOG_INFO_REPLAYABLE)) { ++skipped_cb; continue; } + if ((n++ % stride) != 0) continue; + sel.push_back(i); + } + std::stable_sort(sel.begin(), sel.end(), [&](uint64_t a, uint64_t b) { + return rays[a].epoch < rays[b].epoch; + }); + printf("selected %zu rays (%llu skipped: decided by a shader callback)\n", + sel.size(), (unsigned long long)skipped_cb); + } + if (sel.empty()) { printf("nothing to replay\n"); return 0; } + + // ── device + scene reservation ──────────────────────────────────── + RT_CHECK(vx_device_open(0, &device)); + vx_queue_info_t qi = { sizeof(qi), nullptr, VX_QUEUE_PRIORITY_NORMAL, 0 }; + RT_CHECK(vx_queue_create(device, &qi, &queue)); + + // Claim the recorded scene addresses before anything else is allocated, in + // page-aligned ranges (nearby pages merged to keep the buffer count low). + std::vector ranges; + { + const uint64_t page = 4096, merge_gap = 64 * 1024; + std::vector pages; + for (auto& l : lines) { + for (uint64_t p = l.addr & ~(page - 1); p < l.addr + lb; p += page) pages.push_back(p); + } + std::sort(pages.begin(), pages.end()); + pages.erase(std::unique(pages.begin(), pages.end()), pages.end()); + for (uint64_t p : pages) { + if (!ranges.empty() && p <= ranges.back().base + ranges.back().size + merge_gap) { + ranges.back().size = p + page - ranges.back().base; + } else { + ranges.push_back({p, page, nullptr}); + } + } + uint64_t total = 0; + for (auto& r : ranges) { + RT_CHECK(vx_buffer_reserve(device, r.base, r.size, VX_MEM_READ, &r.buf)); + scene_bufs.push_back(r.buf); + total += r.size; + } + printf("scene: %zu reserved range(s), %llu KB\n", ranges.size(), (unsigned long long)(total >> 10)); + } + + RT_CHECK(vx_module_load_file(device, kernel_file, &module_)); + RT_CHECK(vx_module_get_kernel(module_, "main", &kernel)); + + const uint32_t NT = VX_CFG_NUM_THREADS; + uint64_t cap = 0; + + std::map cats; + uint64_t replayed = 0, mismatched = 0, printed = 0; + FILE* of = out_path ? fopen(out_path, "w") : nullptr; + if (of) fprintf(of, "# rt_replay mismatches of %s: then ref/got detail\n", log_path); + double kernel_s = 0; + int32_t loaded_epoch = -1; + + for (size_t pos = 0; pos < sel.size();) { + // A batch never spans epochs: each epoch replays against its own image. + const uint32_t epoch = rays[sel[pos]].epoch; + size_t end = pos; + while (end < sel.size() && end - pos < batch && rays[sel[end]].epoch == epoch) ++end; + + if ((int32_t)epoch != loaded_epoch) { + // Image of `epoch`: per address, the newest version recorded at or before + // it, else the first one recorded (the line was not rewritten before then). + std::map img; + for (auto& l : lines) { + auto it = img.find(l.addr); + if (it == img.end()) { img[l.addr] = &l; continue; } + if (l.epoch <= epoch && (it->second->epoch > epoch || l.epoch >= it->second->epoch)) it->second = &l; + } + std::vector> hosts; // alive until the queue drains + for (auto& r : ranges) { + hosts.emplace_back(r.size, 0); + auto& host = hosts.back(); + for (auto it = img.lower_bound(r.base); it != img.end() && it->first < r.base + r.size; ++it) { + uint64_t off = it->first - r.base; + memcpy(host.data() + off, it->second->data.data(), std::min(lb, r.size - off)); + } + RT_CHECK(vx_enqueue_write(queue, r.buf, 0, host.data(), r.size, 0, nullptr, nullptr)); + } + RT_CHECK(vx_queue_finish(queue, VX_TIMEOUT_INFINITE)); + loaded_epoch = epoch; + } + + // Group by the warp-uniform trace config; pad each group to whole warps + // with copies of its last ray (their results are dropped). + std::map, std::vector> groups; + for (size_t k = pos; k < end; ++k) { + const RaylogRay& r = rays[sel[k]]; + groups[{r.scene_root, r.ray_flags & ~(VX_RT_FLAG_ENABLE_CHS | VX_RT_FLAG_ENABLE_MISS), r.cull_mask}].push_back(sel[k]); + } + std::vector dev_rays; + std::vector owner; // ray index, or -1 for padding + for (auto& g : groups) { + auto& v = g.second; + size_t padded = (v.size() + NT - 1) / NT * NT; + for (size_t k = 0; k < padded; ++k) { + uint64_t idx = v[std::min(k, v.size() - 1)]; + const RaylogRay& r = rays[idx]; + replay_ray_t d{}; + d.scene = r.scene_root; + d.flags = std::get<1>(g.first); + d.cull = r.cull_mask; + for (int j = 0; j < 3; ++j) { d.origin[j] = fbits(r.origin[j]); d.dir[j] = fbits(r.dir[j]); } + d.tmin = fbits(r.tmin); + d.tmax = fbits(r.tmax); + dev_rays.push_back(d); + owner.push_back(k < v.size() ? (int64_t)idx : -1); + } + } + const uint32_t count = (uint32_t)dev_rays.size(); + if (count > cap) { + if (rays_buf) vx_buffer_release(rays_buf); + if (res_buf) vx_buffer_release(res_buf); + cap = std::max(count, batch + NT); + RT_CHECK(vx_buffer_create(device, cap * sizeof(replay_ray_t), VX_MEM_READ, &rays_buf)); + RT_CHECK(vx_buffer_create(device, cap * sizeof(replay_result_t), VX_MEM_WRITE, &res_buf)); + } + kernel_arg_t arg{}; + RT_CHECK(vx_buffer_address(rays_buf, &arg.rays_addr)); + RT_CHECK(vx_buffer_address(res_buf, &arg.results_addr)); + arg.count = count; + RT_CHECK(vx_enqueue_write(queue, rays_buf, 0, dev_rays.data(), count * sizeof(replay_ray_t), 0, nullptr, nullptr)); + RT_CHECK(vx_queue_finish(queue, VX_TIMEOUT_INFINITE)); + + if (verbose) { + printf("launch [%zu, %zu): %u threads\n", pos, end, count); + fflush(stdout); + } + auto t0 = std::chrono::steady_clock::now(); + vx_event_h lev = nullptr, rev = nullptr; + vx_launch_info_t li = {}; + li.struct_size = sizeof(li); + li.kernel = kernel; + li.args_host = &arg; + li.args_size = sizeof(arg); + li.ndim = 1; + li.grid_dim[0] = count / NT; + li.block_dim[0] = NT; + RT_CHECK(vx_enqueue_launch(queue, &li, 0, nullptr, &lev)); + std::vector res(count); + RT_CHECK(vx_enqueue_read(queue, res.data(), res_buf, 0, count * sizeof(replay_result_t), 1, &lev, &rev)); + RT_CHECK(vx_event_wait_value(rev, 1, VX_TIMEOUT_INFINITE)); + vx_event_release(rev); + vx_event_release(lev); + double dt = std::chrono::duration(std::chrono::steady_clock::now() - t0).count(); + kernel_s += dt; + + uint64_t bad = 0; + for (uint32_t k = 0; k < count; ++k) { + if (owner[k] < 0) continue; + const RaylogRay& r = rays[owner[k]]; + ++replayed; + if (matches(r, res[k])) continue; + ++bad; + const char* cat = classify(r, res[k]); + ++cats[cat]; + if (of) print_ray(of, owner[k], r, res[k], cat); + if (printed < max_print) { print_ray(stdout, owner[k], r, res[k], cat); ++printed; } + } + mismatched += bad; + if (of) fflush(of); // a model that hangs or aborts later keeps what it reported + if (verbose || sel.size() > batch) { + printf("batch [%zu, %zu) epoch %u: %llu rays, %llu mismatches, %.1fs\n", + pos, end, epoch, (unsigned long long)(end - pos), (unsigned long long)bad, dt); + fflush(stdout); + } + pos = end; + } + if (of) fclose(of); + + printf("replayed %llu rays in %.1fs: %llu match, %llu mismatch\n", + (unsigned long long)replayed, kernel_s, + (unsigned long long)(replayed - mismatched), (unsigned long long)mismatched); + for (auto& c : cats) printf(" %-18s %llu\n", c.first.c_str(), (unsigned long long)c.second); + + cleanup(); + if (mismatched) { + printf("FAILED!\n"); + return 1; + } + printf("PASSED!\n"); + return 0; +} diff --git a/tests/raytracing/rt_smoke_ahs_geom/kernel.cpp b/tests/raytracing/rt_smoke_ahs_geom/kernel.cpp index f3bf38da34..cd33ff2f2b 100644 --- a/tests/raytracing/rt_smoke_ahs_geom/kernel.cpp +++ b/tests/raytracing/rt_smoke_ahs_geom/kernel.cpp @@ -44,7 +44,8 @@ __kernel void kernel_main(kernel_arg_t* arg) { // Read the candidate geometry/instance attributes and the capture buffer // pointer (staged as the trace payload) from the register window, stash. uint32_t cand_ptr = vx_rt_get_attr(VX_RT_PAYLOAD_PTR_LO, sts); - uint32_t cand_geom = vx_rt_get_attr(VX_RT_HIT_GEOMETRY_INDEX, sts); + uint32_t cand_geom = vx_rt_get_attr(VX_RT_HIT_GEOMETRY_INDEX, sts) + & VX_RT_HIT_GEOMETRY_MASK; uint32_t cand_inst = vx_rt_get_attr(VX_RT_HIT_INSTANCE_ID, sts); uint32_t* cand = (uint32_t*)(uintptr_t)cand_ptr; cand[0] = cand_geom; // cand->cand_geometry diff --git a/tests/raytracing/rt_smoke_ahs_multi/Makefile b/tests/raytracing/rt_smoke_ahs_multi/Makefile new file mode 100644 index 0000000000..8a0a1f8777 --- /dev/null +++ b/tests/raytracing/rt_smoke_ahs_multi/Makefile @@ -0,0 +1,20 @@ +ROOT_DIR := $(realpath ../../..) +include $(ROOT_DIR)/config.mk + +CONFIGS := $(if $(findstring -DVX_CFG_EXT_RTU_ENABLE,$(CONFIGS)),$(CONFIGS),$(CONFIGS) -DVX_CFG_EXT_RTU_ENABLE) +# CW-BVH4 scene -> build the RTU as a CW-BVH4 walker. +CONFIGS += -DVX_CFG_RTU_BVH_WIDTH=4 + +PROJECT := rt_smoke_ahs_multi + +SRC_DIR := $(VORTEX_HOME)/tests/raytracing/$(PROJECT) + +SRCS := $(SRC_DIR)/main.cpp + +VX_SRCS := $(SRC_DIR)/kernel.cpp + +OPTS ?= + +KERNEL_LIB := vortex2 + +include ../common.mk diff --git a/tests/raytracing/rt_smoke_ahs_multi/common.h b/tests/raytracing/rt_smoke_ahs_multi/common.h new file mode 100644 index 0000000000..cd2ea37c70 --- /dev/null +++ b/tests/raytracing/rt_smoke_ahs_multi/common.h @@ -0,0 +1,62 @@ +// Copyright © 2019-2023 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// RTU multi-candidate any-hit smoke. +// +// Several non-opaque triangles lie along one ray -- two of them coplanar at the +// same t -- with an opaque triangle behind them. Every candidate the walk meets +// must be offered, one per callback round, in ascending (t, record) order; a +// verdict that does not end the ray resumes the walk above the decided +// candidate. Each lane accepts a different primitive and records the order it +// was offered candidates in. + +#ifndef _RTU_SMOKE_AHS_MULTI_COMMON_H_ +#define _RTU_SMOKE_AHS_MULTI_COMMON_H_ + +#include + +// CW-BVH4 scene layout (matches rt_smoke_bvh_basic). +#define VX_BVH_SCENE_KIND 2 +#define VX_BVH_SCENE_HDR_BYTES 16 +#define VX_BVH_LEAF_HDR_BYTES 16 +#define VX_BVH_TRI_STRIDE 40 +#define VX_BVH_TRI_FLAGS_OFFSET 36 +#define VX_BVH_KIND_LEAF_TRI 1 +#define VX_BVH_COUNT_SHIFT 8 +#define VX_BVH_TRI_FLAG_OPAQUE 0x1u + +#define RTU_MULTI_NUM_TRIS 6 +#define RTU_MULTI_MAX_OFFER 8 +#define RTU_MULTI_NONE 0xffu // accept nothing + +typedef struct { + uint32_t status; + float hit_t; + uint32_t primitive_id; + uint32_t num_offered; + uint8_t offered[RTU_MULTI_MAX_OFFER]; // primitive ids, in callback order +} rtu_result_t; + +typedef struct { + uint64_t scene_addr; + uint64_t results_addr; + uint64_t targets_addr; // uint8_t per lane: the primitive it accepts + uint32_t num_lanes; + uint32_t pad; + float ray_origin[3]; + float ray_direction[3]; + float tmin; + float tmax; +} kernel_arg_t; + +#endif // _RTU_SMOKE_AHS_MULTI_COMMON_H_ diff --git a/tests/raytracing/rt_smoke_ahs_multi/kernel.cpp b/tests/raytracing/rt_smoke_ahs_multi/kernel.cpp new file mode 100644 index 0000000000..fdbe122e8d --- /dev/null +++ b/tests/raytracing/rt_smoke_ahs_multi/kernel.cpp @@ -0,0 +1,58 @@ +// Copyright © 2019-2023 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// RTU multi-candidate any-hit smoke kernel: every lane of the warp traces the +// same ray and ACCEPTs only its target primitive, IGNOREing every other +// candidate, so the lanes' walks diverge -- some end early, others keep being +// offered candidates (and read PENDING while they wait on the others). + +#include +#include +#include "common.h" + +__kernel void kernel_main(kernel_arg_t* arg) { + uint32_t tid = threadIdx.x; + if (tid >= arg->num_lanes) return; + uint32_t target = ((const uint8_t*)(uintptr_t)arg->targets_addr)[tid]; + + vx_ray_t ray = { + {arg->ray_origin[0], arg->ray_origin[1], arg->ray_origin[2]}, + {arg->ray_direction[0], arg->ray_direction[1], arg->ray_direction[2]}, + arg->tmin, + arg->tmax, + }; + + rtu_result_t* res = (rtu_result_t*)((uintptr_t)arg->results_addr) + tid; + uint32_t n = 0; + + uint32_t scene_lo = (uint32_t)(arg->scene_addr & 0xffffffffu); + uint32_t h = vx_rt_wtrace(scene_lo, 0u, 0u, 0xffu, &ray); + vx_hit_t hit; + uint32_t sts = vx_rt_wait(h, &hit); + while (vx_rt_sts_is_yield(sts)) { + uint32_t action = VX_RT_CB_IGNORE; + if (vx_rt_sts_has_candidate(sts)) { + if (n < RTU_MULTI_MAX_OFFER) + res->offered[n] = (uint8_t)hit.primitive_id; + ++n; + if (hit.primitive_id == target) + action = VX_RT_CB_ACCEPT; + } + sts = vx_rt_continue(h, action, hit.t, 0u, &hit); + } + + res->status = sts; + res->hit_t = hit.t; + res->primitive_id = hit.primitive_id; + res->num_offered = n; +} diff --git a/tests/raytracing/rt_smoke_ahs_multi/main.cpp b/tests/raytracing/rt_smoke_ahs_multi/main.cpp new file mode 100644 index 0000000000..ab7176e2c4 --- /dev/null +++ b/tests/raytracing/rt_smoke_ahs_multi/main.cpp @@ -0,0 +1,204 @@ +// Copyright © 2019-2023 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// RTU multi-candidate any-hit smoke -- host driver. +// +// One CW-BVH4 leaf, six triangles stacked along the ray (+z from z=0): +// prim 0 t=7 non-opaque +// prim 1 t=3 non-opaque +// prim 2 t=5 non-opaque +// prim 3 t=5 non-opaque (coplanar with prim 2: a t tie) +// prim 4 t=8 OPAQUE +// prim 5 t=9 non-opaque (behind the opaque one: never offered) +// A lane that accepts nothing is offered 1, 2, 3, 0 -- ascending t, the tie +// broken by record order -- and ends on the opaque prim 4. A lane that accepts +// prim P is offered the prefix of that sequence up to P and ends on P. + +#include +#include +#include +#include +#include +#include + +#include +#include +#include "common.h" + +#define RT_CHECK(_expr) \ + do { \ + int _ret = _expr; \ + if (0 == _ret) break; \ + printf("Error: '%s' returned %d!\n", #_expr, (int)_ret); \ + cleanup(); \ + exit(-1); \ + } while (false) + +const char* kernel_file = "kernel.vxbin"; +uint32_t num_lanes = 12; + +vx_device_h device = nullptr; +vx_buffer_h scene_buffer = nullptr; +vx_buffer_h res_buffer = nullptr; +vx_buffer_h tgt_buffer = nullptr; +vx_queue_h queue = nullptr; +vx_module_h module_ = nullptr; +vx_kernel_h kernel = nullptr; +kernel_arg_t kernel_arg = {}; + +static void show_usage() { + std::cout << "RTU multi-candidate any-hit smoke test." << std::endl; + std::cout << "Usage: [-k kernel] [-n lanes] [-h]" << std::endl; +} + +static void parse_args(int argc, char** argv) { + int c; + while ((c = getopt(argc, argv, "n:k:h")) != -1) { + switch (c) { + case 'n': num_lanes = atoi(optarg); break; + case 'k': kernel_file = optarg; break; + case 'h': show_usage(); exit(0); + default: show_usage(); exit(-1); + } + } +} + +void cleanup() { + if (device) { + if (scene_buffer) vx_buffer_release(scene_buffer); + if (res_buffer) vx_buffer_release(res_buffer); + if (tgt_buffer) vx_buffer_release(tgt_buffer); + if (kernel) vx_kernel_release(kernel); + if (module_) vx_module_release(module_); + if (queue) vx_queue_release(queue); + vx_device_release(device); + } +} + +int main(int argc, char* argv[]) { + parse_args(argc, argv); + + RT_CHECK(vx_device_open(0, &device)); + vx_queue_info_t qi = { sizeof(qi), nullptr, VX_QUEUE_PRIORITY_NORMAL, 0 }; + RT_CHECK(vx_queue_create(device, &qi, &queue)); + + static const float tri_t[RTU_MULTI_NUM_TRIS] = {7.f, 3.f, 5.f, 5.f, 8.f, 9.f}; + static const uint32_t tri_fl[RTU_MULTI_NUM_TRIS] = {0, 0, 0, 0, VX_BVH_TRI_FLAG_OPAQUE, 0}; + + std::vector scene_bytes(VX_BVH_SCENE_HDR_BYTES + VX_BVH_LEAF_HDR_BYTES + + RTU_MULTI_NUM_TRIS * VX_BVH_TRI_STRIDE, 0); + uint32_t* sh = reinterpret_cast(scene_bytes.data()); + sh[0] = VX_BVH_SCENE_HDR_BYTES; // root_node_offset + sh[1] = VX_BVH_SCENE_KIND; + sh[2] = (uint32_t)scene_bytes.size(); + sh[3] = 1; // leaf_count + uint32_t* lh = reinterpret_cast(scene_bytes.data() + VX_BVH_SCENE_HDR_BYTES); + lh[0] = VX_BVH_KIND_LEAF_TRI | (RTU_MULTI_NUM_TRIS << VX_BVH_COUNT_SHIFT); + for (uint32_t i = 0; i < RTU_MULTI_NUM_TRIS; ++i) { + uint8_t* rec = scene_bytes.data() + VX_BVH_SCENE_HDR_BYTES + VX_BVH_LEAF_HDR_BYTES + + i * VX_BVH_TRI_STRIDE; + float v[9] = {0.f, 0.f, tri_t[i], 1.f, 0.f, tri_t[i], 0.f, 1.f, tri_t[i]}; + memcpy(rec, v, sizeof(v)); + memcpy(rec + VX_BVH_TRI_FLAGS_OFFSET, &tri_fl[i], sizeof(uint32_t)); + } + + // lane targets cycle through: none, then each primitive in turn + static const uint8_t target_cycle[] = {RTU_MULTI_NONE, 1, 2, 3, 0, 5}; + const uint32_t ncycle = sizeof(target_cycle) / sizeof(target_cycle[0]); + std::vector targets(num_lanes); + for (uint32_t i = 0; i < num_lanes; ++i) + targets[i] = target_cycle[i % ncycle]; + + uint32_t scene_sz = (uint32_t)scene_bytes.size(); + uint32_t res_size = num_lanes * sizeof(rtu_result_t); + RT_CHECK(vx_buffer_create(device, scene_sz, VX_MEM_READ, &scene_buffer)); + RT_CHECK(vx_buffer_address(scene_buffer, &kernel_arg.scene_addr)); + RT_CHECK(vx_buffer_create(device, res_size, VX_MEM_READ_WRITE, &res_buffer)); + RT_CHECK(vx_buffer_address(res_buffer, &kernel_arg.results_addr)); + RT_CHECK(vx_buffer_create(device, num_lanes, VX_MEM_READ, &tgt_buffer)); + RT_CHECK(vx_buffer_address(tgt_buffer, &kernel_arg.targets_addr)); + + kernel_arg.num_lanes = num_lanes; + kernel_arg.ray_origin[0] = 0.25f; + kernel_arg.ray_origin[1] = 0.25f; + kernel_arg.ray_origin[2] = 0.0f; + kernel_arg.ray_direction[2] = 1.0f; + kernel_arg.tmin = 0.001f; + kernel_arg.tmax = 1e30f; + + std::vector zero(num_lanes); + memset(zero.data(), 0, res_size); + RT_CHECK(vx_enqueue_write(queue, scene_buffer, 0, scene_bytes.data(), scene_sz, 0, nullptr, nullptr)); + RT_CHECK(vx_enqueue_write(queue, tgt_buffer, 0, targets.data(), num_lanes, 0, nullptr, nullptr)); + RT_CHECK(vx_enqueue_write(queue, res_buffer, 0, zero.data(), res_size, 0, nullptr, nullptr)); + RT_CHECK(vx_module_load_file(device, kernel_file, &module_)); + RT_CHECK(vx_module_get_kernel(module_, "main", &kernel)); + + std::cout << "bvh4: 1 leaf, " << RTU_MULTI_NUM_TRIS << " tris, lanes=" << num_lanes << std::endl; + + vx_event_h launch_ev = nullptr, read_ev = nullptr; + { + vx_launch_info_t li = {}; + li.struct_size = sizeof(li); + li.kernel = kernel; + li.args_host = &kernel_arg; + li.args_size = sizeof(kernel_arg); + li.ndim = 1; + li.grid_dim[0] = 1; + li.block_dim[0] = num_lanes; + RT_CHECK(vx_enqueue_launch(queue, &li, 0, nullptr, &launch_ev)); + } + std::vector results(num_lanes); + RT_CHECK(vx_enqueue_read(queue, results.data(), res_buffer, 0, res_size, 1, &launch_ev, &read_ev)); + RT_CHECK(vx_event_wait_value(read_ev, 1, VX_TIMEOUT_INFINITE)); + vx_event_release(read_ev); + vx_event_release(launch_ev); + + // oracle: the offer order when every candidate is ignored + static const uint8_t order[] = {1, 2, 3, 0}; + const uint32_t norder = sizeof(order); + + int errors = 0; + for (uint32_t i = 0; i < num_lanes; ++i) { + uint32_t tgt = targets[i]; + uint32_t exp_n = norder, exp_prim = 4; + for (uint32_t k = 0; k < norder; ++k) { + if (order[k] == tgt) { exp_n = k + 1; exp_prim = tgt; break; } + } + float exp_t = tri_t[exp_prim]; + const rtu_result_t& r = results[i]; + bool ok = (r.status == VX_RT_STS_DONE_HIT) + && (r.primitive_id == exp_prim) + && (std::fabs(r.hit_t - exp_t) < 1e-4f) + && (r.num_offered == exp_n); + for (uint32_t k = 0; ok && k < exp_n; ++k) + ok = (r.offered[k] == order[k]); + if (!ok) { + std::cout << "lane " << i << " (accepts " << tgt << "): status=" << r.status + << " prim=" << r.primitive_id << " t=" << r.hit_t << " offered=["; + for (uint32_t k = 0; k < r.num_offered && k < RTU_MULTI_MAX_OFFER; ++k) + std::cout << (k ? "," : "") << uint32_t(r.offered[k]); + std::cout << "] expected prim=" << exp_prim << " t=" << exp_t + << " offered " << exp_n << std::endl; + ++errors; + } + } + + cleanup(); + if (errors != 0) { + std::cout << "FAILED with " << errors << " errors" << std::endl; + return 1; + } + std::cout << "PASSED!" << std::endl; + return 0; +} diff --git a/tests/raytracing/rt_smoke_bvh_instanced/main.cpp b/tests/raytracing/rt_smoke_bvh_instanced/main.cpp index e2aac68c7b..73ee543537 100644 --- a/tests/raytracing/rt_smoke_bvh_instanced/main.cpp +++ b/tests/raytracing/rt_smoke_bvh_instanced/main.cpp @@ -86,10 +86,10 @@ static void emit_instance(uint8_t* out, float tx, float ty, float tz, uint32_t blas_off, uint32_t custom_id, uint32_t instance_id) { float* xform = reinterpret_cast(out); - // Row-major 3x4 affine: [R t]. Identity R, translation t. - xform[0] = 1.f; xform[1] = 0.f; xform[2] = 0.f; xform[3] = tx; - xform[4] = 0.f; xform[5] = 1.f; xform[6] = 0.f; xform[7] = ty; - xform[8] = 0.f; xform[9] = 0.f; xform[10] = 1.f; xform[11] = tz; + // Row-major 3x4 world->object affine: the inverse of [I t], i.e. [I -t]. + xform[0] = 1.f; xform[1] = 0.f; xform[2] = 0.f; xform[3] = -tx; + xform[4] = 0.f; xform[5] = 1.f; xform[6] = 0.f; xform[7] = -ty; + xform[8] = 0.f; xform[9] = 0.f; xform[10] = 1.f; xform[11] = -tz; *reinterpret_cast(out + VX_BVH_INSTANCE_BLAS_OFF) = blas_off; *reinterpret_cast(out + VX_BVH_INSTANCE_CUSTOM_ID) = custom_id; *reinterpret_cast(out + VX_BVH_INSTANCE_ID_OFFSET) = instance_id; diff --git a/tests/raytracing/rt_smoke_cull_mask/main.cpp b/tests/raytracing/rt_smoke_cull_mask/main.cpp index 91e186ceeb..0664fdc290 100644 --- a/tests/raytracing/rt_smoke_cull_mask/main.cpp +++ b/tests/raytracing/rt_smoke_cull_mask/main.cpp @@ -88,7 +88,7 @@ int main(int /*argc*/, char* /*argv*/[]) { float* xform = reinterpret_cast(inst); xform[0] = 1.f; xform[1] = 0.f; xform[2] = 0.f; xform[3] = 0.f; xform[4] = 0.f; xform[5] = 1.f; xform[6] = 0.f; xform[7] = 0.f; - xform[8] = 0.f; xform[9] = 0.f; xform[10] = 1.f; xform[11] = tz; + xform[8] = 0.f; xform[9] = 0.f; xform[10] = 1.f; xform[11] = -tz; // world->object uint32_t* inst_tail = reinterpret_cast( inst + RTU_INSTANCE_BLAS_OFF_OFF); inst_tail[0] = kBlasOff; diff --git a/tests/raytracing/rt_smoke_deep_stack/main.cpp b/tests/raytracing/rt_smoke_deep_stack/main.cpp index 71ca750bac..77e900cbe9 100644 --- a/tests/raytracing/rt_smoke_deep_stack/main.cpp +++ b/tests/raytracing/rt_smoke_deep_stack/main.cpp @@ -15,9 +15,10 @@ // // Builds a CW-BVH4 over N triangles stacked in depth along the ray, so the // tree is several levels deep — deeper than the modest short stack the Makefile -// configures (VX_CFG_RTU_STACK_DEPTH). A +z ray hits every triangle; the walker -// must overflow, drop far subtrees, and re-descend (restart) to still -// return the CLOSEST hit (nearest triangle, prim 0, t=5). +// configures (VX_CFG_RTU_STACK_DEPTH), so the walker overflows its stack, drops +// subtrees and restarts. Two scenes: every triangle hit (the closest is on the +// first path walked), and decoys whose boxes the ray enters first while only a +// far triangle is hit (reachable only through restarts). #include #include @@ -68,58 +69,46 @@ int main(int /*argc*/, char* /*argv*/[]) { vx_queue_info_t qi = { sizeof(qi), nullptr, VX_QUEUE_PRIORITY_NORMAL, 0 }; RT_CHECK(vx_queue_create(device, &qi, &queue)); - // N opaque triangles all covering the ray's (x,y) footprint, stacked at - // z = 5, 6, ... The SAH builder splits them into a deep tree. The ray hits - // all of them; triangle 0 (z=5) is the closest. - constexpr uint32_t N = 64; - std::vector tris(N); - for (uint32_t i = 0; i < N; ++i) { - float z = 5.0f + (float)i; - tris[i].v0[0] = 0.f; tris[i].v0[1] = 0.f; tris[i].v0[2] = z; - tris[i].v1[0] = 1.f; tris[i].v1[1] = 0.f; tris[i].v1[2] = z; - tris[i].v2[0] = 0.f; tris[i].v2[1] = 1.f; tris[i].v2[2] = z; - tris[i].flags = RTU_BVH_FLAG_OPAQUE; - } + int errors = 0; - host_bvh_t src = { tris.data(), N, /*geometry_index*/ 0 }; - std::vector scene; - uint64_t root_offset = 0; - if (!build_bvh_scene<4>(src, scene, root_offset)) { - std::cout << "build_bvh_scene failed" << std::endl; - cleanup(); - return 1; - } - std::cout << "scene: " << scene.size() << " B, " << N - << " tris (deep CW-BVH4)" << std::endl; - - RT_CHECK(vx_buffer_create(device, (uint32_t)scene.size(), VX_MEM_READ, &scene_buffer)); - RT_CHECK(vx_buffer_address(scene_buffer, &kernel_arg.scene_addr)); - - uint32_t res_size = sizeof(rtu_result_t); - RT_CHECK(vx_buffer_create(device, res_size, VX_MEM_WRITE, &res_buffer)); - RT_CHECK(vx_buffer_address(res_buffer, &kernel_arg.results_addr)); - - kernel_arg.ray_origin[0] = 0.25f; - kernel_arg.ray_origin[1] = 0.25f; - kernel_arg.ray_origin[2] = 0.0f; - kernel_arg.ray_direction[0] = 0.0f; - kernel_arg.ray_direction[1] = 0.0f; - kernel_arg.ray_direction[2] = 1.0f; - kernel_arg.tmin = 0.001f; - kernel_arg.tmax = 1e30f; - - std::cout << "scene_addr=0x" << std::hex << kernel_arg.scene_addr << std::dec - << " deep CW-BVH4 (closest hit must survive short-stack overflow)" - << std::endl; - - RT_CHECK(vx_enqueue_write(queue, scene_buffer, 0, scene.data(), - (uint32_t)scene.size(), 0, nullptr, nullptr)); - RT_CHECK(vx_module_load_file(device, kernel_file, &module_)); - RT_CHECK(vx_module_get_kernel(module_, "main", &kernel)); - - std::cout << "launch kernel" << std::endl; - vx_event_h launch_ev = nullptr, read_ev = nullptr; - { + // One trace of the +z ray at (0.25, 0.25) against a CW-BVH4 over `tris`; + // the walk must return the closest hit (t, prim). + auto run_case = [&](const char* name, const std::vector& tris, + float exp_t, uint32_t exp_prim) { + host_bvh_t src = { tris.data(), (uint32_t)tris.size(), /*geometry_index*/ 0 }; + std::vector scene; + uint64_t root_offset = 0; + if (!build_bvh_scene<4>(src, scene, root_offset)) { + std::cout << name << ": build_bvh_scene failed" << std::endl; + ++errors; + return; + } + std::cout << name << ": scene " << scene.size() << " B, " << tris.size() + << " tris (deep CW-BVH4)" << std::endl; + + RT_CHECK(vx_buffer_create(device, (uint32_t)scene.size(), VX_MEM_READ, &scene_buffer)); + RT_CHECK(vx_buffer_address(scene_buffer, &kernel_arg.scene_addr)); + uint32_t res_size = sizeof(rtu_result_t); + RT_CHECK(vx_buffer_create(device, res_size, VX_MEM_WRITE, &res_buffer)); + RT_CHECK(vx_buffer_address(res_buffer, &kernel_arg.results_addr)); + + kernel_arg.ray_origin[0] = 0.25f; + kernel_arg.ray_origin[1] = 0.25f; + kernel_arg.ray_origin[2] = 0.0f; + kernel_arg.ray_direction[0] = 0.0f; + kernel_arg.ray_direction[1] = 0.0f; + kernel_arg.ray_direction[2] = 1.0f; + kernel_arg.tmin = 0.001f; + kernel_arg.tmax = 1e30f; + + RT_CHECK(vx_enqueue_write(queue, scene_buffer, 0, scene.data(), + (uint32_t)scene.size(), 0, nullptr, nullptr)); + if (!kernel) { + RT_CHECK(vx_module_load_file(device, kernel_file, &module_)); + RT_CHECK(vx_module_get_kernel(module_, "main", &kernel)); + } + + vx_event_h launch_ev = nullptr, read_ev = nullptr; vx_launch_info_t li = {}; li.struct_size = sizeof(li); li.kernel = kernel; @@ -129,30 +118,52 @@ int main(int /*argc*/, char* /*argv*/[]) { li.grid_dim[0] = 1; li.block_dim[0] = 1; RT_CHECK(vx_enqueue_launch(queue, &li, 0, nullptr, &launch_ev)); - } - - rtu_result_t result = {}; - RT_CHECK(vx_enqueue_read(queue, &result, res_buffer, 0, res_size, - 1, &launch_ev, &read_ev)); - RT_CHECK(vx_event_wait_value(read_ev, 1, VX_TIMEOUT_INFINITE)); - vx_event_release(read_ev); - vx_event_release(launch_ev); - const uint32_t exp_status = VX_RT_STS_DONE_HIT; - const float exp_t = 5.f; // nearest triangle - const uint32_t exp_prim = 0; // source index of the z=5 triangle - std::cout << "oracle: HIT t=" << exp_t << " prim=" << exp_prim << std::endl; + rtu_result_t result = {}; + RT_CHECK(vx_enqueue_read(queue, &result, res_buffer, 0, res_size, + 1, &launch_ev, &read_ev)); + RT_CHECK(vx_event_wait_value(read_ev, 1, VX_TIMEOUT_INFINITE)); + vx_event_release(read_ev); + vx_event_release(launch_ev); + vx_buffer_release(scene_buffer); scene_buffer = nullptr; + vx_buffer_release(res_buffer); res_buffer = nullptr; + + std::cout << name << ": oracle HIT t=" << exp_t << " prim=" << exp_prim << std::endl; + if (result.status != VX_RT_STS_DONE_HIT || std::fabs(result.hit_t - exp_t) >= 1e-4f + || result.primitive_id != exp_prim) { + std::cout << name << ": result status=" << result.status + << " hit_t=" << result.hit_t + << " prim=" << result.primitive_id << std::endl; + ++errors; + } + }; + + auto tri_at = [](float z, float x0) { + host_tri_t t = {}; + t.v0[0] = x0; t.v0[1] = 0.f; t.v0[2] = z; + t.v1[0] = 1.f; t.v1[1] = 0.f; t.v1[2] = z; + t.v2[0] = 0.f; t.v2[1] = 1.f; t.v2[2] = z; + t.flags = RTU_BVH_FLAG_OPAQUE; + return t; + }; - int errors = 0; - bool sts_ok = (result.status == exp_status); - bool t_ok = std::fabs(result.hit_t - exp_t) < 1e-4f; - bool prim_ok = (result.primitive_id == exp_prim); - if (!sts_ok || !t_ok || !prim_ok) { - std::cout << "result: status=" << result.status - << " hit_t=" << result.hit_t - << " prim=" << result.primitive_id << std::endl; - ++errors; - } + // N opaque triangles all covering the ray's (x,y) footprint, stacked at + // z = 5, 6, ... The SAH builder splits them into a deep tree. The ray hits + // all of them; triangle 0 (z=5) is the closest, on the walk's first path. + constexpr uint32_t N = 64; + std::vector tris; + for (uint32_t i = 0; i < N; ++i) tris.push_back(tri_at(5.0f + (float)i, 0.f)); + run_case("stacked", tris, 5.f, 0); + + // The same stack, but only one triangle, deep in the far half, covers the + // ray: the others' boxes do (so they are walked first) while the triangles + // miss it. The hit sits in subtrees the short stack drops, so the walk has + // to restart, repeatedly, to reach it. + constexpr uint32_t kHit = 41; + tris.clear(); + for (uint32_t i = 0; i < N; ++i) + tris.push_back(tri_at(5.0f + (float)i, (i == kHit) ? 0.f : 0.6f)); + run_case("decoys", tris, 5.f + (float)kHit, kHit); cleanup(); diff --git a/tests/raytracing/rt_smoke_inst_flags/main.cpp b/tests/raytracing/rt_smoke_inst_flags/main.cpp index 285b65e943..5d19e2b97c 100644 --- a/tests/raytracing/rt_smoke_inst_flags/main.cpp +++ b/tests/raytracing/rt_smoke_inst_flags/main.cpp @@ -111,7 +111,7 @@ int main(int argc, char* argv[]) { float* xform = reinterpret_cast(inst); xform[0] = 1.f; xform[1] = 0.f; xform[2] = 0.f; xform[3] = 0.f; xform[4] = 0.f; xform[5] = 1.f; xform[6] = 0.f; xform[7] = 0.f; - xform[8] = 0.f; xform[9] = 0.f; xform[10] = 1.f; xform[11] = 5.f; + xform[8] = 0.f; xform[9] = 0.f; xform[10] = 1.f; xform[11] = -5.f; // world->object uint32_t* blas_off = reinterpret_cast(inst + RTU_INSTANCE_BLAS_OFF_OFF); *blas_off = kBlasOff; uint32_t* custom_id = reinterpret_cast(inst + RTU_INSTANCE_CUSTOM_ID_OFF); diff --git a/tests/raytracing/rt_smoke_partial_mask/Makefile b/tests/raytracing/rt_smoke_partial_mask/Makefile new file mode 100644 index 0000000000..0432b3400c --- /dev/null +++ b/tests/raytracing/rt_smoke_partial_mask/Makefile @@ -0,0 +1,20 @@ +ROOT_DIR := $(realpath ../../..) +include $(ROOT_DIR)/config.mk + +CONFIGS := $(if $(findstring -DVX_CFG_EXT_RTU_ENABLE,$(CONFIGS)),$(CONFIGS),$(CONFIGS) -DVX_CFG_EXT_RTU_ENABLE) +# CW-BVH4 scenes -> build the RTU as a CW-BVH4 walker. +CONFIGS += -DVX_CFG_RTU_BVH_WIDTH=4 + +PROJECT := rt_smoke_partial_mask + +SRC_DIR := $(VORTEX_HOME)/tests/raytracing/$(PROJECT) + +SRCS := $(SRC_DIR)/main.cpp + +VX_SRCS := $(SRC_DIR)/kernel.cpp + +OPTS ?= + +KERNEL_LIB := vortex2 + +include ../common.mk diff --git a/tests/raytracing/rt_smoke_partial_mask/common.h b/tests/raytracing/rt_smoke_partial_mask/common.h new file mode 100644 index 0000000000..982d6cb0f9 --- /dev/null +++ b/tests/raytracing/rt_smoke_partial_mask/common.h @@ -0,0 +1,51 @@ +// Copyright © 2019-2023 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// RTU partial-warp trace smoke. +// +// A trace's warp-uniform config (scene, payload, flags|cull) is gathered into +// lanes 1-3 of its config register. A trace issued by a warp whose low lanes +// are masked off must still deliver its own config: the kernel first traces an +// EMPTY scene from every lane (leaving that scene in the register's lanes), +// then traces the triangle scene from the warp's last lane only. + +#ifndef _RTU_SMOKE_PARTIAL_MASK_COMMON_H_ +#define _RTU_SMOKE_PARTIAL_MASK_COMMON_H_ + +#include + +#define VX_BVH_SCENE_KIND 2 +#define VX_BVH_SCENE_HDR_BYTES 16 +#define VX_BVH_LEAF_HDR_BYTES 16 +#define VX_BVH_TRI_STRIDE 40 +#define VX_BVH_TRI_FLAGS_OFFSET 36 +#define VX_BVH_KIND_LEAF_TRI 1 +#define VX_BVH_COUNT_SHIFT 8 +#define VX_BVH_TRI_FLAG_OPAQUE 0x1u + +typedef struct { + uint32_t first_status; // the all-lane trace of the empty scene + uint32_t second_status; // the last-lane trace of the triangle scene + float second_t; + uint32_t pad; +} rtu_result_t; + +typedef struct { + uint64_t empty_scene_addr; + uint64_t tri_scene_addr; + uint64_t results_addr; + uint32_t num_lanes; + uint32_t pad; +} kernel_arg_t; + +#endif // _RTU_SMOKE_PARTIAL_MASK_COMMON_H_ diff --git a/tests/raytracing/rt_smoke_partial_mask/kernel.cpp b/tests/raytracing/rt_smoke_partial_mask/kernel.cpp new file mode 100644 index 0000000000..157f3533af --- /dev/null +++ b/tests/raytracing/rt_smoke_partial_mask/kernel.cpp @@ -0,0 +1,44 @@ +// Copyright © 2019-2023 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// RTU partial-warp trace smoke kernel: an all-lane trace of the empty scene, +// then a trace of the triangle scene from the warp's last lane alone. + +#include +#include +#include "common.h" + +static uint32_t trace_one(uint32_t scene, float* t_out) { + vx_ray_t ray = {{0.25f, 0.25f, 0.f}, {0.f, 0.f, 1.f}, 0.001f, 1e30f}; + uint32_t h = vx_rt_wtrace(scene, 0u, 0u, 0xffu, &ray); + vx_hit_t hit; + uint32_t sts = vx_rt_wait(h, &hit); + while (vx_rt_sts_is_yield(sts)) + sts = vx_rt_continue(h, VX_RT_CB_ACCEPT, hit.t, 0u, &hit); + *t_out = hit.t; + return sts; +} + +__kernel void kernel_main(kernel_arg_t* arg) { + uint32_t tid = threadIdx.x; + if (tid >= arg->num_lanes) return; + rtu_result_t* res = (rtu_result_t*)((uintptr_t)arg->results_addr) + tid; + + float t; + res->first_status = trace_one((uint32_t)arg->empty_scene_addr, &t); + res->second_status = 0xffffffffu; + if (tid == arg->num_lanes - 1) { + res->second_status = trace_one((uint32_t)arg->tri_scene_addr, &t); + res->second_t = t; + } +} diff --git a/tests/raytracing/rt_smoke_partial_mask/main.cpp b/tests/raytracing/rt_smoke_partial_mask/main.cpp new file mode 100644 index 0000000000..e429da77dd --- /dev/null +++ b/tests/raytracing/rt_smoke_partial_mask/main.cpp @@ -0,0 +1,150 @@ +// Copyright © 2019-2023 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// RTU partial-warp trace smoke -- host driver. One warp: every lane traces an +// empty CW-BVH4 scene (a miss), then the last lane alone traces a scene holding +// one opaque triangle at t=5 (a hit). + +#include +#include +#include +#include +#include + +#include +#include +#include "common.h" + +#define RT_CHECK(_expr) \ + do { \ + int _ret = _expr; \ + if (0 == _ret) break; \ + printf("Error: '%s' returned %d!\n", #_expr, (int)_ret); \ + cleanup(); \ + exit(-1); \ + } while (false) + +const char* kernel_file = "kernel.vxbin"; + +vx_device_h device = nullptr; +vx_buffer_h scene_buffer = nullptr; +vx_buffer_h res_buffer = nullptr; +vx_queue_h queue = nullptr; +vx_module_h module_ = nullptr; +vx_kernel_h kernel = nullptr; +kernel_arg_t kernel_arg = {}; + +void cleanup() { + if (device) { + if (scene_buffer) vx_buffer_release(scene_buffer); + if (res_buffer) vx_buffer_release(res_buffer); + if (kernel) vx_kernel_release(kernel); + if (module_) vx_module_release(module_); + if (queue) vx_queue_release(queue); + vx_device_release(device); + } +} + +int main(int argc, char* argv[]) { + int c; + while ((c = getopt(argc, argv, "k:h")) != -1) { + if (c == 'k') kernel_file = optarg; + else { std::cout << "Usage: [-k kernel] [-h]" << std::endl; return c == 'h' ? 0 : -1; } + } + + RT_CHECK(vx_device_open(0, &device)); + vx_queue_info_t qi = { sizeof(qi), nullptr, VX_QUEUE_PRIORITY_NORMAL, 0 }; + RT_CHECK(vx_queue_create(device, &qi, &queue)); + + uint64_t num_threads = 0; + RT_CHECK(vx_device_query(device, VX_CAPS_NUM_THREADS, &num_threads)); + const uint32_t num_lanes = (uint32_t)num_threads; // exactly one warp + + // Two scenes in one buffer, 64-B aligned: the empty one (a leaf of zero + // triangles) at 0, the one-triangle one at 128. + const uint32_t kTriOff = 128; + std::vector bytes(kTriOff + VX_BVH_SCENE_HDR_BYTES + VX_BVH_LEAF_HDR_BYTES + + VX_BVH_TRI_STRIDE, 0); + for (uint32_t s = 0; s < 2; ++s) { + uint8_t* base = bytes.data() + (s ? kTriOff : 0); + uint32_t ntri = s; // empty, then one triangle + uint32_t sh[4] = {VX_BVH_SCENE_HDR_BYTES, VX_BVH_SCENE_KIND, + VX_BVH_SCENE_HDR_BYTES + VX_BVH_LEAF_HDR_BYTES + ntri * VX_BVH_TRI_STRIDE, 1}; + memcpy(base, sh, sizeof(sh)); + uint32_t lh[4] = {VX_BVH_KIND_LEAF_TRI | (ntri << VX_BVH_COUNT_SHIFT), 0, 0, 0}; + memcpy(base + VX_BVH_SCENE_HDR_BYTES, lh, sizeof(lh)); + if (ntri) { + uint8_t* rec = base + VX_BVH_SCENE_HDR_BYTES + VX_BVH_LEAF_HDR_BYTES; + float v[9] = {0.f, 0.f, 5.f, 1.f, 0.f, 5.f, 0.f, 1.f, 5.f}; + uint32_t fl = VX_BVH_TRI_FLAG_OPAQUE; + memcpy(rec, v, sizeof(v)); + memcpy(rec + VX_BVH_TRI_FLAGS_OFFSET, &fl, sizeof(fl)); + } + } + + uint32_t res_size = num_lanes * sizeof(rtu_result_t); + uint64_t scene_addr = 0; + RT_CHECK(vx_buffer_create(device, bytes.size(), VX_MEM_READ, &scene_buffer)); + RT_CHECK(vx_buffer_address(scene_buffer, &scene_addr)); + RT_CHECK(vx_buffer_create(device, res_size, VX_MEM_READ_WRITE, &res_buffer)); + RT_CHECK(vx_buffer_address(res_buffer, &kernel_arg.results_addr)); + kernel_arg.empty_scene_addr = scene_addr; + kernel_arg.tri_scene_addr = scene_addr + kTriOff; + kernel_arg.num_lanes = num_lanes; + + RT_CHECK(vx_enqueue_write(queue, scene_buffer, 0, bytes.data(), bytes.size(), 0, nullptr, nullptr)); + RT_CHECK(vx_module_load_file(device, kernel_file, &module_)); + RT_CHECK(vx_module_get_kernel(module_, "main", &kernel)); + + std::cout << "one warp of " << num_lanes << " lanes; last lane traces alone" << std::endl; + + vx_event_h launch_ev = nullptr, read_ev = nullptr; + { + vx_launch_info_t li = {}; + li.struct_size = sizeof(li); + li.kernel = kernel; + li.args_host = &kernel_arg; + li.args_size = sizeof(kernel_arg); + li.ndim = 1; + li.grid_dim[0] = 1; + li.block_dim[0] = num_lanes; + RT_CHECK(vx_enqueue_launch(queue, &li, 0, nullptr, &launch_ev)); + } + std::vector results(num_lanes); + RT_CHECK(vx_enqueue_read(queue, results.data(), res_buffer, 0, res_size, 1, &launch_ev, &read_ev)); + RT_CHECK(vx_event_wait_value(read_ev, 1, VX_TIMEOUT_INFINITE)); + vx_event_release(read_ev); + vx_event_release(launch_ev); + + int errors = 0; + for (uint32_t i = 0; i < num_lanes; ++i) { + const rtu_result_t& r = results[i]; + bool last = (i == num_lanes - 1); + bool ok = (r.first_status == VX_RT_STS_DONE_MISS) + && (last ? (r.second_status == VX_RT_STS_DONE_HIT && std::fabs(r.second_t - 5.f) < 1e-4f) + : (r.second_status == 0xffffffffu)); + if (!ok) { + std::cout << "lane " << i << ": first=" << r.first_status << " second=" << r.second_status + << " t=" << r.second_t << std::endl; + ++errors; + } + } + + cleanup(); + if (errors != 0) { + std::cout << "FAILED with " << errors << " errors" << std::endl; + return 1; + } + std::cout << "PASSED!" << std::endl; + return 0; +} diff --git a/tests/raytracing/rt_smoke_proc_inst/Makefile b/tests/raytracing/rt_smoke_proc_inst/Makefile new file mode 100644 index 0000000000..0379341561 --- /dev/null +++ b/tests/raytracing/rt_smoke_proc_inst/Makefile @@ -0,0 +1,20 @@ +ROOT_DIR := $(realpath ../../..) +include $(ROOT_DIR)/config.mk + +CONFIGS := $(if $(findstring -DVX_CFG_EXT_RTU_ENABLE,$(CONFIGS)),$(CONFIGS),$(CONFIGS) -DVX_CFG_EXT_RTU_ENABLE) +# CW-BVH4 scene -> build the RTU as a CW-BVH4 walker. +CONFIGS += -DVX_CFG_RTU_BVH_WIDTH=4 + +PROJECT := rt_smoke_proc_inst + +SRC_DIR := $(VORTEX_HOME)/tests/raytracing/$(PROJECT) + +SRCS := $(SRC_DIR)/main.cpp + +VX_SRCS := $(SRC_DIR)/kernel.cpp + +OPTS ?= + +KERNEL_LIB := vortex2 + +include ../common.mk diff --git a/tests/raytracing/rt_smoke_proc_inst/common.h b/tests/raytracing/rt_smoke_proc_inst/common.h new file mode 100644 index 0000000000..1c9918f59a --- /dev/null +++ b/tests/raytracing/rt_smoke_proc_inst/common.h @@ -0,0 +1,59 @@ +// Copyright © 2019-2023 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef _RTU_SMOKE_PROC_INST_COMMON_H_ +#define _RTU_SMOKE_PROC_INST_COMMON_H_ + +#include + +// Mirror of sim/simx/rtu/rtu_bvh.h. +#define VX_BVH_SCENE_KIND 2 // kRtuSceneKindBvh4 +#define VX_BVH_SCENE_HDR_BYTES 16 +#define VX_BVH_LEAF_HDR_BYTES 16 +#define VX_BVH_PROC_AABB_BYTES 24 +#define VX_BVH_INSTANCE_STRIDE 64 +#define VX_BVH_INSTANCE_BLAS_OFF 48 +#define VX_BVH_INSTANCE_CUSTOM_ID 52 +#define VX_BVH_INSTANCE_ID_OFFSET 56 +#define VX_BVH_INSTANCE_CULL_MASK 60 +#define VX_BVH_KIND_LEAF_INST 2 +#define VX_BVH_KIND_LEAF_PROC 3 +#define VX_BVH_COUNT_SHIFT 8 + +// Object-space unit sphere the IS intersects, shared by every instance. +#define RTU_SPHERE_CZ 5.0f +#define RTU_SPHERE_R 1.0f + +#define NUM_RAYS 2 + +typedef struct { + uint32_t status; // terminal status + float hit_t; // committed hit distance + uint32_t hit_inst; // committed gl_InstanceID + uint32_t hit_cust; // committed gl_InstanceCustomIndexEXT + uint32_t is_calls; // YIELD_PROC candidates the IS saw + uint32_t is_inst; // gl_InstanceID as the IS saw it + uint32_t is_cust; // gl_InstanceCustomIndexEXT as the IS saw it + float obj_ray[6]; // object-space ray the IS saw (diagnostics) +} rtu_result_t; + +typedef struct { + uint64_t scene_addr; + uint64_t results_addr; + float ray_origin[NUM_RAYS][3]; + float ray_direction[3]; + float tmin; + float tmax; +} kernel_arg_t; + +#endif // _RTU_SMOKE_PROC_INST_COMMON_H_ diff --git a/tests/raytracing/rt_smoke_proc_inst/kernel.cpp b/tests/raytracing/rt_smoke_proc_inst/kernel.cpp new file mode 100644 index 0000000000..41785c2e73 --- /dev/null +++ b/tests/raytracing/rt_smoke_proc_inst/kernel.cpp @@ -0,0 +1,72 @@ +// Copyright © 2019-2023 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Procedural candidates inside instances: the intersection shader must see the +// candidate's own gl_InstanceID / gl_InstanceCustomIndexEXT, and the committed +// hit must carry them too. One ray per lane, each through a different instance. + +#include +#include +#include "common.h" + +__kernel void kernel_main(kernel_arg_t* arg) { + uint32_t tid = threadIdx.x; + if (tid >= NUM_RAYS) return; + + vx_ray_t ray = { + { arg->ray_origin[tid][0], arg->ray_origin[tid][1], arg->ray_origin[tid][2] }, + { arg->ray_direction[0], arg->ray_direction[1], arg->ray_direction[2] }, + arg->tmin, arg->tmax + }; + + uint32_t h = vx_rt_wtrace((uint32_t)arg->scene_addr, 0u, 0u, 0xffu, &ray); + vx_hit_t hit; + uint32_t sts = vx_rt_wait(h, &hit); + uint32_t is_calls = 0, is_inst = ~0u, is_cust = ~0u; + vx_objray_t o = {}; + while (vx_rt_sts_is_yield(sts)) { + uint32_t action = VX_RT_CB_IGNORE; + float hit_t = 0.0f; + if (sts == VX_RT_STS_YIELD_PROC) { + ++is_calls; + is_inst = hit.instance_id; + is_cust = hit.instance_custom; + vx_rt_get_objray(&o); + // |o + t d - C|^2 = r^2 with C = (0,0,CZ) + float ocz = o.origin[2] - RTU_SPHERE_CZ; + float a = o.dir[0]*o.dir[0] + o.dir[1]*o.dir[1] + o.dir[2]*o.dir[2]; + float b = 2.0f * (o.origin[0]*o.dir[0] + o.origin[1]*o.dir[1] + ocz*o.dir[2]); + float c = o.origin[0]*o.origin[0] + o.origin[1]*o.origin[1] + ocz*ocz + - RTU_SPHERE_R*RTU_SPHERE_R; + float disc = b*b - 4.0f*a*c; + if (disc >= 0.0f) { + hit_t = (-b - __builtin_sqrtf(disc)) / (2.0f * a); + action = VX_RT_CB_ACCEPT; + } + } + sts = vx_rt_continue(h, action, hit_t, 0u, &hit); + } + + rtu_result_t* r = (rtu_result_t*)((uintptr_t)arg->results_addr) + tid; + r->status = sts; + r->hit_t = hit.t; + r->hit_inst = hit.instance_id; + r->hit_cust = hit.instance_custom; + r->is_calls = is_calls; + r->is_inst = is_inst; + r->is_cust = is_cust; + for (int i = 0; i < 3; ++i) { + r->obj_ray[i] = o.origin[i]; + r->obj_ray[3 + i] = o.dir[i]; + } +} diff --git a/tests/raytracing/rt_smoke_proc_inst/main.cpp b/tests/raytracing/rt_smoke_proc_inst/main.cpp new file mode 100644 index 0000000000..539a8d3aa1 --- /dev/null +++ b/tests/raytracing/rt_smoke_proc_inst/main.cpp @@ -0,0 +1,195 @@ +// Copyright © 2019-2023 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// PRISM RTU smoke: procedural primitives inside instances. +// +// A TLAS leaf holds two instances of ONE procedural BLAS (a unit sphere at +// object-space (0,0,5)), translated to x=-3 (id 5, custom 0xa5) and x=+3 +// (id 9, custom 0xa9). Two lanes of one warp each fire a +z ray through their +// own instance. Each lane's intersection shader must see the candidate's +// gl_InstanceID / gl_InstanceCustomIndexEXT, and the committed hit must carry +// them: an IS that indexes per-instance data by instance id (every sphere of a +// "Ray Tracing in One Weekend" scene does) reads the wrong sphere otherwise. +// The ids are non-zero on purpose, so a never-written register can't pass. +// +// Scene layout (200 B): +// + 0 VxBvhSceneHeader { root_offset=16, scene_kind=2 } +// + 16 VxBvhLeafHeader { kind=LeafInst|(2<<8) } +// + 32 VxBvhInstance id 5, translate (-3,0,0) -> blas_off=160 +// + 96 VxBvhInstance id 9, translate (+3,0,0) -> blas_off=160 +// +160 VxBvhLeafHeader { kind=LeafProc|(1<<8) } +// +176 VxBvhProcAabb { min=(-1,-1,4), max=(1,1,6) } +// +// Expected per lane: DONE_HIT, t=4, one IS call, IS and hit ids = the lane's +// instance. + +#include +#include +#include + +#include +#include +#include "common.h" + +#define RT_CHECK(_expr) \ + do { \ + int _ret = _expr; \ + if (0 == _ret) break; \ + printf("Error: '%s' returned %d!\n", #_expr, (int)_ret); \ + cleanup(); \ + exit(-1); \ + } while (false) + +const char* kernel_file = "kernel.vxbin"; + +vx_device_h device = nullptr; +vx_buffer_h scene_buffer = nullptr; +vx_buffer_h res_buffer = nullptr; +vx_queue_h queue = nullptr; +vx_module_h module_ = nullptr; +vx_kernel_h kernel = nullptr; +kernel_arg_t kernel_arg = {}; + +void cleanup() { + if (device) { + if (scene_buffer) vx_buffer_release(scene_buffer); + if (res_buffer) vx_buffer_release(res_buffer); + if (kernel) vx_kernel_release(kernel); + if (module_) vx_module_release(module_); + if (queue) vx_queue_release(queue); + vx_device_release(device); + } +} + +static const uint32_t kInstId[NUM_RAYS] = { 5, 9 }; +static const uint32_t kInstCust[NUM_RAYS] = { 0xa5, 0xa9 }; +static const float kInstTx[NUM_RAYS] = { -3.f, 3.f }; + +// Identity rotation + translation (tx,0,0); the record holds its inverse +// (world->object). +static void emit_instance(uint8_t* out, float tx, uint32_t blas_off, + uint32_t custom_id, uint32_t instance_id) { + float* x = reinterpret_cast(out); + x[0] = 1.f; x[1] = 0.f; x[2] = 0.f; x[3] = -tx; + x[4] = 0.f; x[5] = 1.f; x[6] = 0.f; x[7] = 0.f; + x[8] = 0.f; x[9] = 0.f; x[10] = 1.f; x[11] = 0.f; + *reinterpret_cast(out + VX_BVH_INSTANCE_BLAS_OFF) = blas_off; + *reinterpret_cast(out + VX_BVH_INSTANCE_CUSTOM_ID) = custom_id; + *reinterpret_cast(out + VX_BVH_INSTANCE_ID_OFFSET) = instance_id; + *reinterpret_cast(out + VX_BVH_INSTANCE_CULL_MASK) = 0xffu; +} + +int main(int /*argc*/, char* /*argv*/[]) { + RT_CHECK(vx_device_open(0, &device)); + vx_queue_info_t qi = { sizeof(qi), nullptr, VX_QUEUE_PRIORITY_NORMAL, 0 }; + RT_CHECK(vx_queue_create(device, &qi, &queue)); + + const uint32_t blas_off = 32 + NUM_RAYS * VX_BVH_INSTANCE_STRIDE; // 160 + std::vector scene(blas_off + VX_BVH_LEAF_HDR_BYTES + VX_BVH_PROC_AABB_BYTES, 0); + + uint32_t* sh = reinterpret_cast(scene.data()); + sh[0] = VX_BVH_SCENE_HDR_BYTES; // root_node_offset = 16 + sh[1] = VX_BVH_SCENE_KIND; // = 2 (BVH4) + sh[2] = (uint32_t)scene.size(); // total scene bytes (pre-fetch) + sh[3] = 2; // leaf_count (1 inst leaf + 1 proc leaf) + + uint32_t* rlh = reinterpret_cast(scene.data() + VX_BVH_SCENE_HDR_BYTES); + rlh[0] = VX_BVH_KIND_LEAF_INST | ((uint32_t)NUM_RAYS << VX_BVH_COUNT_SHIFT); + for (int i = 0; i < NUM_RAYS; ++i) { + emit_instance(scene.data() + 32 + i * VX_BVH_INSTANCE_STRIDE, + kInstTx[i], blas_off, kInstCust[i], kInstId[i]); + } + + uint32_t* blh = reinterpret_cast(scene.data() + blas_off); + blh[0] = VX_BVH_KIND_LEAF_PROC | (1u << VX_BVH_COUNT_SHIFT); + float* aabb = reinterpret_cast(scene.data() + blas_off + VX_BVH_LEAF_HDR_BYTES); + aabb[0] = -1.f; aabb[1] = -1.f; aabb[2] = 4.f; // min + aabb[3] = 1.f; aabb[4] = 1.f; aabb[5] = 6.f; // max + + RT_CHECK(vx_buffer_create(device, (uint32_t)scene.size(), VX_MEM_READ, &scene_buffer)); + RT_CHECK(vx_buffer_address(scene_buffer, &kernel_arg.scene_addr)); + + const uint32_t res_size = NUM_RAYS * sizeof(rtu_result_t); + RT_CHECK(vx_buffer_create(device, res_size, VX_MEM_WRITE, &res_buffer)); + RT_CHECK(vx_buffer_address(res_buffer, &kernel_arg.results_addr)); + + for (int i = 0; i < NUM_RAYS; ++i) { + kernel_arg.ray_origin[i][0] = kInstTx[i]; + kernel_arg.ray_origin[i][1] = 0.f; + kernel_arg.ray_origin[i][2] = 0.f; + } + kernel_arg.ray_direction[0] = 0.f; + kernel_arg.ray_direction[1] = 0.f; + kernel_arg.ray_direction[2] = 1.f; + kernel_arg.tmin = 0.001f; + kernel_arg.tmax = 1e30f; + + std::cout << "scene_addr=0x" << std::hex << kernel_arg.scene_addr << std::dec + << " bvh4 (2 instances of 1 leaf_proc sphere)" << std::endl; + + RT_CHECK(vx_enqueue_write(queue, scene_buffer, 0, scene.data(), + (uint32_t)scene.size(), 0, nullptr, nullptr)); + RT_CHECK(vx_module_load_file(device, kernel_file, &module_)); + RT_CHECK(vx_module_get_kernel(module_, "main", &kernel)); + + std::cout << "launch kernel" << std::endl; + vx_event_h launch_ev = nullptr, read_ev = nullptr; + { + vx_launch_info_t li = {}; + li.struct_size = sizeof(li); + li.kernel = kernel; + li.args_host = &kernel_arg; + li.args_size = sizeof(kernel_arg); + li.ndim = 1; + li.grid_dim[0] = 1; + li.block_dim[0] = NUM_RAYS; // both rays in one warp + RT_CHECK(vx_enqueue_launch(queue, &li, 0, nullptr, &launch_ev)); + } + + rtu_result_t res[NUM_RAYS] = {}; + RT_CHECK(vx_enqueue_read(queue, res, res_buffer, 0, res_size, 1, &launch_ev, &read_ev)); + RT_CHECK(vx_event_wait_value(read_ev, 1, VX_TIMEOUT_INFINITE)); + vx_event_release(read_ev); + vx_event_release(launch_ev); + + int errors = 0; + for (int i = 0; i < NUM_RAYS; ++i) { + const rtu_result_t& r = res[i]; + std::cout << "lane " << i << ": status=" << r.status << " t=" << r.hit_t + << " hit_inst=" << r.hit_inst << " hit_cust=0x" << std::hex << r.hit_cust + << std::dec << " is_calls=" << r.is_calls << " is_inst=" << r.is_inst + << " is_cust=0x" << std::hex << r.is_cust << std::dec + << " obj_ray=(" << r.obj_ray[0] << "," << r.obj_ray[1] << "," << r.obj_ray[2] + << ")+(" << r.obj_ray[3] << "," << r.obj_ray[4] << "," << r.obj_ray[5] << ")" + << std::endl; + bool ok = (r.status == VX_RT_STS_DONE_HIT) + && (std::fabs(r.hit_t - 4.f) <= 1e-4f) + && (r.is_calls == 1) + && (r.is_inst == kInstId[i]) && (r.is_cust == kInstCust[i]) + && (r.hit_inst == kInstId[i]) && (r.hit_cust == kInstCust[i]); + if (!ok) { + std::cout << " expected: status=" << VX_RT_STS_DONE_HIT << " t=4 is_calls=1" + << " inst=" << kInstId[i] << " cust=0x" << std::hex << kInstCust[i] + << std::dec << " (IS and hit)" << std::endl; + ++errors; + } + } + + cleanup(); + if (errors != 0) { + std::cout << "FAILED with " << errors << " errors" << std::endl; + return 1; + } + std::cout << "PASSED!" << std::endl; + return 0; +} diff --git a/tests/raytracing/rt_smoke_tlas/main.cpp b/tests/raytracing/rt_smoke_tlas/main.cpp index fb8cf1bab8..aa7d38a850 100644 --- a/tests/raytracing/rt_smoke_tlas/main.cpp +++ b/tests/raytracing/rt_smoke_tlas/main.cpp @@ -87,10 +87,11 @@ int main(int /*argc*/, char* /*argv*/[]) { uint8_t* inst = scene_bytes.data() + RTU_SCENE_HDR_BYTES + idx * RTU_INSTANCE_STRIDE; float* xform = reinterpret_cast(inst); - // 3x4 affine row-major; identity R + translation t=(0,0,tz). + // 3x4 world->object affine row-major: the inverse of translation + // (0,0,tz). xform[0] = 1.f; xform[1] = 0.f; xform[2] = 0.f; xform[3] = 0.f; xform[4] = 0.f; xform[5] = 1.f; xform[6] = 0.f; xform[7] = 0.f; - xform[8] = 0.f; xform[9] = 0.f; xform[10] = 1.f; xform[11] = tz; + xform[8] = 0.f; xform[9] = 0.f; xform[10] = 1.f; xform[11] = -tz; uint32_t* inst_tail = reinterpret_cast( inst + RTU_INSTANCE_BLAS_OFF_OFF); inst_tail[0] = kBlasOff; // shared inline BLAS diff --git a/tests/regression/diverge_loop/Makefile b/tests/regression/diverge_loop/Makefile new file mode 100644 index 0000000000..4559039b6e --- /dev/null +++ b/tests/regression/diverge_loop/Makefile @@ -0,0 +1,16 @@ +ROOT_DIR := $(realpath ../../..) +include $(ROOT_DIR)/config.mk + +PROJECT := diverge_loop + +SRC_DIR := $(VORTEX_HOME)/tests/regression/$(PROJECT) + +SRCS := $(SRC_DIR)/main.cpp + +VX_SRCS := $(SRC_DIR)/kernel.cpp + +OPTS ?= -n256 + +KERNEL_LIB := vortex2 + +include ../common.mk \ No newline at end of file diff --git a/tests/regression/diverge_loop/common.h b/tests/regression/diverge_loop/common.h new file mode 100644 index 0000000000..cf485d40d1 --- /dev/null +++ b/tests/regression/diverge_loop/common.h @@ -0,0 +1,20 @@ +#ifndef _COMMON_H_ +#define _COMMON_H_ + +#include + +#ifndef TYPE +#define TYPE float +#endif + +typedef struct { + uint32_t num_points; + uint32_t num_samples; + uint32_t num_shadows; + uint64_t t_addr; + uint64_t color_addr; + uint64_t occl_addr; + uint64_t dst_addr; +} kernel_arg_t; + +#endif diff --git a/tests/regression/diverge_loop/kernel.cpp b/tests/regression/diverge_loop/kernel.cpp new file mode 100644 index 0000000000..558367613c --- /dev/null +++ b/tests/regression/diverge_loop/kernel.cpp @@ -0,0 +1,35 @@ +#include +#include "common.h" + +// A divergent early `continue` out of a loop body whose other path carries its +// own loop, both feeding one accumulator that is live around the outer loop: +// the reconvergence shape of a ray-tracing raygen shader (a missed ray adds the +// sky and continues, a hit one first attenuates by its shadow rays). +__kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { + auto t_ptr = reinterpret_cast(arg->t_addr); + auto color_ptr = reinterpret_cast(arg->color_addr); + auto occl_ptr = reinterpret_cast(arg->occl_addr); + auto dst_ptr = reinterpret_cast(arg->dst_addr); + uint32_t idx = blockIdx.x * blockDim.x + threadIdx.x; + if (idx >= arg->num_points) + return; + + float acc = 0.0f; + for (uint32_t s = 0; s < arg->num_samples; ++s) { + float t = t_ptr[idx * arg->num_samples + s]; + float h = color_ptr[idx]; + if (t < 0.0f) { + acc += h; + continue; + } + for (uint32_t j = 0; j < arg->num_shadows; ++j) { + float occluded = 1.0f; + if (occl_ptr[idx] > 0.5f) + occluded = t - occl_ptr[idx]; + if (occluded > 0.0f) + h *= 0.3f; + } + acc += h; + } + dst_ptr[idx] = acc; +} diff --git a/tests/regression/diverge_loop/main.cpp b/tests/regression/diverge_loop/main.cpp new file mode 100644 index 0000000000..f6c05dd374 --- /dev/null +++ b/tests/regression/diverge_loop/main.cpp @@ -0,0 +1,168 @@ +// Copyright © 2019-2023 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 + +// diverge_loop — divergent continue + inner loop reconverging on one accumulator. +// +// Async pattern: uploads are fire-and-forget; the launch produces an event; +// the dst readback gates on that event; the host waits once at the end. +// The per-queue worker serializes ops in FIFO order, so no inter-step host sync is needed. + +#include +#include "common.h" + +#include +#include +#include +#include +#include +#include +#include + +#define CHECK(expr) do { \ + vx_result_t _r = (expr); \ + if (_r != VX_SUCCESS) { \ + std::fprintf(stderr, "FAIL %s:%d: '%s' returned %s\n", \ + __FILE__, __LINE__, #expr, vx_result_string(_r)); \ + std::exit(1); \ + } \ +} while (0) + +namespace { +const char* kernel_file = "kernel.vxbin"; +uint32_t size = 16; + +void parse_args(int argc, char** argv) { + int c; + while ((c = getopt(argc, argv, "n:k:h")) != -1) { + switch (c) { + case 'n': size = std::atoi(optarg); break; + case 'k': kernel_file = optarg; break; + default: + std::cout << "Usage: [-k kernel] [-n words] [-h]" << std::endl; + std::exit(c == 'h' ? 0 : -1); + } + } +} + +bool float_eq(float a, float b) { + union fi { float f; int32_t i; }; + fi fa{a}, fb{b}; + return std::abs(fa.i - fb.i) <= 6; +} +} // namespace + +int main(int argc, char** argv) { + parse_args(argc, argv); + std::srand(50); + + const uint32_t num_points = size; + const uint64_t buf_size = num_points * sizeof(TYPE); + const uint32_t num_samples = 2, num_shadows = 2; + std::cout << "diverge_loop vortex2: n=" << num_points << std::endl; + + vx_device_h dev = nullptr; + CHECK(vx_device_open(0, &dev)); + + vx_queue_info_t qi = { sizeof(qi), nullptr, VX_QUEUE_PRIORITY_NORMAL, 0 }; + vx_queue_h q = nullptr; + CHECK(vx_queue_create(dev, &qi, &q)); + + vx_buffer_h t_buf=nullptr, src0_buf=nullptr, src1_buf=nullptr, dst_buf=nullptr; + CHECK(vx_buffer_create(dev, buf_size * num_samples, VX_MEM_READ, &t_buf)); + CHECK(vx_buffer_create(dev, buf_size, VX_MEM_READ, &src0_buf)); + CHECK(vx_buffer_create(dev, buf_size, VX_MEM_READ, &src1_buf)); + CHECK(vx_buffer_create(dev, buf_size, VX_MEM_WRITE, &dst_buf)); + + vx_module_h mod = nullptr; + vx_kernel_h kern = nullptr; + CHECK(vx_module_load_file(dev, kernel_file, &mod)); + CHECK(vx_module_get_kernel(mod, "main", &kern)); + + kernel_arg_t kernel_arg{}; + kernel_arg.num_points = num_points; + kernel_arg.num_samples = num_samples; + kernel_arg.num_shadows = num_shadows; + CHECK(vx_buffer_address(t_buf, &kernel_arg.t_addr)); + CHECK(vx_buffer_address(src0_buf, &kernel_arg.color_addr)); + CHECK(vx_buffer_address(src1_buf, &kernel_arg.occl_addr)); + CHECK(vx_buffer_address(dst_buf, &kernel_arg.dst_addr)); + + // Every divergence pattern within a warp: a lane's samples miss or hit + // independently, and a hit lane is occluded or not. + std::vector h_t(num_points * num_samples), h_src0(num_points), + h_src1(num_points), h_dst(num_points); + for (uint32_t i = 0; i < num_points; ++i) { + for (uint32_t s = 0; s < num_samples; ++s) + h_t[i * num_samples + s] = ((i >> s) & 1) ? -1.0f : 10.0f + i; + h_src0[i] = 1.0f + static_cast(std::rand()) / RAND_MAX; + h_src1[i] = ((i >> 2) & 1) ? 1.0f + static_cast(std::rand()) / RAND_MAX : 0.0f; + } + + // ----- Async chain: 2 writes → launch → read → 1 wait ----- + // The kernel-args block is passed as a host blob — no args device buffer needed. + CHECK(vx_enqueue_write(q, t_buf, 0, h_t.data(), buf_size * num_samples, 0,nullptr,nullptr)); + CHECK(vx_enqueue_write(q, src0_buf, 0, h_src0.data(), buf_size, 0,nullptr,nullptr)); + CHECK(vx_enqueue_write(q, src1_buf, 0, h_src1.data(), buf_size, 0,nullptr,nullptr)); + + uint32_t grid[1], block[1]; + CHECK(vx_device_max_occupancy_grid(dev, 1, &num_points, grid, block)); + + vx_launch_info_t li{}; + li.struct_size = sizeof(li); + li.kernel = kern; + li.args_host = &kernel_arg; + li.args_size = sizeof(kernel_arg); + li.ndim = 1; + li.grid_dim[0] = grid[0]; + li.block_dim[0]= block[0]; + + vx_event_h launch_ev=nullptr, read_ev=nullptr; + CHECK(vx_enqueue_launch(q, &li, 0, nullptr, &launch_ev)); + CHECK(vx_enqueue_read(q, h_dst.data(), dst_buf, 0, buf_size, + 1, &launch_ev, &read_ev)); + CHECK(vx_event_wait_value(read_ev, 1, VX_TIMEOUT_INFINITE)); + + int errors = 0; + for (uint32_t i = 0; i < num_points; ++i) { + TYPE ref = 0.0f; + for (uint32_t s = 0; s < num_samples; ++s) { + float t = h_t[i * num_samples + s]; + float h = h_src0[i]; + if (t < 0.0f) { ref += h; continue; } + for (uint32_t j = 0; j < num_shadows; ++j) { + float occluded = 1.0f; + if (h_src1[i] > 0.5f) occluded = t - h_src1[i]; + if (occluded > 0.0f) h *= 0.3f; + } + ref += h; + } + if (!float_eq(h_dst[i], ref)) { + if (errors < 16) + std::printf("*** [%u] expected=%f actual=%f\n", i, ref, h_dst[i]); + ++errors; + } + } + + vx_event_release(read_ev); + vx_event_release(launch_ev); + vx_buffer_release(dst_buf); + vx_buffer_release(src1_buf); + vx_buffer_release(src0_buf); + vx_buffer_release(t_buf); + vx_kernel_release(kern); + vx_module_release(mod); + vx_queue_release(q); + vx_device_dump_perf(dev, stdout); + vx_device_release(dev); + + if (errors) { + std::cout << "Found " << errors << " errors!\nFAILED!" << std::endl; + return 1; + } + std::cout << "PASSED!" << std::endl; + return 0; +}