perf(codegen,runtime): polymorphic key-add sites are served inline from hashed home ways

changelog.d/11436-keyadd-poly.md

@@ -0,0 +1,14 @@
+Polymorphic key-adding stores are now served inline. A store site that holds
+several key-add memos (one per receiver pre-shape, e.g. a base-class
+constructor seeing each subclass) places each further memo at its pre-shape's
+home way, a hash of the ShapeId, and the emitted hit compares that one way
+after the primary memo: one extra compare whatever the number of shapes. The
+key-add memo is also compared before the existing-key ways, the per-object
+header checks are one test on the hot path, and a key-add on a typed-layout
+receiver (an object literal) no longer takes the typed-feedback registry lock
+when typed feedback is off, which cut about 180 instructions from each such
+add.
+A memo the runtime serves from beyond the two ways the emitted hit compares
+(its home was taken by an earlier, often transient, shape) now moves into one
+of them, so a site's hot shapes end up served inline; on tsc this cut
+runtime-served key-adds per transpile from 406,464 to 152,424.

crates/perry-codegen/src/expr/store_census.rs

@@ -40,6 +40,9 @@ pub(crate) const BY_NAME_RUNTIME: usize = 11;
 pub(crate) const BY_NAME_PUT_VALUE: usize = 12;
 /// Key-add inline hit that first retired the receiver's layout record.
 pub(crate) const ADD_LAYOUT_FORGET: usize = 13;
+/// Key-add inline hit on one of the runtime block's first ways (counted on
+/// its own edge, then also as [`ADD_HIT`]).
+pub(crate) const ADD_WAY_HIT: usize = 14;
 
 pub(crate) fn enabled() -> bool {
     static ON: std::sync::OnceLock<bool> = std::sync::OnceLock::new();

crates/perry-codegen/tests/native_proof_regressions.rs

@@ -15433,6 +15433,38 @@ fn static_put_value_uses_write_pic_for_call_free_rhs() {
         ir.contains("put.add.check") && ir.contains("put.add.hit.store"),
         "a word and way miss must compare the key-add memo before the call:\n{ir}"
     );
+    // The add memo's primary pre-shape is compared right after the word, and
+    // only its miss reaches the existing-key ways.
+    let check_block: Vec<&str> = ir
+        .lines()
+        .skip_while(|l| !(l.starts_with("put.add.check") && l.trim_end().ends_with(':')))
+        .skip(1)
+        .take_while(|l| !l.trim_end().ends_with(':'))
+        .collect();
+    let check_br = check_block
+        .iter()
+        .find(|l| l.contains(" br "))
+        .copied()
+        .unwrap_or("");
+    assert!(
+        check_br.contains("%put.add.chain") && check_br.contains("%put.pic.ways"),
+        "the key-add primary compare must branch to the add hit or the existing-key ways:\n{ir}"
+    );
+    // After the primary memo, TWO key-add ways: the receiver ShapeId's home,
+    // `(sid * 0x9E3779B1) >> 26` (packed_add::add_way_home), and the next.
+    let way_blocks: Vec<&str> = ir
+        .lines()
+        .filter(|l| l.starts_with("put.add.way.") && l.trim_end().ends_with(':'))
+        .collect();
+    assert_eq!(
+        way_blocks.len(),
+        2,
+        "after the primary memo the home way and the next are compared inline:\n{ir}"
+    );
+    assert!(
+        ir.contains("mul i32") && ir.contains("-1640531535") && ir.contains("lshr i32"),
+        "the inline way is the ShapeId's home, sid * ADD_WAY_HASH >> 26:\n{ir}"
+    );
     assert_eq!(
         ir.lines()
             .filter(|l| l.starts_with("put.pic.way.") && l.trim_end().ends_with(':'))

crates/perry-runtime/src/proxy/put_value/packed_add.rs

@@ -95,27 +95,59 @@ pub struct PackedSetSite {
     /// A `*mut AddWays` (0 = none): the memos of further pre-shapes, served
     /// by [`packed_add_try`]. A base-class constructor's key-add sees one
     /// pre-shape per subclass (the prototype is part of the shape), so such
-    /// a site is polymorphic by construction. Emitted code never reads it.
+    /// a site is polymorphic by construction. The emitted hit compares the
+    /// ways at the pre-shape's home ([`add_way_home`]) and the one after it,
+    /// after the primary words.
     pub add_ways: AtomicU64,
 }
 
-/// One further memo, in the primary words' format.
+/// One further memo, in the primary words' format (`add_shapes`,
+/// `add_guard` are a way too: the emitted hit reads either through one
+/// pointer).
 #[repr(C)]
 pub struct AddWay {
     shapes: AtomicU64,
     guard: AtomicU64,
 }
 
-/// Further memos per site. Filled in order, never evicted (so a site with
-/// more stable pre-shapes than ways settles instead of cycling); a site that
-/// overflows them re-primes its primary words. 48, not 8: Zod 3's `ZodType`
+/// Further memos per site, never evicted (so a site with more stable
+/// pre-shapes than ways settles instead of cycling); a site that overflows
+/// them re-primes its primary words. A memo is placed at its pre-shape's HOME
+/// way ([`add_way_home`]) when that way is free, and otherwise at the next
+/// free way from it; the emitted hit compares the home and the way after it
+/// ([`ADD_WAY_PROBES`]), the runtime every way, and a memo the runtime serves
+/// from further away is moved into one of the two ([`promote_way`]). Placement by pre-shape rather
+/// than by arrival matters: on
+/// tsc the hot memo of a polymorphic site is typically NOT among its first
+/// (8 sites whose hits all land on their 3rd way, 10 on their 15th, behind
+/// transient first-instance shapes), so no fixed prefix of an in-order list
+/// is where the hits are.
+///
+/// 64 (a power of two for the home hash), not 8: Zod 3's `ZodType`
 /// constructor adds its keys to one pre-shape per subclass (36 of them), and
 /// with 8 ways 15,069 of its 78,250 executed key-adds per 200 parses re-ran
-/// the full `[[Set]]` and re-primed. A linear scan of 48 words is a small
-/// fraction of that walk, and only a polymorphic site allocates them.
-pub const ADD_WAYS: usize = 48;
+/// the full `[[Set]]` and re-primed. Only a polymorphic site allocates them.
+pub const ADD_WAYS: usize = 1 << ADD_WAYS_LOG2;
+/// `log2(ADD_WAYS)`: the home is the top bits of a 32-bit product.
+pub const ADD_WAYS_LOG2: u32 = 6;
+/// The multiplier of [`add_way_home`] (2^32 / golden ratio): consecutive
+/// ShapeIds, which subclass shapes minted in sequence are, land far apart.
+pub const ADD_WAY_HASH: u32 = 0x9E37_79B1;
+/// Ways the emitted hit compares from the home on (the home, then the next
+/// mod [`ADD_WAYS`]): a memo whose home an earlier memo holds lands on the
+/// next free way, which is most often the very next.
+#[cfg_attr(not(test), allow(dead_code))]
+pub const ADD_WAY_PROBES: usize = 2;
 type AddWays = [AddWay; ADD_WAYS];
 
+/// The way the emitted hit compares for a receiver of ShapeId `pre`:
+/// the top [`ADD_WAYS_LOG2`] bits of `pre * ADD_WAY_HASH` (mod 2^32).
+/// **perry-codegen computes the same (`emit_static_store_ic`).**
+#[inline]
+pub fn add_way_home(pre: u32) -> usize {
+    (pre.wrapping_mul(ADD_WAY_HASH) >> (32 - ADD_WAYS_LOG2)) as usize
+}
+
 impl PackedSetSite {
     pub const fn empty() -> Self {
         Self {
@@ -134,6 +166,11 @@ pub const ADD_SHAPES_WORD: usize = 1;
 pub const ADD_GUARD_WORD: usize = 2;
 #[cfg_attr(not(test), allow(dead_code))]
 pub const PACKED_SET_SITE_WORDS: usize = 4;
+#[cfg_attr(not(test), allow(dead_code))]
+pub const ADD_WAYS_WORD: usize = 3;
+/// Words of one [`AddWay`].
+#[cfg_attr(not(test), allow(dead_code))]
+pub const ADD_WAY_WORDS: usize = 2;
 /// Low bits of the guard word that hold the slot.
 pub const ADD_SLOT_BITS: u32 = 16;
 const ADD_SLOT_MASK: u64 = (1 << ADD_SLOT_BITS) - 1;
@@ -263,7 +300,7 @@ const CENSUS_NAMES: [&str; 32] = [
     "emit.by_name.runtime",
     "emit.by_name.put_value",
     "emit.add.layout_forget",
-    "emit.14",
+    "emit.add.way_hit",
     "emit.15",
     "rt.add.memo_inline",
     "rt.add.memo_spill",
@@ -364,13 +401,21 @@ pub(crate) unsafe fn packed_add_try(
     let (shapes, guard) = if matches(primary) {
         (primary, (*site).add_guard.load(Ordering::Relaxed))
     } else {
-        let way = site_ways(site)?
-            .iter()
-            .find(|way| matches(way.shapes.load(Ordering::Relaxed)))?;
-        (
+        let ways = site_ways(site)?;
+        // A memo sits at its home way unless that was taken when it was
+        // placed; the emitted hit has already compared the home.
+        let home = add_way_home(sid);
+        let distance = (0..ADD_WAYS)
+            .find(|&i| matches(ways[(home + i) % ADD_WAYS].shapes.load(Ordering::Relaxed)))?;
+        let way = &ways[(home + distance) % ADD_WAYS];
+        let found = (
             way.shapes.load(Ordering::Relaxed),
             way.guard.load(Ordering::Relaxed),
-        )
+        );
+        if distance >= ADD_WAY_PROBES {
+            promote_way(ways, home, distance);
+        }
+        found
     };
     let spill = shapes as u32 != sid;
     if guard >> ADD_SLOT_BITS != add_generation() {
@@ -421,6 +466,51 @@ pub(crate) unsafe fn packed_add_try(
     Some(value)
 }
 
+/// A memo the runtime just served lies beyond the ways the emitted hit
+/// compares (its home and the next were taken when it was placed, typically
+/// by a polymorphic site's transient first-instance shapes). Move it into
+/// one of those two ways, so its next receiver is served inline, and move
+/// that way's memo to where it was. A way whose memo sits at its OWN home is
+/// kept (its receivers are served inline already); with both kept nothing
+/// moves. Every memo stays in the block, so the runtime still serves each.
+///
+/// Only the primary agent publishes a site's memos (see `# Agents`), and only
+/// it ever matches them, so the moves are ordered with its own reads. Each
+/// way is retired (`shapes` EMPTY) before its guard changes and republished
+/// last, as [`packed_add_prime`] does.
+fn promote_way(ways: &AddWays, home: usize, distance: usize) {
+    if crate::agent::current_agent() != crate::agent::PRIMARY_AGENT {
+        return;
+    }
+    let from = (home + distance) % ADD_WAYS;
+    let shapes = ways[from].shapes.load(Ordering::Relaxed);
+    if shapes as u32 != unflip(shapes as u32) {
+        // A spill memo: the emitted hit never takes it, wherever it sits.
+        return;
+    }
+    let at_own_home = |idx: usize| {
+        let word = ways[idx].shapes.load(Ordering::Relaxed);
+        word != PACKED_SET_EMPTY && add_way_home(unflip(word as u32)) == idx
+    };
+    let second = (home + 1) % ADD_WAYS;
+    let Some(to) = [second, home].into_iter().find(|&idx| !at_own_home(idx)) else {
+        return;
+    };
+    let (to_shapes, to_guard) = (
+        ways[to].shapes.load(Ordering::Relaxed),
+        ways[to].guard.load(Ordering::Relaxed),
+    );
+    let guard = ways[from].guard.load(Ordering::Relaxed);
+    ways[from].shapes.store(PACKED_SET_EMPTY, Ordering::Relaxed);
+    ways[to].shapes.store(PACKED_SET_EMPTY, Ordering::Relaxed);
+    ways[to].guard.store(guard, Ordering::Relaxed);
+    ways[to].shapes.store(shapes, Ordering::Relaxed);
+    if to_shapes != PACKED_SET_EMPTY {
+        ways[from].guard.store(to_guard, Ordering::Relaxed);
+        ways[from].shapes.store(to_shapes, Ordering::Relaxed);
+    }
+}
+
 /// The key-add hit's layout retirement: a receiver whose layout record
 /// (side mask or typed descriptor) described its PRE-shape. Exactly what the
 /// transition lane runs before its stamp. Edits header bits and layout /
@@ -629,10 +719,14 @@ pub(crate) unsafe fn packed_add_prime(
         }
         let displaced_pre = unflip(primary as u32);
         if let Some(way) = site_ways(site_ptr).and_then(|ways| {
-            ways.iter().find(|way| {
-                let word = way.shapes.load(Ordering::Relaxed);
-                word == PACKED_SET_EMPTY || unflip(word as u32) == displaced_pre
-            })
+            // The home way first, then the rest in order from it.
+            let home = add_way_home(displaced_pre);
+            (0..ADD_WAYS)
+                .map(|i| &ways[(home + i) % ADD_WAYS])
+                .find(|way| {
+                    let word = way.shapes.load(Ordering::Relaxed);
+                    word == PACKED_SET_EMPTY || unflip(word as u32) == displaced_pre
+                })
         }) {
             way.shapes.store(PACKED_SET_EMPTY, Ordering::Relaxed);
             way.guard.store(displaced_guard, Ordering::Relaxed);

crates/perry-runtime/src/proxy/put_value/packed_add_tests.rs

@@ -22,6 +22,36 @@ fn packed_set_site_layout_matches_codegen() {
         8 * ADD_GUARD_WORD
     );
     assert_eq!((ADD_SHAPES_WORD, ADD_GUARD_WORD, ADD_SLOT_BITS), (1, 2, 16));
+    // ADD_WAYS_WORD, ADD_WAY_WORDS, ADD_WAYS_LOG2, ADD_WAY_HASH: the emitted
+    // hit reads way `add_way_home(sid)` at `block + 8 * ADD_WAY_WORDS * i`,
+    // the primary pair through the same pointer arithmetic from
+    // `ADD_SHAPES_WORD`.
+    assert_eq!(
+        std::mem::offset_of!(PackedSetSite, add_ways),
+        8 * ADD_WAYS_WORD
+    );
+    assert_eq!(std::mem::size_of::<AddWay>(), 8 * ADD_WAY_WORDS);
+    assert_eq!(std::mem::offset_of!(AddWay, shapes), 0);
+    assert_eq!(
+        std::mem::offset_of!(AddWay, guard),
+        std::mem::offset_of!(PackedSetSite, add_guard)
+            - std::mem::offset_of!(PackedSetSite, add_shapes)
+    );
+    assert_eq!(
+        (ADD_WAYS_WORD, ADD_WAY_WORDS, ADD_WAYS_LOG2, ADD_WAY_HASH),
+        (3, 2, 6, 0x9E37_79B1)
+    );
+    // ADD_WAY_PROBES: the home and the next way.
+    assert_eq!(ADD_WAY_PROBES, 2);
+    assert_eq!(ADD_WAYS, 1 << ADD_WAYS_LOG2);
+    // The home is a way of the block for every ShapeId.
+    for sid in [0u32, 1, 0x8000_0000, 0x8000_0001, u32::MAX] {
+        assert!(add_way_home(sid) < ADD_WAYS);
+    }
+    assert_eq!(
+        add_way_home(0x8000_0001),
+        (0x8000_0001u32.wrapping_mul(0x9E37_79B1) >> 26) as usize
+    );
     // An empty site's pre half is unmatchable.
     let empty = PackedSetSite::empty();
     assert!(empty.add_shapes.load(Ordering::Relaxed) as u32 >= crate::object::shapes::SHAPE_ID_END);
@@ -164,3 +194,104 @@ fn a_published_memo_owns_both_shapes_across_a_full_trace() {
         "a published memo must own its pre- and post-shape"
     );
 }
+
+/// A polymorphic site's displaced memos sit at their pre-shape's HOME way —
+/// the one way the emitted hit compares — unless that way was already
+/// taken, and the runtime serves every memo wherever it sits. Sabotage:
+/// placement in arrival order (way 0, 1, ...) -> a home way stays empty
+/// while its memo sits elsewhere.
+#[test]
+fn a_displaced_memo_sits_at_its_home_way() {
+    let key = interned(b"added_home");
+    let srcs: [&[u8]; 6] = [
+        b"{\"h0\":1}",
+        b"{\"h1\":1}",
+        b"{\"h2\":1}",
+        b"{\"h3\":1}",
+        b"{\"h4\":1}",
+        b"{\"h5\":1}",
+    ];
+    let site = leaked_site();
+    let mut pres = Vec::new();
+    for src in srcs {
+        let first = parsed(src);
+        pres.push(stamp(first));
+        miss(site, first, key, 1.0);
+    }
+    let ways = unsafe { site_ways(site) }.expect("a polymorphic site has ways");
+    let primary = site.add_shapes.load(Ordering::Relaxed) as u32;
+    assert_eq!(primary, *pres.last().unwrap(), "the newest memo is primary");
+    for &pre in &pres[..pres.len() - 1] {
+        let at_home = ways[add_way_home(pre)].shapes.load(Ordering::Relaxed);
+        assert!(
+            at_home as u32 == pre || at_home != PACKED_SET_EMPTY,
+            "memo {pre:#x}: its home way {} is empty, so it must sit there",
+            add_way_home(pre)
+        );
+    }
+    for (i, src) in srcs.iter().enumerate() {
+        let next = parsed(src);
+        assert_eq!(stamp(next), pres[i]);
+        let served = if pres[i] == primary {
+            Some(2.0)
+        } else {
+            unsafe { packed_add_try(site, next, 2.0) }
+        };
+        assert_eq!(served, Some(2.0), "memo {i} must be served");
+    }
+}
+
+fn fresh_ways() -> Box<AddWays> {
+    Box::new(std::array::from_fn(|_| AddWay {
+        shapes: AtomicU64::new(PACKED_SET_EMPTY),
+        guard: AtomicU64::new(0),
+    }))
+}
+
+/// The first ShapeId at or after `from` whose home is `home`.
+fn sid_with_home(home: usize, from: u32) -> u32 {
+    (from..).find(|&sid| add_way_home(sid) == home).unwrap()
+}
+
+/// A memo the runtime serves from beyond the two ways the emitted hit
+/// compares moves into the second of them, trading places with a memo that
+/// was not at its own home; a way whose memo IS at its own home is kept.
+/// Sabotage: `promote_way` moves nothing -> the hot memo stays out of reach
+/// of the emitted hit (tsc: 8 sites, every hit 2-3 ways from home).
+#[test]
+fn a_far_memo_moves_into_the_inline_ways() {
+    let base = crate::object::shapes::SHAPE_ID_BASE;
+    let h = 10usize;
+    let hot = sid_with_home(h, base);
+    let at_home = sid_with_home(h, hot + 1);
+    let stray = sid_with_home(40, base);
+    let ways = fresh_ways();
+    let word = |sid: u32| u64::from(sid) | (u64::from(sid + 1) << 32);
+    ways[h].shapes.store(word(at_home), Ordering::Relaxed);
+    ways[h].guard.store(1, Ordering::Relaxed);
+    ways[h + 1].shapes.store(word(stray), Ordering::Relaxed);
+    ways[h + 1].guard.store(2, Ordering::Relaxed);
+    ways[h + 3].shapes.store(word(hot), Ordering::Relaxed);
+    ways[h + 3].guard.store(3, Ordering::Relaxed);
+    promote_way(&ways, h, 3);
+    let at = |i: usize| {
+        (
+            ways[i].shapes.load(Ordering::Relaxed) as u32,
+            ways[i].guard.load(Ordering::Relaxed),
+        )
+    };
+    assert_eq!(at(h), (at_home, 1), "a memo at its own home is kept");
+    assert_eq!(
+        at(h + 1),
+        (hot, 3),
+        "the served memo moves in with its guard"
+    );
+    assert_eq!(at(h + 3), (stray, 2), "the displaced memo takes its place");
+    // Both inline ways hold memos at their own homes: nothing moves.
+    let next_home = sid_with_home(h + 1, base);
+    ways[h + 1].shapes.store(word(next_home), Ordering::Relaxed);
+    ways[h + 3].shapes.store(word(hot), Ordering::Relaxed);
+    promote_way(&ways, h, 3);
+    assert_eq!(at(h + 1).0, next_home);
+    assert_eq!(at(h + 3).0, hot);
+}