diff --git a/changelog.d/11436-keyadd-poly.md b/changelog.d/11436-keyadd-poly.md new file mode 100644 index 0000000000..ad40a1c8b3 --- /dev/null +++ b/changelog.d/11436-keyadd-poly.md @@ -0,0 +1,14 @@ +Polymorphic key-adding stores are now served inline. A store site that holds +several key-add memos (one per receiver pre-shape, e.g. a base-class +constructor seeing each subclass) places each further memo at its pre-shape's +home way, a hash of the ShapeId, and the emitted hit compares that one way +after the primary memo: one extra compare whatever the number of shapes. The +key-add memo is also compared before the existing-key ways, the per-object +header checks are one test on the hot path, and a key-add on a typed-layout +receiver (an object literal) no longer takes the typed-feedback registry lock +when typed feedback is off, which cut about 180 instructions from each such +add. +A memo the runtime serves from beyond the two ways the emitted hit compares +(its home was taken by an earlier, often transient, shape) now moves into one +of them, so a site's hot shapes end up served inline; on tsc this cut +runtime-served key-adds per transpile from 406,464 to 152,424. diff --git a/crates/perry-codegen/src/expr/put_value_store_ic.rs b/crates/perry-codegen/src/expr/put_value_store_ic.rs index 17de8138f5..07bb5294f2 100644 --- a/crates/perry-codegen/src/expr/put_value_store_ic.rs +++ b/crates/perry-codegen/src/expr/put_value_store_ic.rs @@ -138,6 +138,22 @@ pub(crate) const PACKED_SET_SITE_WORDS: usize = 4; pub(crate) const ADD_SHAPES_WORD: usize = 1; pub(crate) const ADD_GUARD_WORD: usize = 2; pub(crate) const ADD_SLOT_BITS: u32 = 16; +/// The site word holding the runtime's `*AddWay` block (0 = none), and how +/// the emitted code finds the ways of it that it compares after the primary +/// memo: from the receiver ShapeId's HOME, the top `ADD_WAYS_LOG2` bits of +/// `sid * ADD_WAY_HASH` (mod 2^32). A way is two words in the primary pair's +/// format, `{shapes, guard}`, so the site's words +/// `ADD_SHAPES_WORD..=ADD_GUARD_WORD` are a way too. **Must equal +/// `perry_runtime::proxy::put_value::packed_add::{ADD_WAYS_WORD, +/// ADD_WAY_WORDS, ADD_WAYS_LOG2, ADD_WAY_HASH}`** (and `add_way_home`); +/// pinned by the runtime's `packed_set_site_layout_matches_codegen`. +pub(crate) const ADD_WAYS_WORD: usize = 3; +pub(crate) const ADD_WAY_WORDS: usize = 2; +pub(crate) const ADD_WAYS_LOG2: u32 = 6; +pub(crate) const ADD_WAY_HASH: u32 = 0x9E37_79B1; +/// Ways compared from the home on: the home, then the next (mod the block), +/// where the runtime places a memo whose home an earlier one holds. +pub(crate) const ADD_WAY_PROBES: usize = 2; const ADD_SLOT_MASK: u64 = (1 << ADD_SLOT_BITS) - 1; /// Block-name stem of the key-add hit. const ADD_STEM: &str = "put.add"; @@ -160,6 +176,14 @@ fn add_header_refuse_mask() -> i32 { ((ADD_REFUSE_RESERVED << 16) | (ADD_REFUSE_GC_FLAGS << 8)) as i32 } +/// The hot key-add test over the same word: nothing refused AND no layout +/// record to retire (`ADD_LAYOUT_RESERVED`). Its zero is the common case; a +/// non-zero result is sorted out by [`add_header_refuse_mask`] off the hot +/// path. +fn add_header_hot_mask() -> i32 { + add_header_refuse_mask() | (ADD_LAYOUT_RESERVED << 16) as i32 +} + /// The barrier-census stem. Shared with the census registry /// (`barrier_stem_census_tests::VERIFIED_BARRIER_STEMS`). pub(crate) const STORE_IC_STEM: &str = "put.pic"; @@ -283,22 +307,45 @@ pub(crate) fn emit_static_store_ic( let sid = ctx.block().load(I32, &sid_ptr); let stamp = ctx.block().trunc(I64, &word, I32); let shape_eq = ctx.block().icmp_eq(I32, &sid, &stamp); + ctx.block().cond_br(&shape_eq, &kind_label, &add_label); + let mut word_incoming: Vec<(String, String)> = vec![(word, tok_label.clone())]; + + // A word miss: the key-add memo's primary pre-shape next, BEFORE the + // existing-key ways. A site that has only ever added keys then pays one + // compare of the adjacent word instead of the ways-cache load; a site + // with existing-key ways pays that one compare more. The two can never + // both match one ShapeId: an existing-key memo names a shape that HAS + // the key, an add memo one that lacks it. + ctx.current_block = add_idx; let ways_entry_idx = ctx.new_block(&format!("{STORE_IC_STEM}.ways")); let ways_entry_label = ctx.block_label(ways_entry_idx); + let add_ways_idx = ctx.new_block(&format!("{ADD_STEM}.ways")); + let add_ways_label = ctx.block_label(add_ways_idx); + let add_hit_idx = ctx.new_block(&format!("{ADD_STEM}.chain")); + let add_hit_label = ctx.block_label(add_hit_idx); + let primary_ptr = ctx + .block() + .gep(I64, &packed_ref, &[(I64, &ADD_SHAPES_WORD.to_string())]); + let primary = ctx.block().load_atomic_monotonic(I64, &primary_ptr, 8); + let primary_pre = ctx.block().trunc(I64, &primary, I32); + let primary_eq = ctx.block().icmp_eq(I32, &sid, &primary_pre); ctx.block() - .cond_br(&shape_eq, &kind_label, &ways_entry_label); - let mut word_incoming: Vec<(String, String)> = vec![(word, tok_label.clone())]; - - // A word miss: compare the first ways of the site's cache (the read path's - // #7753 structure), each in the word's own format, so a hit on any of them - // is the same ONE ShapeId compare and flows into the same store. A spill + .cond_br(&primary_eq, &add_hit_label, &ways_entry_label); + // Each entry: (the memo's shapes word, the address of its pair, block). + let mut memo_incoming: Vec<(String, String, String)> = + vec![(primary, primary_ptr, add_label.clone())]; + + // Compare the first ways of the existing-key cache (the read path's #7753 + // structure), each in the word's own format, so a hit on any of them is + // the same ONE ShapeId compare and flows into the same store. A spill // entry is flipped out of the ShapeId range and never matches here. The // cache is lazily allocated: a site that has never primed has none. ctx.current_block = ways_entry_idx; let ways = super::emit_inline_cache_slot(ctx, &cache_name); let first_way_idx = ctx.new_block(&format!("{STORE_IC_STEM}.way")); let mut way_label = ctx.block_label(first_way_idx); - ctx.block().cond_br(&ways.present, &way_label, &add_label); + ctx.block() + .cond_br(&ways.present, &way_label, &add_ways_label); let mut way_idx = first_way_idx; for w in 0..PACKED_SET_INLINE_WAYS { ctx.current_block = way_idx; @@ -310,13 +357,74 @@ pub(crate) fn emit_static_store_ic( way_idx = ctx.new_block(&format!("{STORE_IC_STEM}.way")); ctx.block_label(way_idx) } else { - add_label.clone() + add_ways_label.clone() }; ctx.block().cond_br(&way_eq, &kind_label, &next_label); word_incoming.push((entry, way_label.clone())); way_label = next_label; } + // The key-add ways at the receiver ShapeId's home in the runtime's block + // (`packed_add::add_way_home`) and the one after it: a displaced memo is + // placed at its home, or when an earlier memo holds that, at the next + // free way from it. Whichever pre-shapes a polymorphic site keeps hot, + // each is one or two compares away, the same compare as the primary's. + // A hit reads its guard from the same pair. + ctx.current_block = add_ways_idx; + let block_ptr = ctx + .block() + .gep(I64, &packed_ref, &[(I64, &ADD_WAYS_WORD.to_string())]); + let block_word = ctx.block().load_atomic_monotonic(I64, &block_ptr, 8); + let has_block = ctx.block().icmp_ne(I64, &block_word, "0"); + let add_block = ctx.block().inttoptr(I64, &block_word); + let mut add_way_idx = ctx.new_block(&format!("{ADD_STEM}.way")); + let mut add_way_label = ctx.block_label(add_way_idx); + ctx.block().cond_br(&has_block, &add_way_label, &miss_label); + ctx.current_block = add_way_idx; + let hashed = ctx + .block() + .mul(I32, &sid, &(ADD_WAY_HASH as i32).to_string()); + let home = ctx + .block() + .lshr(I32, &hashed, &(32 - ADD_WAYS_LOG2).to_string()); + for probe in 0..ADD_WAY_PROBES { + ctx.current_block = add_way_idx; + let way = if probe == 0 { + home.clone() + } else { + let next = ctx.block().add(I32, &home, &probe.to_string()); + ctx.block() + .and(I32, &next, &((1u32 << ADD_WAYS_LOG2) - 1).to_string()) + }; + let way_wide = ctx.block().zext(I32, &way, I64); + let word_index = ctx.block().mul(I64, &way_wide, &ADD_WAY_WORDS.to_string()); + let pair_ptr = ctx.block().gep(I64, &add_block, &[(I64, &word_index)]); + let shapes = ctx.block().load_atomic_monotonic(I64, &pair_ptr, 8); + let pre = ctx.block().trunc(I64, &shapes, I32); + let way_eq = ctx.block().icmp_eq(I32, &sid, &pre); + let next_label = if probe + 1 < ADD_WAY_PROBES { + add_way_idx = ctx.new_block(&format!("{ADD_STEM}.way")); + ctx.block_label(add_way_idx) + } else { + miss_label.clone() + }; + let hit_label = if super::store_census::enabled() { + // A census build counts a way hit on its own edge. + let count_idx = ctx.new_block(&format!("{ADD_STEM}.way.census")); + let count_label = ctx.block_label(count_idx); + ctx.block().cond_br(&way_eq, &count_label, &next_label); + ctx.current_block = count_idx; + super::store_census::bump(ctx, super::store_census::ADD_WAY_HIT); + ctx.block().br(&add_hit_label); + count_label + } else { + ctx.block().cond_br(&way_eq, &add_hit_label, &next_label); + add_way_label.clone() + }; + memo_incoming.push((shapes, pair_ptr, hit_label)); + add_way_label = next_label; + } + // The per-object facts the shape does not carry (see the module doc), as // a branch chain rather than one flat predicate (#7883). ctx.current_block = kind_idx; @@ -376,12 +484,25 @@ pub(crate) fn emit_static_store_ic( let hit_end_label = ctx.block().label.clone(); ctx.block().br(&merge_label); - ctx.current_block = add_idx; + ctx.current_block = add_hit_idx; + let (shapes, pair_ptr) = { + let shapes_in: Vec<(&str, &str)> = memo_incoming + .iter() + .map(|(s, _, l)| (s.as_str(), l.as_str())) + .collect(); + let pairs_in: Vec<(&str, &str)> = memo_incoming + .iter() + .map(|(_, p, l)| (p.as_str(), l.as_str())) + .collect(); + let shapes = ctx.block().phi(I64, &shapes_in); + let pair_ptr = ctx.block().phi(PTR, &pairs_in); + (shapes, pair_ptr) + }; let add_end_label = emit_key_add_hit( ctx, - &packed_ref, + &shapes, + &pair_ptr, &handle, - &sid, value_double, value_bits, &miss_label, @@ -417,67 +538,61 @@ pub(crate) fn emit_static_store_ic( ) } -/// The key-add hit: `k` is not own on the receiver, and the site's add words -/// (`perry_runtime::proxy::put_value::packed_add`) memo the transition from -/// the receiver's PRE-shape. Entered after the existing-key word and ways -/// missed, with the receiver's handle and ShapeId already loaded. Returns the -/// label of the block that branches to `merge_label` on a hit; every refusal -/// branches to `miss_label` with nothing written. +/// The key-add hit: `k` is not own on the receiver, and one of the site's +/// add memos (`perry_runtime::proxy::put_value::packed_add`) names the +/// receiver's PRE-shape. Entered from the pre-shape compare that matched, +/// with that memo's `shapes` word and the address of its `{shapes, guard}` +/// pair. Returns the label of the block that branches to `merge_label` on a +/// hit; every refusal branches to `miss_label` with nothing written. /// /// ```text -/// ONE pre-shape compare sid == low half of word 1 -/// the chain verdict PROTO_VALIDITY + VTABLE_GEN == word 2 >> 16 -/// per-object facts GcHeader word: not TENURED, layout state -/// UNKNOWN / POINTER_FREE, no numeric proof, -/// tombstones or descriptor flag; meta == null; -/// the receiver-kind admission -/// the successor ShapeId high half of word 1 -> handle + 4 -/// the store and barrier slot = word 2 & 0xFFFF +/// ONE pre-shape compare sid == low half of a memo's shapes (caller) +/// the chain verdict PROTO_VALIDITY + VTABLE_GEN == guard >> 16 +/// the receiver-kind admission class id / _reserved, as the existing-key hit +/// per-object facts ONE test of the GcHeader word: not TENURED, +/// no numeric proof, tombstones or descriptor +/// flag, and no layout record to retire +/// the successor ShapeId high half of shapes -> handle + 4 +/// the store and barrier slot = guard & 0xFFFF /// ``` /// +/// A receiver with a layout record (side mask or typed layout) leaves the +/// one test for a cold block that refuses exactly what the hot test refuses +/// and retires the record before the same store. +/// /// Nothing between the caller's re-read of the receiver and the stores can /// collect: plain loads, compares, and two stores. #[allow(clippy::too_many_arguments)] fn emit_key_add_hit( ctx: &mut FnCtx<'_>, - packed_ref: &str, + shapes: &str, + pair_ptr: &str, handle: &str, - sid: &str, value_double: &str, value_bits: &str, miss_label: &str, merge_label: &str, ) -> String { - let gen_idx = ctx.new_block(&format!("{ADD_STEM}.chain")); - let layout_idx = ctx.new_block(&format!("{ADD_STEM}.layout")); - let forget_idx = ctx.new_block(&format!("{ADD_STEM}.layout.forget")); let obj_idx = ctx.new_block(&format!("{ADD_STEM}.object")); - let class_idx = ctx.new_block(&format!("{ADD_STEM}.class")); let classless_idx = ctx.new_block(&format!("{ADD_STEM}.classless")); + let layout_idx = ctx.new_block(&format!("{ADD_STEM}.layout")); + let slow_idx = ctx.new_block(&format!("{ADD_STEM}.layout.slow")); + let forget_idx = ctx.new_block(&format!("{ADD_STEM}.layout.forget")); let store_idx = ctx.new_block(&format!("{ADD_STEM}.hit.store")); - let gen_label = ctx.block_label(gen_idx); let obj_label = ctx.block_label(obj_idx); - let class_label = ctx.block_label(class_idx); let classless_label = ctx.block_label(classless_idx); - let store_label = ctx.block_label(store_idx); let layout_label = ctx.block_label(layout_idx); + let slow_label = ctx.block_label(slow_idx); let forget_label = ctx.block_label(forget_idx); + let store_label = ctx.block_label(store_idx); - // The pre-shape compare. A spill-slot memo is published flipped out of - // the ShapeId range, so it can never match here. - let shapes_ptr = ctx - .block() - .gep(I64, packed_ref, &[(I64, &ADD_SHAPES_WORD.to_string())]); - let shapes = ctx.block().load_atomic_monotonic(I64, &shapes_ptr, 8); - let pre = ctx.block().trunc(I64, &shapes, I32); - let pre_eq = ctx.block().icmp_eq(I32, sid, &pre); - ctx.block().cond_br(&pre_eq, &gen_label, miss_label); - - // The chain verdict's generation: the one global prototype-validity word. - ctx.current_block = gen_idx; - let guard_ptr = ctx - .block() - .gep(I64, packed_ref, &[(I64, &ADD_GUARD_WORD.to_string())]); + // The chain verdict's generation: the one global prototype-validity word, + // against the guard of the memo that matched. + let guard_ptr = ctx.block().gep( + I64, + pair_ptr, + &[(I64, &(ADD_GUARD_WORD - ADD_SHAPES_WORD).to_string())], + ); let guard = ctx.block().load_atomic_monotonic(I64, &guard_ptr, 8); let now = ctx .block() @@ -486,19 +601,14 @@ fn emit_key_add_hit( let gen_eq = ctx.block().icmp_eq(I64, &now, &recorded); ctx.block().cond_br(&gen_eq, &obj_label, miss_label); - // The GcHeader's first word: obj_type | gc_flags << 8 | _reserved << 16. + // The GcHeader's first word (obj_type | gc_flags << 8 | _reserved << 16) + // and the receiver-kind admission, as the existing-key hit. ctx.current_block = obj_idx; let hdr_addr = ctx.block().sub(I64, handle, "8"); let hdr_ptr = ctx.block().inttoptr(I64, &hdr_addr); let hdr = ctx.block().load(I32, &hdr_ptr); - let refused = ctx - .block() - .and(I32, &hdr, &add_header_refuse_mask().to_string()); - let hdr_ok = ctx.block().icmp_eq(I32, &refused, "0"); - ctx.block().cond_br(&hdr_ok, &class_label, miss_label); - - // The receiver-kind admission, as the existing-key hit. - ctx.current_block = class_idx; + let reserved_i32 = ctx.block().lshr(I32, &hdr, "16"); + let reserved = ctx.block().trunc(I32, &reserved_i32, I16); let class_ptr = ctx.block().inttoptr(I64, handle); let class_id = ctx.block().load(I32, &class_ptr); let class_biased = ctx.block().add(I32, &class_id, "2"); @@ -507,36 +617,50 @@ fn emit_key_add_hit( .cond_br(&has_class, &layout_label, &classless_label); ctx.current_block = classless_idx; - let reserved_i32 = ctx.block().lshr(I32, &hdr, "16"); - let reserved = ctx.block().trunc(I32, &reserved_i32, I16); let admit_bits = ctx.block().and(I16, &reserved, CLASSLESS_ADMIT_MASK_I16); let admitted = ctx.block().icmp_eq(I16, &admit_bits, CLASSLESS_ADMIT_I16); let classless = ctx.block().icmp_eq(I32, &class_id, "0"); let plain_ok = ctx.block().and(I1, &admitted, &classless); ctx.block().cond_br(&plain_ok, &layout_label, miss_label); - // A side-mask or typed-layout receiver: its layout record describes the - // PRE-shape, so retire it first, exactly as the transition lane does. The - // callee edits side tables only and cannot collect. + // ONE test: nothing refused and no layout record. ctx.current_block = layout_idx; - let layout_bits = ctx + let hot_bits = ctx .block() - .and(I32, &hdr, &((ADD_LAYOUT_RESERVED << 16) as i32).to_string()); - let layout_plain = ctx.block().icmp_eq(I32, &layout_bits, "0"); - ctx.block() - .cond_br(&layout_plain, &store_label, &forget_label); + .and(I32, &hdr, &add_header_hot_mask().to_string()); + let hot_ok = ctx.block().icmp_eq(I32, &hot_bits, "0"); + ctx.block().cond_br(&hot_ok, &store_label, &slow_label); + + // Cold: a refused receiver misses; a side-mask or typed-layout receiver's + // layout record describes the PRE-shape, so it is retired first, exactly + // as the transition lane does. The callee edits side tables only and + // cannot collect. + ctx.current_block = slow_idx; + let refused = ctx + .block() + .and(I32, &hdr, &add_header_refuse_mask().to_string()); + let hdr_ok = ctx.block().icmp_eq(I32, &refused, "0"); + ctx.block().cond_br(&hdr_ok, &forget_label, miss_label); ctx.current_block = forget_idx; super::store_census::bump(ctx, super::store_census::ADD_LAYOUT_FORGET); ctx.block() .call_void("js_gc_key_add_layout_unknown", &[(I64, handle)]); + // Re-read: the call changed the layout bits the bookkeeping tests. + let reserved_addr = ctx.block().sub(I64, handle, "6"); + let reserved_ptr = ctx.block().inttoptr(I64, &reserved_addr); + let reserved_after = ctx.block().load(I16, &reserved_ptr); ctx.block().br(&store_label); // The transition: stamp the successor, then store the value, then the // GC's obligations for the bits stored. ctx.current_block = store_idx; + let reserved = ctx.block().phi( + I16, + &[(&reserved, &layout_label), (&reserved_after, &forget_label)], + ); super::store_census::bump(ctx, super::store_census::ADD_HIT); - let post_wide = ctx.block().lshr(I64, &shapes, "32"); + let post_wide = ctx.block().lshr(I64, shapes, "32"); let post = ctx.block().trunc(I64, &post_wide, I32); let sid_addr = ctx.block().add(I64, handle, "4"); let sid_ptr = ctx.block().inttoptr(I64, &sid_addr); @@ -578,10 +702,6 @@ fn emit_key_add_hit( // bookkeeping below is guarded only by live tests of the stored bits and // of the receiver's header. ctx.block().store(DOUBLE, &fixed, &slot_ptr); - // Re-read: the layout call above may have changed the layout bits. - let reserved_addr = ctx.block().sub(I64, handle, "6"); - let reserved_ptr = ctx.block().inttoptr(I64, &reserved_addr); - let reserved = ctx.block().load(I16, &reserved_ptr); emit_static_store_ic_bookkeeping( ctx, handle, &slot, &slot_ptr, &reserved, &fixed, value_bits, "put.pic", ); diff --git a/crates/perry-codegen/src/expr/store_census.rs b/crates/perry-codegen/src/expr/store_census.rs index 3d80c07920..2f1bb94e49 100644 --- a/crates/perry-codegen/src/expr/store_census.rs +++ b/crates/perry-codegen/src/expr/store_census.rs @@ -40,6 +40,9 @@ pub(crate) const BY_NAME_RUNTIME: usize = 11; pub(crate) const BY_NAME_PUT_VALUE: usize = 12; /// Key-add inline hit that first retired the receiver's layout record. pub(crate) const ADD_LAYOUT_FORGET: usize = 13; +/// Key-add inline hit on one of the runtime block's first ways (counted on +/// its own edge, then also as [`ADD_HIT`]). +pub(crate) const ADD_WAY_HIT: usize = 14; pub(crate) fn enabled() -> bool { static ON: std::sync::OnceLock = std::sync::OnceLock::new(); diff --git a/crates/perry-codegen/tests/native_proof_regressions.rs b/crates/perry-codegen/tests/native_proof_regressions.rs index 87e100d6c6..ed626c030f 100644 --- a/crates/perry-codegen/tests/native_proof_regressions.rs +++ b/crates/perry-codegen/tests/native_proof_regressions.rs @@ -15433,6 +15433,38 @@ fn static_put_value_uses_write_pic_for_call_free_rhs() { ir.contains("put.add.check") && ir.contains("put.add.hit.store"), "a word and way miss must compare the key-add memo before the call:\n{ir}" ); + // The add memo's primary pre-shape is compared right after the word, and + // only its miss reaches the existing-key ways. + let check_block: Vec<&str> = ir + .lines() + .skip_while(|l| !(l.starts_with("put.add.check") && l.trim_end().ends_with(':'))) + .skip(1) + .take_while(|l| !l.trim_end().ends_with(':')) + .collect(); + let check_br = check_block + .iter() + .find(|l| l.contains(" br ")) + .copied() + .unwrap_or(""); + assert!( + check_br.contains("%put.add.chain") && check_br.contains("%put.pic.ways"), + "the key-add primary compare must branch to the add hit or the existing-key ways:\n{ir}" + ); + // After the primary memo, TWO key-add ways: the receiver ShapeId's home, + // `(sid * 0x9E3779B1) >> 26` (packed_add::add_way_home), and the next. + let way_blocks: Vec<&str> = ir + .lines() + .filter(|l| l.starts_with("put.add.way.") && l.trim_end().ends_with(':')) + .collect(); + assert_eq!( + way_blocks.len(), + 2, + "after the primary memo the home way and the next are compared inline:\n{ir}" + ); + assert!( + ir.contains("mul i32") && ir.contains("-1640531535") && ir.contains("lshr i32"), + "the inline way is the ShapeId's home, sid * ADD_WAY_HASH >> 26:\n{ir}" + ); assert_eq!( ir.lines() .filter(|l| l.starts_with("put.pic.way.") && l.trim_end().ends_with(':')) diff --git a/crates/perry-runtime/src/proxy/put_value/packed_add.rs b/crates/perry-runtime/src/proxy/put_value/packed_add.rs index 88f50d4ddc..a3aa5e202c 100644 --- a/crates/perry-runtime/src/proxy/put_value/packed_add.rs +++ b/crates/perry-runtime/src/proxy/put_value/packed_add.rs @@ -95,27 +95,59 @@ pub struct PackedSetSite { /// A `*mut AddWays` (0 = none): the memos of further pre-shapes, served /// by [`packed_add_try`]. A base-class constructor's key-add sees one /// pre-shape per subclass (the prototype is part of the shape), so such - /// a site is polymorphic by construction. Emitted code never reads it. + /// a site is polymorphic by construction. The emitted hit compares the + /// ways at the pre-shape's home ([`add_way_home`]) and the one after it, + /// after the primary words. pub add_ways: AtomicU64, } -/// One further memo, in the primary words' format. +/// One further memo, in the primary words' format (`add_shapes`, +/// `add_guard` are a way too: the emitted hit reads either through one +/// pointer). #[repr(C)] pub struct AddWay { shapes: AtomicU64, guard: AtomicU64, } -/// Further memos per site. Filled in order, never evicted (so a site with -/// more stable pre-shapes than ways settles instead of cycling); a site that -/// overflows them re-primes its primary words. 48, not 8: Zod 3's `ZodType` +/// Further memos per site, never evicted (so a site with more stable +/// pre-shapes than ways settles instead of cycling); a site that overflows +/// them re-primes its primary words. A memo is placed at its pre-shape's HOME +/// way ([`add_way_home`]) when that way is free, and otherwise at the next +/// free way from it; the emitted hit compares the home and the way after it +/// ([`ADD_WAY_PROBES`]), the runtime every way, and a memo the runtime serves +/// from further away is moved into one of the two ([`promote_way`]). Placement by pre-shape rather +/// than by arrival matters: on +/// tsc the hot memo of a polymorphic site is typically NOT among its first +/// (8 sites whose hits all land on their 3rd way, 10 on their 15th, behind +/// transient first-instance shapes), so no fixed prefix of an in-order list +/// is where the hits are. +/// +/// 64 (a power of two for the home hash), not 8: Zod 3's `ZodType` /// constructor adds its keys to one pre-shape per subclass (36 of them), and /// with 8 ways 15,069 of its 78,250 executed key-adds per 200 parses re-ran -/// the full `[[Set]]` and re-primed. A linear scan of 48 words is a small -/// fraction of that walk, and only a polymorphic site allocates them. -pub const ADD_WAYS: usize = 48; +/// the full `[[Set]]` and re-primed. Only a polymorphic site allocates them. +pub const ADD_WAYS: usize = 1 << ADD_WAYS_LOG2; +/// `log2(ADD_WAYS)`: the home is the top bits of a 32-bit product. +pub const ADD_WAYS_LOG2: u32 = 6; +/// The multiplier of [`add_way_home`] (2^32 / golden ratio): consecutive +/// ShapeIds, which subclass shapes minted in sequence are, land far apart. +pub const ADD_WAY_HASH: u32 = 0x9E37_79B1; +/// Ways the emitted hit compares from the home on (the home, then the next +/// mod [`ADD_WAYS`]): a memo whose home an earlier memo holds lands on the +/// next free way, which is most often the very next. +#[cfg_attr(not(test), allow(dead_code))] +pub const ADD_WAY_PROBES: usize = 2; type AddWays = [AddWay; ADD_WAYS]; +/// The way the emitted hit compares for a receiver of ShapeId `pre`: +/// the top [`ADD_WAYS_LOG2`] bits of `pre * ADD_WAY_HASH` (mod 2^32). +/// **perry-codegen computes the same (`emit_static_store_ic`).** +#[inline] +pub fn add_way_home(pre: u32) -> usize { + (pre.wrapping_mul(ADD_WAY_HASH) >> (32 - ADD_WAYS_LOG2)) as usize +} + impl PackedSetSite { pub const fn empty() -> Self { Self { @@ -134,6 +166,11 @@ pub const ADD_SHAPES_WORD: usize = 1; pub const ADD_GUARD_WORD: usize = 2; #[cfg_attr(not(test), allow(dead_code))] pub const PACKED_SET_SITE_WORDS: usize = 4; +#[cfg_attr(not(test), allow(dead_code))] +pub const ADD_WAYS_WORD: usize = 3; +/// Words of one [`AddWay`]. +#[cfg_attr(not(test), allow(dead_code))] +pub const ADD_WAY_WORDS: usize = 2; /// Low bits of the guard word that hold the slot. pub const ADD_SLOT_BITS: u32 = 16; const ADD_SLOT_MASK: u64 = (1 << ADD_SLOT_BITS) - 1; @@ -263,7 +300,7 @@ const CENSUS_NAMES: [&str; 32] = [ "emit.by_name.runtime", "emit.by_name.put_value", "emit.add.layout_forget", - "emit.14", + "emit.add.way_hit", "emit.15", "rt.add.memo_inline", "rt.add.memo_spill", @@ -364,13 +401,21 @@ pub(crate) unsafe fn packed_add_try( let (shapes, guard) = if matches(primary) { (primary, (*site).add_guard.load(Ordering::Relaxed)) } else { - let way = site_ways(site)? - .iter() - .find(|way| matches(way.shapes.load(Ordering::Relaxed)))?; - ( + let ways = site_ways(site)?; + // A memo sits at its home way unless that was taken when it was + // placed; the emitted hit has already compared the home. + let home = add_way_home(sid); + let distance = (0..ADD_WAYS) + .find(|&i| matches(ways[(home + i) % ADD_WAYS].shapes.load(Ordering::Relaxed)))?; + let way = &ways[(home + distance) % ADD_WAYS]; + let found = ( way.shapes.load(Ordering::Relaxed), way.guard.load(Ordering::Relaxed), - ) + ); + if distance >= ADD_WAY_PROBES { + promote_way(ways, home, distance); + } + found }; let spill = shapes as u32 != sid; if guard >> ADD_SLOT_BITS != add_generation() { @@ -421,6 +466,51 @@ pub(crate) unsafe fn packed_add_try( Some(value) } +/// A memo the runtime just served lies beyond the ways the emitted hit +/// compares (its home and the next were taken when it was placed, typically +/// by a polymorphic site's transient first-instance shapes). Move it into +/// one of those two ways, so its next receiver is served inline, and move +/// that way's memo to where it was. A way whose memo sits at its OWN home is +/// kept (its receivers are served inline already); with both kept nothing +/// moves. Every memo stays in the block, so the runtime still serves each. +/// +/// Only the primary agent publishes a site's memos (see `# Agents`), and only +/// it ever matches them, so the moves are ordered with its own reads. Each +/// way is retired (`shapes` EMPTY) before its guard changes and republished +/// last, as [`packed_add_prime`] does. +fn promote_way(ways: &AddWays, home: usize, distance: usize) { + if crate::agent::current_agent() != crate::agent::PRIMARY_AGENT { + return; + } + let from = (home + distance) % ADD_WAYS; + let shapes = ways[from].shapes.load(Ordering::Relaxed); + if shapes as u32 != unflip(shapes as u32) { + // A spill memo: the emitted hit never takes it, wherever it sits. + return; + } + let at_own_home = |idx: usize| { + let word = ways[idx].shapes.load(Ordering::Relaxed); + word != PACKED_SET_EMPTY && add_way_home(unflip(word as u32)) == idx + }; + let second = (home + 1) % ADD_WAYS; + let Some(to) = [second, home].into_iter().find(|&idx| !at_own_home(idx)) else { + return; + }; + let (to_shapes, to_guard) = ( + ways[to].shapes.load(Ordering::Relaxed), + ways[to].guard.load(Ordering::Relaxed), + ); + let guard = ways[from].guard.load(Ordering::Relaxed); + ways[from].shapes.store(PACKED_SET_EMPTY, Ordering::Relaxed); + ways[to].shapes.store(PACKED_SET_EMPTY, Ordering::Relaxed); + ways[to].guard.store(guard, Ordering::Relaxed); + ways[to].shapes.store(shapes, Ordering::Relaxed); + if to_shapes != PACKED_SET_EMPTY { + ways[from].guard.store(to_guard, Ordering::Relaxed); + ways[from].shapes.store(to_shapes, Ordering::Relaxed); + } +} + /// The key-add hit's layout retirement: a receiver whose layout record /// (side mask or typed descriptor) described its PRE-shape. Exactly what the /// transition lane runs before its stamp. Edits header bits and layout / @@ -629,10 +719,14 @@ pub(crate) unsafe fn packed_add_prime( } let displaced_pre = unflip(primary as u32); if let Some(way) = site_ways(site_ptr).and_then(|ways| { - ways.iter().find(|way| { - let word = way.shapes.load(Ordering::Relaxed); - word == PACKED_SET_EMPTY || unflip(word as u32) == displaced_pre - }) + // The home way first, then the rest in order from it. + let home = add_way_home(displaced_pre); + (0..ADD_WAYS) + .map(|i| &ways[(home + i) % ADD_WAYS]) + .find(|way| { + let word = way.shapes.load(Ordering::Relaxed); + word == PACKED_SET_EMPTY || unflip(word as u32) == displaced_pre + }) }) { way.shapes.store(PACKED_SET_EMPTY, Ordering::Relaxed); way.guard.store(displaced_guard, Ordering::Relaxed); diff --git a/crates/perry-runtime/src/proxy/put_value/packed_add_tests.rs b/crates/perry-runtime/src/proxy/put_value/packed_add_tests.rs index 1fd7539792..50ca188bf4 100644 --- a/crates/perry-runtime/src/proxy/put_value/packed_add_tests.rs +++ b/crates/perry-runtime/src/proxy/put_value/packed_add_tests.rs @@ -22,6 +22,36 @@ fn packed_set_site_layout_matches_codegen() { 8 * ADD_GUARD_WORD ); assert_eq!((ADD_SHAPES_WORD, ADD_GUARD_WORD, ADD_SLOT_BITS), (1, 2, 16)); + // ADD_WAYS_WORD, ADD_WAY_WORDS, ADD_WAYS_LOG2, ADD_WAY_HASH: the emitted + // hit reads way `add_way_home(sid)` at `block + 8 * ADD_WAY_WORDS * i`, + // the primary pair through the same pointer arithmetic from + // `ADD_SHAPES_WORD`. + assert_eq!( + std::mem::offset_of!(PackedSetSite, add_ways), + 8 * ADD_WAYS_WORD + ); + assert_eq!(std::mem::size_of::(), 8 * ADD_WAY_WORDS); + assert_eq!(std::mem::offset_of!(AddWay, shapes), 0); + assert_eq!( + std::mem::offset_of!(AddWay, guard), + std::mem::offset_of!(PackedSetSite, add_guard) + - std::mem::offset_of!(PackedSetSite, add_shapes) + ); + assert_eq!( + (ADD_WAYS_WORD, ADD_WAY_WORDS, ADD_WAYS_LOG2, ADD_WAY_HASH), + (3, 2, 6, 0x9E37_79B1) + ); + // ADD_WAY_PROBES: the home and the next way. + assert_eq!(ADD_WAY_PROBES, 2); + assert_eq!(ADD_WAYS, 1 << ADD_WAYS_LOG2); + // The home is a way of the block for every ShapeId. + for sid in [0u32, 1, 0x8000_0000, 0x8000_0001, u32::MAX] { + assert!(add_way_home(sid) < ADD_WAYS); + } + assert_eq!( + add_way_home(0x8000_0001), + (0x8000_0001u32.wrapping_mul(0x9E37_79B1) >> 26) as usize + ); // An empty site's pre half is unmatchable. let empty = PackedSetSite::empty(); assert!(empty.add_shapes.load(Ordering::Relaxed) as u32 >= crate::object::shapes::SHAPE_ID_END); @@ -164,3 +194,104 @@ fn a_published_memo_owns_both_shapes_across_a_full_trace() { "a published memo must own its pre- and post-shape" ); } + +/// A polymorphic site's displaced memos sit at their pre-shape's HOME way — +/// the one way the emitted hit compares — unless that way was already +/// taken, and the runtime serves every memo wherever it sits. Sabotage: +/// placement in arrival order (way 0, 1, ...) -> a home way stays empty +/// while its memo sits elsewhere. +#[test] +fn a_displaced_memo_sits_at_its_home_way() { + let key = interned(b"added_home"); + let srcs: [&[u8]; 6] = [ + b"{\"h0\":1}", + b"{\"h1\":1}", + b"{\"h2\":1}", + b"{\"h3\":1}", + b"{\"h4\":1}", + b"{\"h5\":1}", + ]; + let site = leaked_site(); + let mut pres = Vec::new(); + for src in srcs { + let first = parsed(src); + pres.push(stamp(first)); + miss(site, first, key, 1.0); + } + let ways = unsafe { site_ways(site) }.expect("a polymorphic site has ways"); + let primary = site.add_shapes.load(Ordering::Relaxed) as u32; + assert_eq!(primary, *pres.last().unwrap(), "the newest memo is primary"); + for &pre in &pres[..pres.len() - 1] { + let at_home = ways[add_way_home(pre)].shapes.load(Ordering::Relaxed); + assert!( + at_home as u32 == pre || at_home != PACKED_SET_EMPTY, + "memo {pre:#x}: its home way {} is empty, so it must sit there", + add_way_home(pre) + ); + } + for (i, src) in srcs.iter().enumerate() { + let next = parsed(src); + assert_eq!(stamp(next), pres[i]); + let served = if pres[i] == primary { + Some(2.0) + } else { + unsafe { packed_add_try(site, next, 2.0) } + }; + assert_eq!(served, Some(2.0), "memo {i} must be served"); + } +} + +fn fresh_ways() -> Box { + Box::new(std::array::from_fn(|_| AddWay { + shapes: AtomicU64::new(PACKED_SET_EMPTY), + guard: AtomicU64::new(0), + })) +} + +/// The first ShapeId at or after `from` whose home is `home`. +fn sid_with_home(home: usize, from: u32) -> u32 { + (from..).find(|&sid| add_way_home(sid) == home).unwrap() +} + +/// A memo the runtime serves from beyond the two ways the emitted hit +/// compares moves into the second of them, trading places with a memo that +/// was not at its own home; a way whose memo IS at its own home is kept. +/// Sabotage: `promote_way` moves nothing -> the hot memo stays out of reach +/// of the emitted hit (tsc: 8 sites, every hit 2-3 ways from home). +#[test] +fn a_far_memo_moves_into_the_inline_ways() { + let base = crate::object::shapes::SHAPE_ID_BASE; + let h = 10usize; + let hot = sid_with_home(h, base); + let at_home = sid_with_home(h, hot + 1); + let stray = sid_with_home(40, base); + let ways = fresh_ways(); + let word = |sid: u32| u64::from(sid) | (u64::from(sid + 1) << 32); + ways[h].shapes.store(word(at_home), Ordering::Relaxed); + ways[h].guard.store(1, Ordering::Relaxed); + ways[h + 1].shapes.store(word(stray), Ordering::Relaxed); + ways[h + 1].guard.store(2, Ordering::Relaxed); + ways[h + 3].shapes.store(word(hot), Ordering::Relaxed); + ways[h + 3].guard.store(3, Ordering::Relaxed); + promote_way(&ways, h, 3); + let at = |i: usize| { + ( + ways[i].shapes.load(Ordering::Relaxed) as u32, + ways[i].guard.load(Ordering::Relaxed), + ) + }; + assert_eq!(at(h), (at_home, 1), "a memo at its own home is kept"); + assert_eq!( + at(h + 1), + (hot, 3), + "the served memo moves in with its guard" + ); + assert_eq!(at(h + 3), (stray, 2), "the displaced memo takes its place"); + // Both inline ways hold memos at their own homes: nothing moves. + let next_home = sid_with_home(h + 1, base); + ways[h + 1].shapes.store(word(next_home), Ordering::Relaxed); + ways[h + 3].shapes.store(word(hot), Ordering::Relaxed); + promote_way(&ways, h, 3); + assert_eq!(at(h + 1).0, next_home); + assert_eq!(at(h + 3).0, hot); +} diff --git a/crates/perry-runtime/src/typed_feedback.rs b/crates/perry-runtime/src/typed_feedback.rs index 5f72e10861..06c9455036 100644 --- a/crates/perry-runtime/src/typed_feedback.rs +++ b/crates/perry-runtime/src/typed_feedback.rs @@ -2949,7 +2949,16 @@ pub(crate) fn invalidate_method_change(class_id: u32) { const REPRESENTATION_INVALIDATION_SCAN_BUDGET: u64 = 50_000_000; pub(crate) fn invalidate_representation_change(obj_addr: usize) { - if obj_addr == 0 { + invalidate_representation_change_when(obj_addr, typed_feedback_enabled()); +} + +fn invalidate_representation_change_when(obj_addr: usize, feedback_on: bool) { + // Both counters this bumps are read only by the typed-feedback trace, and + // every site this could credit is recorded only while feedback is on. Off + // (every production run), taking the registry lock to learn that cost a + // key-add on a typed-layout receiver ~250 instructions: the lock, the + // GC-root lock depth, and the deferred-collection flush on its release. + if obj_addr == 0 || !feedback_on { return; } let mut reg = registry(); diff --git a/crates/perry-runtime/src/typed_feedback/tests.rs b/crates/perry-runtime/src/typed_feedback/tests.rs index 248f6867a4..109f148e70 100644 --- a/crates/perry-runtime/src/typed_feedback/tests.rs +++ b/crates/perry-runtime/src/typed_feedback/tests.rs @@ -3183,3 +3183,23 @@ fn class_field_get_ic_throws_a_type_error_on_a_nullish_receiver() { ); } } + +/// A key-add on a typed-layout receiver retires its layout record through +/// `invalidate_representation_change`. With feedback off (every production +/// run) nothing can read what it counts, so it must return before the +/// registry lock; the control arm proves the counter this test reads moves. +#[test] +fn representation_change_takes_no_registry_lock_with_feedback_off() { + let _guard = typed_feedback_test_lock(); + reset_typed_feedback_for_tests(); + let addr = 0x7000_0000usize; + invalidate_representation_change_when(addr, false); + assert_eq!( + typed_feedback_snapshot().representation_invalidations, + 0, + "feedback off: the registry must not be touched" + ); + invalidate_representation_change_when(addr, true); + assert_eq!(typed_feedback_snapshot().representation_invalidations, 1); + reset_typed_feedback_for_tests(); +} diff --git a/crates/perry/tests/keyadd_store_ic.rs b/crates/perry/tests/keyadd_store_ic.rs index abeea691e6..616175ad1d 100644 --- a/crates/perry/tests/keyadd_store_ic.rs +++ b/crates/perry/tests/keyadd_store_ic.rs @@ -19,6 +19,21 @@ fn perry_bin() -> PathBuf { /// Compile `source` with the store census, run it, and return (stdout, the /// inline add hits, the runtime memo serves). fn run(source: &str) -> (String, u64, u64) { + let (stdout, stderr) = run_census(source); + let memo = census(&stderr, "rt.add.memo_inline") + census(&stderr, "rt.add.memo_spill"); + (stdout, census(&stderr, "emit.add.inline_hit"), memo) +} + +/// One counter of the census line the binary printed to stderr. +fn census(stderr: &str, name: &str) -> u64 { + stderr + .split_whitespace() + .find_map(|w| w.strip_prefix(name)?.strip_prefix('=')?.parse().ok()) + .unwrap_or(0) +} + +/// Compile `source` with the store census, run it, and return (stdout, stderr). +fn run_census(source: &str) -> (String, String) { let dir = tempfile::tempdir().expect("tempdir"); let entry = dir.path().join("main.ts"); let output = dir.path().join("main_bin"); @@ -50,17 +65,9 @@ fn run(source: &str) -> (String, u64, u64) { "binary failed ({:?})\nstderr:\n{stderr}", run.status ); - let count = |name: &str| -> u64 { - stderr - .split_whitespace() - .find_map(|w| w.strip_prefix(name)?.strip_prefix('=')?.parse().ok()) - .unwrap_or(0) - }; - let memo = count("rt.add.memo_inline") + count("rt.add.memo_spill"); ( String::from_utf8_lossy(&run.stdout).trim().to_owned(), - count("emit.add.inline_hit"), - memo, + stderr, ) } @@ -314,3 +321,57 @@ console.log(s, before, after, Object.keys(P).join(","), Q.k, Object.keys(Q).leng "the prototype must miss the inline add (first receiver primes)" ); } + +/// One site fed receivers of four key lists holds four memos: the primary +/// and three ways of the runtime's block, each at its pre-shape's home way +/// unless an earlier memo took it, then at the next free way (the emitted +/// hit compares the home and the next). Each +/// has its own successor shape and slot (`{}` adds at slot 0, the others at +/// slot 1), and an inherited setter appearing later must stop every one of +/// them. Sabotage: a way hit reads the PRIMARY pair's guard -> `z` of a +/// one-key receiver lands in slot 0 over its first key. +#[test] +fn a_polymorphic_site_serves_its_first_ways_inline() { + let (stdout, stderr) = run_census( + r#"// One key-add site fed receivers of four key lists, so it holds four memos: +// the primary and three ways, with different successor shapes and slots. +function addZ(o: any, v: number) { o.z = v; } +function mk(i: number): any { + const o: any = {}; + const k = i % 4; + if (k === 1) o.a = i; + if (k === 2) o.b = i; + if (k === 3) o.c = i; + return o; +} +const objs: any[] = []; +for (let i = 0; i < 4000; i++) { const o = mk(i); addZ(o, i * 2); objs.push(o); } +let s = 0; +for (let i = 0; i < 4000; i++) s += objs[i].z * (i % 4 + 1); +const shapes = [objs[3996], objs[3997], objs[3998], objs[3999]].map((o) => Object.keys(o).join("") + "=" + Object.values(o).join(",")).join(" "); +// An inherited setter for the key appears: every memo, way or primary, must refuse. +const log: number[] = []; +Object.defineProperty(Object.prototype, "z", { set(v: number) { log.push(v); }, configurable: true }); +for (let i = 0; i < 4; i++) addZ(mk(i), 100 + i); +delete (Object.prototype as any).z; +console.log(s, shapes, log.join(",")); +"#, + ); + assert_eq!( + stdout, + r#"40000000 z=7992 az=3997,7994 bz=3998,7996 cz=3999,7998 100,101,102,103"# + ); + // Each shape's first receiver primes: 999 more of each hit a memo, the + // three displaced ones inline at their home way or the next. + let way_hits = census(&stderr, "emit.add.way_hit"); + let memo = census(&stderr, "rt.add.memo_inline"); + assert_eq!( + way_hits + memo, + 2997, + "the three displaced memos serve 999 adds each (census: {stderr})" + ); + assert!( + way_hits >= 1998, + "at most one memo can sit beyond its home's next way (census: {stderr})" + ); +}