rdeaton · Sep 23, 2026
@@ -16,6 +16,21 @@ and this project adheres to [Semantic Versioning](https://semver.org/).
   [nvidia.github.io/cuda-rust/cutile](https://nvidia.github.io/cuda-rust/cutile/)
   and release tags are namespaced `cutile-rs/vX.Y.Z`. Crate names and versions
   are unchanged.
+- A generated launcher's launch site keeps every specialization it resolved
+  instead of only the most recent one, so a kernel launched at alternating
+  shapes no longer resolves through the kernel cache on every launch.
+- Every eviction that removes a kernel (`clear_kernel_cache`, `evict_kernel`,
+  `retain_kernels`) empties every launch site in the process, so the next
+  launch of each specialization resolves again. `LaunchSite`,
+  `SiteResolution` and the new `CacheEpoch` are `#[doc(hidden)]` launcher
+  support for generated code: `SiteResolution::new` takes the `CacheEpoch`
+  read before resolving, and `LaunchSite::store` takes `&'static self`.
+
+### Fixed
+
+- An eviction releases the modules that launch sites held. Before, a launch
+  site that had resolved an evicted kernel kept its function, and the
+  module, loaded until that site stored another resolution.
 
 ## [0.4.0] - 2026-09-25
 

@@ -7,18 +7,26 @@
 //!
 //! Times only the enqueue (`async_on`), never the wait: a batch of launches
 //! is timed, then the stream is drained outside the timer so the launch
-//! queue never fills. Two shapes that matter for decode-style serving:
+//! queue never fills. Cases that matter for decode-style serving include:
 //!
 //! - `launch`: one tiny 4-tensor kernel launch (argument retention, launch
 //!   validation, `cuLaunchKernel`).
 //! - `graph_replay`: one replay of a graph recording 32 such launches over
 //!   8 buffers (per-replay resource reacquisition plus `cuGraphLaunch`).
+//! - `launch_alternating`: two launches per iteration whose inputs
+//!   alternate between two specializations (extent divisible by 16, and by
+//!   8 only), through the generated launcher.
+//! - `launch_12_specializations`: twelve launches per iteration, each at a
+//!   different specialization, through the generated launcher: a kernel
+//!   launched at many shapes must find each one in its launch site instead
+//!   of resolving it again.
 
 use criterion::{criterion_group, criterion_main, Criterion};
 use cuda_async::cuda_graph::CudaGraph;
-use cuda_core::Device;
+use cuda_core::{Device, Stream};
 use cutile::prelude::*;
 use std::future::IntoFuture;
+use std::sync::Arc;
 use std::time::{Duration, Instant};
 
 #[cutile::module]
@@ -174,6 +182,67 @@ fn host_overhead(c: &mut Criterion) {
         });
     });
 
+    // ── Alternating specializations ─────────────────────────────────────
+    // Inputs of 128 elements (divisible by 16) and 136 (by 8 only): the
+    // same kernel at two specializations, as a model forward runs it.
+    let mut zt = api::zeros::<f32>(&[N]).sync_on(&stream).expect("zt");
+    let odd = || -> Arc<Tensor<f32>> {
+        api::ones::<f32>(&[N + 8])
+            .sync_on(&stream)
+            .expect("odd")
+            .into()
+    };
+    let (a2, b2, c2) = (odd(), odd(), odd());
+    kernels::add3((&mut zt).partition([N]), &*a2, &*b2, &*c2)
+        .sync_on(&stream)
+        .expect("warm second specialization");
+
+    group.bench_function("launch_alternating", |bench| {
+        bench.iter_custom(|iters| {
+            enqueue_time(iters, &stream, || {
+                for (x, y, w) in [(&a, &b, &c3), (&a2, &b2, &c2)] {
+                    unsafe {
+                        kernels::add3((&mut zt).partition([N]), &**x, &**y, &**w).async_on(&stream)
+                    }
+                    .expect("launch");
+                }
+            })
+        });
+    });
+
+    // Extents divisible by 16, 8, 4, 2 and 1: `a` and `b` at independent
+    // divisibilities give twelve specializations of one kernel.
+    let extents = [N, N + 8, N + 4, N + 2, N + 1];
+    let input = |len: usize| -> Arc<Tensor<f32>> {
+        api::ones::<f32>(&[len])
+            .sync_on(&stream)
+            .expect("input")
+            .into()
+    };
+    let many: Vec<(Arc<Tensor<f32>>, Arc<Tensor<f32>>)> = extents
+        .iter()
+        .flat_map(|&la| extents[..3].iter().map(move |&lb| (la, lb)))
+        .take(12)
+        .map(|(la, lb)| (input(la), input(lb)))
+        .collect();
+    for (x, y) in &many {
+        kernels::add3((&mut zt).partition([N]), &**x, &**y, &*c3)
+            .sync_on(&stream)
+            .expect("warm specialization");
+    }
+    group.bench_function("launch_12_specializations", |bench| {
+        bench.iter_custom(|iters| {
+            enqueue_time(iters, &stream, || {
+                for (x, y) in &many {
+                    unsafe {
+                        kernels::add3((&mut zt).partition([N]), &**x, &**y, &*c3).async_on(&stream)
+                    }
+                    .expect("launch");
+                }
+            })
+        });
+    });
+
     // A graph of 32 launches over 8 buffers, replayed.
     let mut bufs: Vec<Tensor<f32>> = (0..8)
         .map(|_| api::zeros::<f32>(&[N]).sync_on(&stream).expect("buf"))
@@ -213,6 +282,24 @@ fn host_overhead(c: &mut Criterion) {
     group.finish();
 }
 
+/// Host time to enqueue `iters` calls of `enqueue`, draining the stream every
+/// [`BATCH`] calls outside the timer.
+fn enqueue_time(iters: u64, stream: &Stream, mut enqueue: impl FnMut()) -> Duration {
+    let mut elapsed = Duration::ZERO;
+    let mut done = 0;
+    while done < iters {
+        let batch = BATCH.min(iters - done);
+        let start = Instant::now();
+        for _ in 0..batch {
+            enqueue();
+        }
+        elapsed += start.elapsed();
+        unsafe { stream.synchronize() }.expect("drain");
+        done += batch;
+    }
+    elapsed
+}
+
 fn bench_config() -> Criterion {
     if cfg!(feature = "smoke-test") {
         Criterion::default()

@@ -926,10 +926,11 @@ pub fn generate_kernel_launcher(
         .stmts
         .push(scalar_hints_stmt.clone());
 
-    // Launch-site probe: on the steady state the launcher skips key
-    // construction, toolchain fingerprinting, hashing, and the global cache
-    // entirely — the volatile specialization inputs are compared against the
-    // site's last resolution by value, borrowed from the arguments.
+    // Launch-site probe: on a hit the launcher skips key construction,
+    // toolchain fingerprinting and the global cache. The volatile
+    // specialization inputs, borrowed from the arguments, are compared by
+    // value against the site's newest entry, then against the entries under
+    // their probe hash.
     // Fixed-size arrays: the probe borrows them, so a steady-state launch
     // allocates nothing here.
     launcher_method.block.stmts.push(parse_stmt(format!(
@@ -967,6 +968,9 @@ pub fn generate_kernel_launcher(
                 spec_args.iter().map(|(_, s)| s.clone()).collect();
             let __hints_snapshot = __hint_vals.to_vec();
             let __options_snapshot = __compile_options.clone();
+            // Read before resolving, so the site can discard the resolution
+            // if an eviction removes its function meanwhile (see `CacheEpoch`).
+            let __epoch = {root}::tile_kernel::CacheEpoch::now();
             let (function, validator) = self.jit_compile(
                 ctx, __module_ast_self,
                 module_name, function_name, function_entry,
@@ -976,6 +980,7 @@ pub fn generate_kernel_launcher(
                 _SOURCE_HASH,
             )?;
             __SITE.store({root}::tile_kernel::SiteResolution::new(
+                __epoch,
                 ctx.get_device_id(),
                 __generics_snapshot,
                 __specs_snapshot,

@@ -32,6 +32,7 @@ linkme = { workspace = true }
 serde = { workspace = true }
 serde_json = { workspace = true }
 dashmap = { workspace = true }
+rustc-hash = { workspace = true }
 once_cell = { workspace = true }
 
 [dev-dependencies]