From 415dc1f5ed49495e0ae7b78b1b1f98f5e8452900 Mon Sep 17 00:00:00 2001
From: "T. Fuchs" <tfuchs@cornell.edu>
Date: Mon, 10 Aug 2026 13:54:47 -0400
Subject: [PATCH 01/33] test: pin the pixel-subdivision path against an
 independent reference

Golden digests fail across compilers, so the adaptive rule is
reimplemented in Python and compared record for record at depths 0-4.
---
 .gitignore                                    |   3 +
 .../benchmark_reconstruction_subdivision.py   | 390 ++++++++++++
 .../test/test_reconstruction_subdivision.py   | 581 ++++++++++++++++++
 3 files changed, 974 insertions(+)
 create mode 100644 benchmarks/benchmark_reconstruction_subdivision.py
 create mode 100644 orgui/datautils/xrayutils/test/test_reconstruction_subdivision.py

diff --git a/.gitignore b/.gitignore
index 15a3129..eabe4d1 100644
--- a/.gitignore
+++ b/.gitignore
@@ -15,6 +15,9 @@ subprojects/.wraplock
 orgui.egg-info/
 orgui/_version.py
 benchmarks/roi_sum_results.json
+# Fingerprint/timing baselines are captured per build and per machine, and
+# are only meaningful compared against themselves.
+benchmarks/baselines/
 **/.ipynb_checkpoints/
 
 # Packages #
diff --git a/benchmarks/benchmark_reconstruction_subdivision.py b/benchmarks/benchmark_reconstruction_subdivision.py
new file mode 100644
index 0000000..69054b4
--- /dev/null
+++ b/benchmarks/benchmark_reconstruction_subdivision.py
@@ -0,0 +1,390 @@
+"""Depth-sweep cost and output fingerprints for the pixel-subdivision path.
+
+Two jobs in one tool:
+
+- **Cost.** Coordinate evaluations, leaves and nanoseconds per valid pixel at
+  each ``max_depth``, for both stationary and continuous exposures. This is
+  what shows the shared-corner cache switching off above depth three.
+- **Fingerprints.** XXH3-128 of every output array, so a restructuring of the
+  subdivision can be held to producing bit-identical records. Capture a
+  baseline before changing anything, then ``--compare`` against it.
+
+Fingerprints are only comparable within one build on one machine: floating
+point contraction differs between compilers, so a mismatch against a baseline
+captured elsewhere means nothing. That is why this lives in ``benchmarks``
+rather than the test suite, where
+``test/test_reconstruction_subdivision.py`` covers portable behaviour against
+an independent reference implementation instead.
+
+Runs against the synthetic geometry by default, or a prepared job with
+``--job``.
+
+**Run this with ``PYTHONPATH`` pointing at the checkout.** ``python
+benchmarks/<script>.py`` puts ``benchmarks/`` on ``sys.path``, not the
+repository root, so a pip-installed ``orgui`` shadows the working tree and the
+sweep silently measures a binary you did not just build. Every report records
+the resolved package path, and ``--compare`` refuses baselines that were taken
+against a different one.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+from pathlib import Path
+from time import perf_counter
+
+import numpy as np
+
+from orgui.datautils.xrayutils.reconstruction import (
+    _detector_corner_rays,
+    _native_module,
+    _xxh3_128,
+)
+
+
+RECORD_FIELDS = (
+    "chunk_id",
+    "local_voxel_id",
+    "weighted_intensity",
+    "weighted_variance",
+    "weight",
+    "contributors",
+)
+
+
+def _arguments():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--job", type=Path, help="prepared reconstruction job")
+    parser.add_argument("--frame", type=int, default=1800)
+    parser.add_argument("--rows", type=int, default=256)
+    parser.add_argument("--columns", type=int, default=256)
+    parser.add_argument("--row-origin", type=int, default=1000)
+    parser.add_argument("--column-origin", type=int, default=1000)
+    parser.add_argument("--depths", type=int, nargs="+", default=[0, 1, 2, 3, 4, 5])
+    parser.add_argument(
+        "--exposures",
+        choices=("stationary", "continuous"),
+        nargs="+",
+        default=["stationary", "continuous"],
+    )
+    parser.add_argument("--threads", type=int, default=1)
+    parser.add_argument("--repeat", type=int, default=3)
+    parser.add_argument("--output", type=Path)
+    parser.add_argument(
+        "--compare", type=Path, help="baseline JSON to check fingerprints against"
+    )
+    return parser.parse_args()
+
+
+def _synthetic_case(rows, columns, row_origin, column_origin):
+    """A flat detector off-axis, with a grid sized to the pixel footprint.
+
+    Mirrors the regime the real job runs in -- roughly half a voxel per pixel
+    step -- because a grid much coarser than the detector resolves every cell
+    immediately and measures nothing.
+    """
+    distance_mm = 500.0
+    pitch_mm = 0.172
+    wavevector = 7.601
+    ub = np.array(
+        [
+            [1.6122, -0.5878, 0.0000],
+            [0.0000, 1.4899, -0.1051],
+            [0.0000, 0.0000, 1.2083],
+        ]
+    )
+    row_edges = np.arange(rows + 1) + row_origin - 0.5
+    column_edges = np.arange(columns + 1) + column_origin - 0.5
+    x = np.broadcast_to(column_edges[None, :] * pitch_mm, (rows + 1, columns + 1))
+    z = np.broadcast_to(row_edges[:, None] * pitch_mm, (rows + 1, columns + 1))
+    rays = np.stack((x, np.full_like(x, distance_mm), z), axis=-1).astype(np.float64)
+    rays /= np.linalg.norm(rays, axis=-1, keepdims=True)
+    rays = np.ascontiguousarray(rays)
+
+    angles_start = np.array([0.10471975511965978, 0.7, 0.0, 0.0])
+    sweep = np.array([0.0, 0.00174533, 0.0, 0.0])
+
+    native = _native_module()
+    probe = native.ReconstructionKernel(
+        np.full(3, -1e12),
+        np.ones(3),
+        np.full(3, 2_000_000, dtype=np.int64),
+        np.ones(3, dtype=np.int64),
+        "hkl",
+        wavevector,
+        np.ascontiguousarray(np.linalg.inv(ub)),
+        np.ascontiguousarray(np.eye(3)),
+        0,
+        1,
+        1,
+        1024 * 1024,
+    )
+
+    def at(row, column, t):
+        return np.asarray(
+            probe.coordinate(
+                rays, angles_start, angles_start + sweep, row, column, 0.5, 0.5, t
+            )
+        )
+
+    centre = at(0, 0, 0.5)
+    pitch = np.maximum.reduce(
+        [
+            np.abs(at(0, 1, 0.5) - centre),
+            np.abs(at(1, 0, 0.5) - centre),
+            np.abs(at(0, 0, 1.0) - at(0, 0, 0.0)),
+        ]
+    )
+    step = np.maximum(pitch, np.max(pitch) * 0.2) / 0.5
+    corners = np.asarray(
+        [
+            at(row, column, t)
+            for row in (0, rows - 1)
+            for column in (0, columns - 1)
+            for t in (0.0, 1.0)
+        ]
+    )
+    minimum = corners.min(axis=0) - 4.0 * step - 0.31718 * step
+    shape = np.ceil(
+        (corners.max(axis=0) + 4.0 * step - minimum) / step
+    ).astype(np.int64)
+
+    rng = np.random.default_rng(0)
+    return {
+        "rays": rays,
+        "angles_start": np.ascontiguousarray(angles_start),
+        "sweep": sweep,
+        "minimum": minimum,
+        "step": step,
+        "shape": shape,
+        "chunk_shape": np.array([64, 64, 64], dtype=np.int64),
+        "frame": "hkl",
+        "wavevector": wavevector,
+        "ub_inverse": np.ascontiguousarray(np.linalg.inv(ub)),
+        "u_inverse": np.ascontiguousarray(np.eye(3)),
+        "intensity": rng.uniform(1.0, 100.0, size=(rows, columns)),
+        "variance": np.abs(rng.normal(4.0, 0.5, size=(rows, columns))),
+        "mask": np.zeros((rows, columns), dtype=bool),
+    }
+
+
+def _job_case(job_path, frame, tile):
+    """The same case assembled from a prepared job and a real frame."""
+    from orgui.reconstruction_job import (
+        _correction_pipeline,
+        _load_assets,
+        read_job,
+    )
+
+    job = read_job(job_path)
+    config = job.config_data
+    scan = job.scan
+    spec = job.internal_spec()
+    grid = spec.grids[0]
+    bounds = np.asarray(
+        scan.exposure_angle_bounds(config, fallback=job.angle_fallback),
+        dtype=np.float64,
+    )
+    correct = _correction_pipeline(config, scan, _load_assets(job), {})
+    payload = scan.get_raw_img(frame)
+    image = np.asarray(payload.img)
+    corrected = correct.correct_frame(payload, image, frame)
+    selection = np.s_[tile[0] : tile[1], tile[2] : tile[3]]
+    ub = np.asarray(config.ub_calculator.getUB(), dtype=np.float64)
+    u = np.asarray(config.ub_calculator.getU(), dtype=np.float64)
+    # The job's own bounds may already be swept; a stationary/continuous pair
+    # is built here from its centre so both arms are always available.
+    centre = 0.5 * (bounds[frame, 0] + bounds[frame, 1])
+    following = 0.5 * (bounds[frame + 1, 0] + bounds[frame + 1, 1])
+    return {
+        "rays": _detector_corner_rays(config.detector, tile),
+        "angles_start": np.ascontiguousarray(centre),
+        "sweep": np.ascontiguousarray(following - centre),
+        "minimum": np.asarray(grid.minimum),
+        "step": np.asarray(grid.step),
+        "shape": np.asarray(grid.shape, dtype=np.int64),
+        "chunk_shape": np.asarray(grid.chunk_shape, dtype=np.int64),
+        "frame": grid.frame,
+        "wavevector": float(config.ub_calculator.getK()),
+        "ub_inverse": np.ascontiguousarray(np.linalg.inv(ub)),
+        "u_inverse": np.ascontiguousarray(np.linalg.inv(u)),
+        "intensity": np.ascontiguousarray(corrected[0][selection], dtype=np.float64),
+        "variance": np.ascontiguousarray(corrected[1][selection], dtype=np.float64),
+        "mask": np.ascontiguousarray(corrected[2][selection], dtype=bool),
+    }
+
+
+def _run(case, max_depth, exposure, threads, repeat):
+    native = _native_module()
+    kernel = native.ReconstructionKernel(
+        case["minimum"],
+        case["step"],
+        case["shape"],
+        case["chunk_shape"],
+        case["frame"],
+        case["wavevector"],
+        case["ub_inverse"],
+        case["u_inverse"],
+        max_depth,
+        threads,
+        # The block-size preset halves with depth; the memory ceiling is
+        # lifted because the kernel's own precheck is sized from
+        # children**depth, which rejects any useful tile above depth two.
+        max(1, 16384 >> max_depth),
+        1 << 50,
+    )
+    angles_start = case["angles_start"]
+    angles_end = (
+        angles_start
+        if exposure == "stationary"
+        else np.ascontiguousarray(angles_start + case["sweep"])
+    )
+    best = None
+    for _ in range(repeat):
+        started = perf_counter()
+        result = kernel.accumulate(
+            case["intensity"],
+            case["variance"],
+            case["mask"],
+            case["rays"],
+            angles_start,
+            angles_end,
+            True,
+        )
+        elapsed = perf_counter() - started
+        if best is None or elapsed < best[0]:
+            best = (elapsed, result)
+    elapsed, result = best
+    profile = dict(result["_profile"])
+    valid = max(1, profile["valid_pixels"])
+    return {
+        "depth": max_depth,
+        "exposure": exposure,
+        "valid_pixels": profile["valid_pixels"],
+        "records": int(profile["final_records"]),
+        "evaluations_per_pixel": profile["coordinate_evaluations"] / valid,
+        "leaves_per_pixel": profile["voxel_weights"] / valid,
+        "max_leaves_one_pixel": int(profile["maximum_weights_per_pixel"]),
+        "records_per_pixel": profile["final_records"] / valid,
+        "map_ns_per_pixel": profile["block_mapping_cpu_seconds"] * 1e9 / valid,
+        "wall_seconds": elapsed,
+        "fingerprints": {
+            name: _xxh3_128(result[name]) for name in RECORD_FIELDS
+        },
+    }
+
+
+def _native_build_identity():
+    """Which extension module this process actually imported."""
+    return {
+        "module": _native_module().__file__,
+        "mtime": Path(_native_module().__file__).stat().st_mtime,
+    }
+
+
+def _compare(baseline, current):
+    """Report fingerprint agreement and the speed-up, case by case."""
+    was_module = baseline.get("native", {}).get("module")
+    now_module = current["native"]["module"]
+    if was_module != now_module:
+        raise SystemExit(
+            "Baseline was taken against a different extension module, so "
+            "nothing here would be comparable:\n"
+            f"  baseline: {was_module}\n"
+            f"  current:  {now_module}\n"
+            "Re-capture the baseline with the same PYTHONPATH."
+        )
+    index = {
+        (entry["depth"], entry["exposure"]): entry for entry in baseline["cases"]
+    }
+    identical = True
+    print(
+        f"{'exposure':>11} {'depth':>5} {'records':>9} {'bitwise':>8} "
+        f"{'evals/px':>9} {'was':>9} {'ns/px':>10} {'was':>10} {'speedup':>8}"
+    )
+    for entry in current["cases"]:
+        was = index.get((entry["depth"], entry["exposure"]))
+        if was is None:
+            print(f"{entry['exposure']:>11} {entry['depth']:>5}   (no baseline)")
+            continue
+        same = entry["fingerprints"] == was["fingerprints"]
+        identical = identical and same
+        speedup = (
+            was["map_ns_per_pixel"] / entry["map_ns_per_pixel"]
+            if entry["map_ns_per_pixel"]
+            else float("nan")
+        )
+        print(
+            f"{entry['exposure']:>11} {entry['depth']:>5} {entry['records']:>9} "
+            f"{'same' if same else 'DIFFER':>8} "
+            f"{entry['evaluations_per_pixel']:>9.2f} "
+            f"{was['evaluations_per_pixel']:>9.2f} "
+            f"{entry['map_ns_per_pixel']:>10.1f} "
+            f"{was['map_ns_per_pixel']:>10.1f} {speedup:>8.2f}x"
+        )
+    print(
+        "\nAll fingerprints identical."
+        if identical
+        else "\nFINGERPRINTS DIFFER -- output is not bit-for-bit."
+    )
+    return identical
+
+
+def main():
+    """Sweep depth and exposure, reporting cost and output fingerprints."""
+    arguments = _arguments()
+    if arguments.job:
+        tile = (
+            arguments.row_origin,
+            arguments.row_origin + arguments.rows,
+            arguments.column_origin,
+            arguments.column_origin + arguments.columns,
+        )
+        case = _job_case(arguments.job, arguments.frame, tile)
+        source = {"job": str(arguments.job), "frame": arguments.frame, "tile": tile}
+    else:
+        case = _synthetic_case(
+            arguments.rows,
+            arguments.columns,
+            arguments.row_origin,
+            arguments.column_origin,
+        )
+        source = {
+            "synthetic": True,
+            "rows": arguments.rows,
+            "columns": arguments.columns,
+        }
+
+    cases = []
+    for exposure in arguments.exposures:
+        for max_depth in arguments.depths:
+            entry = _run(
+                case, max_depth, exposure, arguments.threads, arguments.repeat
+            )
+            cases.append(entry)
+            reported = {
+                key: value
+                for key, value in entry.items()
+                if key != "fingerprints"
+            }
+            print(json.dumps(reported, sort_keys=True), flush=True)
+    report = {
+        "source": source,
+        "native": _native_build_identity(),
+        "threads": arguments.threads,
+        "cases": cases,
+    }
+
+    if arguments.compare:
+        baseline = json.loads(arguments.compare.read_text(encoding="utf-8"))
+        print()
+        _compare(baseline, report)
+    if arguments.output:
+        arguments.output.write_text(
+            json.dumps(report, indent=2, sort_keys=True), encoding="utf-8"
+        )
+
+
+if __name__ == "__main__":
+    main()
diff --git a/orgui/datautils/xrayutils/test/test_reconstruction_subdivision.py b/orgui/datautils/xrayutils/test/test_reconstruction_subdivision.py
new file mode 100644
index 0000000..9c802f1
--- /dev/null
+++ b/orgui/datautils/xrayutils/test/test_reconstruction_subdivision.py
@@ -0,0 +1,581 @@
+"""Oracle and invariant tests for the adaptive pixel-subdivision path.
+
+These pin the *behaviour* of ``split_pixel`` and its two fast paths (the
+``max_depth == 0`` centre-only case and ``split_pixel_stationary_depth2``)
+against an independent Python implementation of the same adaptive rule, so
+that the native side can be restructured -- shared-corner cache replaced by
+corner-passing recursion -- with something better than self-consistency to
+check it against.
+
+The reference deliberately obtains every voxel through the kernel's own
+``coordinate()``, so it cannot drift from the kernel on frame conventions,
+ray interpolation or rounding: what it independently reproduces is the
+subdivision *algorithm* -- which lattice points are visited, which cells
+resolve, and what weight each leaf carries.
+"""
+
+from __future__ import annotations
+
+import numpy as np
+import pytest
+
+
+native = pytest.importorskip(
+    "orgui.datautils.xrayutils._reciprocal_reconstruction_cpp"
+)
+
+
+DETECTOR_DISTANCE_MM = 500.0
+PIXEL_PITCH_MM = 0.172
+WAVEVECTOR = 7.601
+# A triclinic-ish UB, so the hkl frame applies a genuinely non-diagonal
+# transform rather than a scaled identity.
+UB = np.array(
+    [
+        [1.6122, -0.5878, 0.0000],
+        [0.0000, 1.4899, -0.1051],
+        [0.0000, 0.0000, 1.2083],
+    ]
+)
+
+
+def _detector_rays(rows, columns, row_origin=0, column_origin=0):
+    """Unit rays at pixel corners of a flat detector facing the beam.
+
+    :param int rows, columns:
+        Tile size in pixels; the returned array is one larger in each axis.
+    :param int row_origin, column_origin:
+        Offset of the tile within a detector centred on the direct beam, so
+        a tile can be placed off-axis where the angular spread is largest.
+    :returns:
+        ``(rows + 1, columns + 1, 3)`` unit rays, beam along ``+y``.
+    """
+    row_edges = np.arange(rows + 1) + row_origin - 0.5
+    column_edges = np.arange(columns + 1) + column_origin - 0.5
+    x = column_edges[None, :] * PIXEL_PITCH_MM
+    z = row_edges[:, None] * PIXEL_PITCH_MM
+    x, z = np.broadcast_arrays(np.broadcast_to(x, (rows + 1, columns + 1)), z)
+    rays = np.stack(
+        (x, np.full_like(x, DETECTOR_DISTANCE_MM), z), axis=-1
+    ).astype(np.float64)
+    rays /= np.linalg.norm(rays, axis=-1, keepdims=True)
+    return np.ascontiguousarray(rays)
+
+
+def _probe_kernel():
+    """A kernel used only through ``coordinate()``, to bound the volume."""
+    return native.ReconstructionKernel(
+        np.full(3, -1e12),
+        np.ones(3),
+        np.full(3, 2_000_000, dtype=np.int64),
+        np.ones(3, dtype=np.int64),
+        "hkl",
+        WAVEVECTOR,
+        np.ascontiguousarray(np.linalg.inv(UB)),
+        np.ascontiguousarray(np.eye(3)),
+        0,
+        1,
+        1,
+        1024 * 1024,
+    )
+
+
+def _grid_for(rays, angles_start, angles_end, voxels_per_pixel, chunk_shape):
+    """A grid whose voxel is a chosen multiple of the pixel footprint.
+
+    Sizing the voxel against the real pixel-to-pixel step is what makes the
+    subdivision actually subdivide: a grid much coarser than the detector
+    resolves every cell at depth zero and exercises nothing.
+
+    :param float voxels_per_pixel:
+        Target sample pitch measured in voxels, applied to whichever of the
+        row, column and exposure directions moves the coordinate furthest
+        along each axis. Below one, neighbouring samples usually share a
+        voxel; above one, most cells straddle a boundary and never resolve.
+    :returns:
+        ``(minimum, step, shape)``.
+    """
+    probe = _probe_kernel()
+    rows = rays.shape[0] - 1
+    columns = rays.shape[1] - 1
+    corners = []
+    for row in (0, rows - 1):
+        for column in (0, columns - 1):
+            for u, v, t in ((0.0, 0.0, 0.0), (1.0, 1.0, 1.0)):
+                corners.append(
+                    probe.coordinate(
+                        rays, angles_start, angles_end, row, column, u, v, t
+                    )
+                )
+    corners = np.asarray(corners)
+
+    def at(row, column, t):
+        return np.asarray(
+            probe.coordinate(
+                rays, angles_start, angles_end, row, column, 0.5, 0.5, t
+            )
+        )
+
+    # Each axis is sized against the largest step any of the three sampling
+    # directions takes along it. Sizing from the column direction alone
+    # leaves the axis a column barely moves along thousands of voxels wide
+    # per *row*, so no cell ever resolves and subdivision never converges.
+    centre = at(0, 0, 0.5)
+    pitch = np.maximum.reduce(
+        [
+            np.abs(at(0, 1, 0.5) - centre),
+            np.abs(at(1, 0, 0.5) - centre),
+            np.abs(at(0, 0, 1.0) - at(0, 0, 0.0)),
+        ]
+    )
+    step = np.maximum(pitch, np.max(pitch) * 0.2) / voxels_per_pixel
+    lower = corners.min(axis=0) - 4.0 * step
+    upper = corners.max(axis=0) + 4.0 * step
+    shape = np.ceil((upper - lower) / step).astype(np.int64)
+    # An offset that is not a whole number of voxels, so no coordinate is
+    # likely to sit within rounding distance of a voxel edge.
+    lower = lower - 0.31718 * step
+    return lower, step, shape
+
+
+def _kernel(minimum, step, shape, chunk_shape, max_depth, threads, block):
+    return native.ReconstructionKernel(
+        np.ascontiguousarray(minimum),
+        np.ascontiguousarray(step),
+        np.ascontiguousarray(shape),
+        np.ascontiguousarray(np.asarray(chunk_shape, dtype=np.int64)),
+        "hkl",
+        WAVEVECTOR,
+        np.ascontiguousarray(np.linalg.inv(UB)),
+        np.ascontiguousarray(np.eye(3)),
+        max_depth,
+        threads,
+        block,
+        1 << 40,
+    )
+
+
+def _reference_leaves(
+    voxel_at, max_depth, stationary
+):
+    """Leaves one pixel's adaptive subdivision produces.
+
+    An independent implementation of ``split_pixel``: a cell resolves when
+    all of its corners agree on one valid voxel, otherwise it subdivides,
+    and a cell still unresolved at ``max_depth`` contributes its whole
+    weight at its centre. Child visit order matches the native explicit
+    stack (children pushed high-to-low, popped from the back), so leaves
+    come out in the same order and floating-point summation matches.
+
+    :param voxel_at:
+        ``(u, v, t) -> (voxel, valid)``.
+    :returns:
+        List of ``(voxel, weight)`` in emission order.
+    """
+    if max_depth == 0:
+        # Mirrors the kernel's centre-only fast path, which skips the
+        # corner test entirely and assigns weight exactly one.
+        voxel, valid = voxel_at(0.5, 0.5, 0.5)
+        return [(voxel, 1.0)] if valid else []
+
+    corner_count = 4 if stationary else 8
+    leaves = []
+    stack = [
+        (
+            0.0,
+            1.0,
+            0.0,
+            1.0,
+            0.5 if stationary else 0.0,
+            0.5 if stationary else 1.0,
+            1.0,
+            0,
+        )
+    ]
+    while stack:
+        u0, u1, v0, v1, t0, t1, weight, depth = stack.pop()
+        first_voxel = None
+        first_valid = False
+        all_same = True
+        for corner in range(corner_count):
+            u = u1 if corner & 1 else u0
+            v = v1 if corner & 2 else v0
+            t = 0.5 if stationary else (t1 if corner & 4 else t0)
+            voxel, valid = voxel_at(u, v, t)
+            if corner == 0:
+                first_valid = valid
+                first_voxel = voxel
+            elif valid != first_valid or (valid and voxel != first_voxel):
+                all_same = False
+        if all_same and first_valid:
+            leaves.append((first_voxel, weight))
+            continue
+        if depth >= max_depth:
+            voxel, valid = voxel_at(
+                0.5 * (u0 + u1), 0.5 * (v0 + v1), 0.5 * (t0 + t1)
+            )
+            if valid:
+                leaves.append((voxel, weight))
+            continue
+        um = 0.5 * (u0 + u1)
+        vm = 0.5 * (v0 + v1)
+        tm = 0.5 * (t0 + t1)
+        child_weight = weight / corner_count
+        for child in range(corner_count - 1, -1, -1):
+            stack.append(
+                (
+                    um if child & 1 else u0,
+                    u1 if child & 1 else um,
+                    vm if child & 2 else v0,
+                    v1 if child & 2 else vm,
+                    0.5 if stationary else (tm if child & 4 else t0),
+                    0.5 if stationary else (t1 if child & 4 else tm),
+                    child_weight,
+                    depth + 1,
+                )
+            )
+    return leaves
+
+
+def _reference_records(
+    kernel,
+    rays,
+    intensity,
+    variance,
+    mask,
+    angles_start,
+    angles_end,
+    minimum,
+    step,
+    shape,
+    chunk_shape,
+    max_depth,
+):
+    """Records the kernel should emit, built from the reference subdivision.
+
+    :returns:
+        ``{(chunk_id, local_voxel_id): [weighted_intensity,
+        weighted_variance, weight, contributors]}``.
+    """
+    stationary = bool(np.array_equal(angles_start, angles_end))
+    chunk_shape = np.asarray(chunk_shape, dtype=np.int64)
+    chunk_grid = np.ceil(np.asarray(shape) / chunk_shape).astype(np.int64)
+    rows, columns = intensity.shape
+    records: dict[tuple[int, int], list] = {}
+
+    for row in range(rows):
+        for column in range(columns):
+            if (
+                mask[row, column]
+                or not np.isfinite(intensity[row, column])
+                or not np.isfinite(variance[row, column])
+                or variance[row, column] < 0.0
+            ):
+                continue
+            cache: dict[tuple[float, float, float], tuple] = {}
+
+            def voxel_at(u, v, t, row=row, column=column, cache=cache):
+                key = (u, v, t)
+                hit = cache.get(key)
+                if hit is not None:
+                    return hit
+                coordinate = np.asarray(
+                    kernel.coordinate(
+                        rays, angles_start, angles_end, row, column, u, v, t
+                    )
+                )
+                index = np.floor((coordinate - minimum) / step)
+                valid = bool(
+                    np.all(index >= 0.0) and np.all(index < np.asarray(shape))
+                )
+                voxel = tuple(index.astype(np.int64)) if valid else None
+                cache[key] = (voxel, valid)
+                return voxel, valid
+
+            leaves = _reference_leaves(voxel_at, max_depth, stationary)
+            if not leaves:
+                continue
+            # The kernel sorts a pixel's leaves by packed voxel id and sums
+            # the weight of each run before touching the accumulator, so a
+            # pixel contributes at most once per distinct voxel.
+            leaves.sort(key=lambda entry: entry[0])
+            offset = 0
+            while offset < len(leaves):
+                voxel = leaves[offset][0]
+                weight = 0.0
+                while offset < len(leaves) and leaves[offset][0] == voxel:
+                    weight += leaves[offset][1]
+                    offset += 1
+                index = np.asarray(voxel, dtype=np.int64)
+                chunk_index, local_index = np.divmod(index, chunk_shape)
+                key = (
+                    int(
+                        (chunk_index[0] * chunk_grid[1] + chunk_index[1])
+                        * chunk_grid[2]
+                        + chunk_index[2]
+                    ),
+                    int(
+                        (local_index[0] * chunk_shape[1] + local_index[1])
+                        * chunk_shape[2]
+                        + local_index[2]
+                    ),
+                )
+                entry = records.setdefault(key, [0.0, 0.0, 0.0, 0])
+                entry[0] += weight * intensity[row, column]
+                entry[1] += weight * weight * variance[row, column]
+                entry[2] += weight
+                entry[3] += 1
+    return records
+
+
+def _payload(rows, columns, seed=0):
+    rng = np.random.default_rng(seed)
+    intensity = rng.uniform(1.0, 100.0, size=(rows, columns))
+    variance = np.abs(rng.normal(4.0, 0.5, size=(rows, columns)))
+    mask = np.zeros((rows, columns), dtype=bool)
+    mask[rows // 3, columns // 2] = True
+    return intensity, variance, mask
+
+
+def _case(rows, columns, *, moving, voxels_per_pixel=0.5, chunk_shape=(16, 16, 16)):
+    """Assemble one complete geometry/payload case."""
+    rays = _detector_rays(rows, columns, row_origin=300, column_origin=400)
+    angles_start = np.array([0.10471975511965978, 0.7, 0.0, 0.0])
+    angles_end = (
+        angles_start + np.array([0.0, 0.00174533, 0.0, 0.0])
+        if moving
+        else angles_start.copy()
+    )
+    minimum, step, shape = _grid_for(
+        rays, angles_start, angles_end, voxels_per_pixel, chunk_shape
+    )
+    intensity, variance, mask = _payload(rows, columns)
+    return {
+        "rays": rays,
+        "angles_start": np.ascontiguousarray(angles_start),
+        "angles_end": np.ascontiguousarray(angles_end),
+        "minimum": minimum,
+        "step": step,
+        "shape": shape,
+        "chunk_shape": chunk_shape,
+        "intensity": intensity,
+        "variance": variance,
+        "mask": mask,
+    }
+
+
+def _accumulate(case, max_depth, threads=1, block=64, profile=False):
+    kernel = _kernel(
+        case["minimum"],
+        case["step"],
+        case["shape"],
+        case["chunk_shape"],
+        max_depth,
+        threads,
+        block,
+    )
+    return kernel.accumulate(
+        case["intensity"],
+        case["variance"],
+        case["mask"],
+        case["rays"],
+        case["angles_start"],
+        case["angles_end"],
+        profile,
+    )
+
+
+@pytest.mark.parametrize("max_depth", [0, 1, 2, 3, 4])
+def test_stationary_subdivision_matches_reference(max_depth):
+    """Every depth of the stationary path reproduces the reference exactly.
+
+    Depth 0 covers the centre-only fast path and depth 2 the hand-written
+    ``split_pixel_stationary_depth2``; the rest exercise generic
+    ``split_pixel``, cached below depth 4 and uncached above it.
+    """
+    case = _case(6, 7, moving=False)
+    result = _accumulate(case, max_depth)
+    kernel = _kernel(
+        case["minimum"],
+        case["step"],
+        case["shape"],
+        case["chunk_shape"],
+        max_depth,
+        1,
+        64,
+    )
+    expected = _reference_records(
+        kernel,
+        case["rays"],
+        case["intensity"],
+        case["variance"],
+        case["mask"],
+        case["angles_start"],
+        case["angles_end"],
+        case["minimum"],
+        case["step"],
+        case["shape"],
+        case["chunk_shape"],
+        max_depth,
+    )
+
+    produced = dict(
+        zip(
+            zip(
+                result["chunk_id"].tolist(),
+                result["local_voxel_id"].tolist(),
+            ),
+            zip(
+                result["weighted_intensity"].tolist(),
+                result["weighted_variance"].tolist(),
+                result["weight"].tolist(),
+                result["contributors"].tolist(),
+            ),
+        )
+    )
+    assert len(expected) > 1, "geometry must reach more than one voxel"
+    assert set(produced) == set(expected)
+    for key, (intensity, variance, weight, contributors) in produced.items():
+        reference = expected[key]
+        assert contributors == reference[3], key
+        assert intensity == pytest.approx(reference[0], rel=1e-12), key
+        assert variance == pytest.approx(reference[1], rel=1e-12), key
+        assert weight == pytest.approx(reference[2], rel=1e-12), key
+
+
+@pytest.mark.parametrize("max_depth", [0, 1, 2, 3, 4])
+def test_moving_subdivision_matches_reference(max_depth):
+    """The eight-corner octree path reproduces the reference exactly.
+
+    Continuous exposures are what ``angle_fallback="midpoint"`` produces and
+    what a rotating scan really does. Depth 4 matters most here: it is where
+    the shared-corner cache used to switch off, so it is the case the
+    corner-passing recursion changed the most.
+    """
+    case = _case(5, 5, moving=True)
+    result = _accumulate(case, max_depth)
+    kernel = _kernel(
+        case["minimum"],
+        case["step"],
+        case["shape"],
+        case["chunk_shape"],
+        max_depth,
+        1,
+        64,
+    )
+    expected = _reference_records(
+        kernel,
+        case["rays"],
+        case["intensity"],
+        case["variance"],
+        case["mask"],
+        case["angles_start"],
+        case["angles_end"],
+        case["minimum"],
+        case["step"],
+        case["shape"],
+        case["chunk_shape"],
+        max_depth,
+    )
+    produced = set(
+        zip(result["chunk_id"].tolist(), result["local_voxel_id"].tolist())
+    )
+    assert len(expected) > 1
+    assert produced == set(expected)
+    weights = dict(
+        zip(
+            zip(
+                result["chunk_id"].tolist(),
+                result["local_voxel_id"].tolist(),
+            ),
+            result["weight"].tolist(),
+        )
+    )
+    for key, weight in weights.items():
+        assert weight == pytest.approx(expected[key][2], rel=1e-12), key
+
+
+@pytest.mark.parametrize("moving", [False, True])
+@pytest.mark.parametrize("max_depth", [0, 1, 2, 3, 4])
+def test_subdivision_conserves_weight_and_intensity(max_depth, moving):
+    """Subdivision partitions each pixel: weights sum to the pixel count.
+
+    A pixel's leaves carry weights summing to exactly one unless part of it
+    leaves the grid, so with a grid that contains the whole tile the total
+    weight is the valid-pixel count and the total weighted intensity is the
+    plain sum of those pixels' intensities -- at every depth. This is the
+    invariant that a corner-passing rewrite could most plausibly break, by
+    dropping or double-counting a child.
+    """
+    case = _case(6, 6, moving=moving, voxels_per_pixel=0.5)
+    result = _accumulate(case, max_depth)
+    valid = ~case["mask"]
+    assert float(np.sum(result["weight"])) == pytest.approx(
+        float(np.count_nonzero(valid)), rel=1e-12
+    )
+    assert float(np.sum(result["weighted_intensity"])) == pytest.approx(
+        float(np.sum(case["intensity"][valid])), rel=1e-12
+    )
+
+
+@pytest.mark.parametrize("moving", [False, True])
+@pytest.mark.parametrize("max_depth", [0, 1, 2, 3, 4])
+def test_subdivision_is_independent_of_threads_and_block_size(
+    max_depth, moving
+):
+    """Splitting the tile differently across workers changes nothing.
+
+    Block size decides how pixels group into per-block accumulators and
+    thread count decides who runs them; neither may reach the scientific
+    result. Checked on the full output arrays, not summary statistics, so
+    an ordering or merge regression cannot hide inside a matching total.
+    """
+    case = _case(6, 7, moving=moving)
+    reference = _accumulate(case, max_depth, threads=1, block=4096)
+    for threads, block in ((1, 7), (4, 5), (8, 4096)):
+        candidate = _accumulate(case, max_depth, threads=threads, block=block)
+        for name in ("chunk_id", "local_voxel_id", "contributors"):
+            np.testing.assert_array_equal(
+                candidate[name], reference[name], err_msg=f"{name} {threads}/{block}"
+            )
+        for name in ("weighted_intensity", "weighted_variance", "weight"):
+            np.testing.assert_allclose(
+                candidate[name],
+                reference[name],
+                rtol=1e-12,
+                err_msg=f"{name} {threads}/{block}",
+            )
+
+
+def test_output_keys_are_sorted_and_unique_at_every_depth():
+    """The merge contract every downstream stage relies on."""
+    for moving in (False, True):
+        case = _case(6, 7, moving=moving)
+        for max_depth in range(0, 5):
+            result = _accumulate(case, max_depth, threads=4, block=8)
+            key = (
+                result["chunk_id"].astype(np.uint64) << np.uint64(32)
+            ) | result["local_voxel_id"].astype(np.uint64)
+            assert key.size > 1
+            assert np.all(np.diff(key) > 0), (moving, max_depth)
+
+
+@pytest.mark.parametrize("moving", [False, True])
+def test_deeper_subdivision_resolves_more_voxels_but_converges(moving):
+    """Depth buys voxels quickly and then only refines weights.
+
+    Guards the premise the subdivision exists for. Note that the count is
+    **not** monotone in depth and must not be asserted to be: a cell still
+    unresolved at ``max_depth`` contributes at its own centre, so raising
+    the depth replaces one centre sample by four elsewhere, and a voxel the
+    coarser sample happened to land in can go unvisited. Real frames average
+    that away over millions of pixels; a small tile does not.
+    """
+    case = _case(16, 16, moving=moving)
+    counts = [
+        _accumulate(case, max_depth)["chunk_id"].size for max_depth in range(0, 6)
+    ]
+    assert counts[1] > counts[0], counts
+    assert counts[-1] < 2 * counts[1], counts
+    # Converged: the last step changes the count by only a few per cent.
+    assert counts[-1] - counts[-2] <= 0.05 * counts[-2], counts

From 89f969c82d898ca311af948dc82e4d3e8ebeb0d9 Mon Sep 17 00:00:00 2001
From: "T. Fuchs" <tfuchs@cornell.edu>
Date: Mon, 10 Aug 2026 13:55:11 -0400
Subject: [PATCH 02/33] perf(phys): pass subdivision corners down instead of
 caching them

The dyadic corner cache switched off above depth 3, so the very_high and
maximum presets recomputed every corner, at 74 CPU-hours for one real
job. Output is bit-identical and 1.65x to 2.98x faster.

Profile counter coordinate_evaluations changes meaning.
---
 .../cpp/reciprocal_reconstruction_cpp.cpp     | 525 +++++++++++-------
 1 file changed, 329 insertions(+), 196 deletions(-)

diff --git a/orgui/datautils/xrayutils/cpp/reciprocal_reconstruction_cpp.cpp b/orgui/datautils/xrayutils/cpp/reciprocal_reconstruction_cpp.cpp
index c84c064..4d52c71 100644
--- a/orgui/datautils/xrayutils/cpp/reciprocal_reconstruction_cpp.cpp
+++ b/orgui/datautils/xrayutils/cpp/reciprocal_reconstruction_cpp.cpp
@@ -519,60 +519,15 @@ struct VoxelWeight {
     double weight;
 };
 
-struct CachedVoxel {
-    std::uint64_t voxel = 0;
-    std::uint64_t generation = 0;
-    bool valid = false;
-};
-
 struct LatticeVoxel {
     std::uint64_t voxel = 0;
     bool valid = false;
 };
 
-struct PixelCoordinateCache {
-    std::size_t side = 0;
-    std::uint64_t generation = 0;
-    std::vector<CachedVoxel> values;
-
-    void begin_pixel(const int max_depth) {
-        // A dense dyadic cache is small through depth 3 (17^3 entries).
-        // Higher depths retain the uncached path to keep memory bounded.
-        if (max_depth > 3) {
-            side = 0;
-            return;
-        }
-        const std::size_t required_side =
-            (static_cast<std::size_t>(1) << (max_depth + 1)) + 1;
-        if (side != required_side) {
-            side = required_side;
-            values.assign(side * side * side, CachedVoxel{});
-        }
-        ++generation;
-        if (generation == 0) {
-            for (auto &value : values) {
-                value.generation = 0;
-            }
-            generation = 1;
-        }
-    }
-};
-
 bool operator<(const VoxelWeight &left, const VoxelWeight &right) {
     return left.voxel < right.voxel;
 }
 
-struct Cell {
-    double u0;
-    double u1;
-    double v0;
-    double v1;
-    double t0;
-    double t1;
-    double weight;
-    int depth;
-};
-
 struct BlockProfile {
     std::uint64_t pixels_seen = 0;
     std::uint64_t valid_pixels = 0;
@@ -1300,60 +1255,6 @@ class ReconstructionKernel {
         return key;
     }
 
-    bool cached_voxel_key(
-        const PixelRays &rays,
-        const double u,
-        const double v,
-        const double t,
-        const std::vector<CoordinateTransform> &transforms,
-        PixelCoordinateCache &cache,
-        std::uint64_t &voxel,
-        BlockProfile *profile
-    ) const {
-        const double rotation_scale =
-            static_cast<double>(transforms.size() - 1);
-        const std::size_t it = static_cast<std::size_t>(
-            std::llround(t * rotation_scale)
-        );
-        if (cache.side == 0) {
-            if (profile != nullptr) {
-                ++profile->coordinate_evaluations;
-            }
-            return voxel_id(
-                coordinate_at(
-                    rays,
-                    u,
-                    v,
-                    transforms[it]
-                ),
-                voxel
-            );
-        }
-        const double scale = static_cast<double>(cache.side - 1);
-        const std::size_t iu = static_cast<std::size_t>(std::llround(u * scale));
-        const std::size_t iv = static_cast<std::size_t>(std::llround(v * scale));
-        CachedVoxel &cached = cache.values[
-            (iu * cache.side + iv) * cache.side + it
-        ];
-        if (cached.generation != cache.generation) {
-            if (profile != nullptr) {
-                ++profile->coordinate_evaluations;
-            }
-            cached.valid = voxel_id(
-                coordinate_at(
-                    rays,
-                    u,
-                    v,
-                    transforms[it]
-                ),
-                cached.voxel
-            );
-            cached.generation = cache.generation;
-        }
-        voxel = cached.voxel;
-        return cached.valid;
-    }
-
     LatticeVoxel stationary_lattice_voxel(
         const PixelRays &rays,
         const std::size_t u_index,
@@ -1528,105 +1429,324 @@ class ReconstructionKernel {
         }
     }
 
-    void split_pixel(
+    // One node of the 3x3x3 dyadic lattice a moving cell's subdivision
+    // spans: (iu, iv, it) each in {0, 1, 2}, low corner to high corner.
+    static std::size_t lattice_index(
+        const std::size_t iu,
+        const std::size_t iv,
+        const std::size_t it
+    ) {
+        return (iu * 3 + iv) * 3 + it;
+    }
+
+    LatticeVoxel evaluate_lattice_voxel(
+        const PixelRays &rays,
+        const double u,
+        const double v,
+        const double t,
+        const std::vector<CoordinateTransform> &transforms,
+        BlockProfile *profile
+    ) const {
+        // t selects one of the rotations frame_rotations() precomputed over
+        // the dyadic lattice, by the same rounding the removed shared-corner
+        // cache used, so coordinates here are bit-for-bit what it produced.
+        LatticeVoxel result;
+        if (profile != nullptr) {
+            ++profile->coordinate_evaluations;
+        }
+        const double rotation_scale =
+            static_cast<double>(transforms.size() - 1);
+        const std::size_t it = static_cast<std::size_t>(
+            std::llround(t * rotation_scale)
+        );
+        result.valid = voxel_id(
+            coordinate_at(rays, u, v, transforms[it]),
+            result.voxel
+        );
+        return result;
+    }
+
+    void subdivide_moving(
         const PixelRays &rays,
         const std::vector<CoordinateTransform> &transforms,
-        const bool stationary,
+        const double u0,
+        const double u1,
+        const double v0,
+        const double v1,
+        const double t0,
+        const double t1,
+        const double weight,
+        const int depth,
+        const LatticeVoxel (&corners)[8],
         std::vector<VoxelWeight> &weights,
-        PixelCoordinateCache &cache,
-        std::vector<Cell> &stack,
         BlockProfile *profile
     ) const {
-        cache.begin_pixel(max_depth_);
-        stack.clear();
-        stack.push_back({
-            0.0,
-            1.0,
-            0.0,
-            1.0,
-            stationary ? 0.5 : 0.0,
-            stationary ? 0.5 : 1.0,
-            1.0,
-            0,
-        });
-        while (!stack.empty()) {
-            const Cell cell = stack.back();
-            stack.pop_back();
-            std::uint64_t first_voxel = 0;
-            bool first_valid = false;
-            bool all_same = true;
-            const int corner_count = stationary ? 4 : 8;
-            for (int corner = 0; corner < corner_count; ++corner) {
-                const double u = (corner & 1) != 0 ? cell.u1 : cell.u0;
-                const double v = (corner & 2) != 0 ? cell.v1 : cell.v0;
-                const double t = stationary
-                    ? 0.5
-                    : ((corner & 4) != 0 ? cell.t1 : cell.t0);
-                std::uint64_t voxel = 0;
-                const bool valid = cached_voxel_key(
-                    rays,
-                    u,
-                    v,
-                    t,
-                    transforms,
-                    cache,
-                    voxel,
-                    profile
-                );
-                if (corner == 0) {
-                    first_valid = valid;
-                    first_voxel = voxel;
-                } else if (
-                    valid != first_valid
-                    || (valid && voxel != first_voxel)
-                ) {
-                    all_same = false;
+        // Corner c sits at (u1 if c&1 else u0, v1 if c&2 else v0,
+        // t1 if c&4 else t0) -- the same numbering the explicit-stack form
+        // used, and the same order children are visited in, so leaves are
+        // emitted in an unchanged sequence and their later summation
+        // associates identically.
+        bool all_same = true;
+        for (int corner = 1; corner < 8; ++corner) {
+            if (
+                corners[corner].valid != corners[0].valid
+                || (corners[0].valid && corners[corner].voxel != corners[0].voxel)
+            ) {
+                all_same = false;
+                break;
+            }
+        }
+        if (all_same && corners[0].valid) {
+            weights.push_back({corners[0].voxel, weight});
+            return;
+        }
+        const double um = 0.5 * (u0 + u1);
+        const double vm = 0.5 * (v0 + v1);
+        const double tm = 0.5 * (t0 + t1);
+        if (depth >= max_depth_) {
+            // A cell whose corners disagree at the deepest level assigns its
+            // whole weight at its centre. Note an entirely out-of-grid cell
+            // reaches here too (all_same holds, but no corner is valid),
+            // exactly as before.
+            const LatticeVoxel centre = evaluate_lattice_voxel(
+                rays, um, vm, tm, transforms, profile
+            );
+            if (centre.valid) {
+                weights.push_back({centre.voxel, weight});
+            }
+            return;
+        }
+        // Eight children share one 27-node lattice, of which the eight
+        // corners are already in hand: only the nineteen nodes with a
+        // midpoint index are new. That is what replaces the dense
+        // per-pixel cache -- the sharing lives in the recursion instead of
+        // in a side array that has to be indexed, generation-stamped, and
+        // sized for a depth it mostly does not use.
+        LatticeVoxel lattice[27];
+        const double us[3] = {u0, um, u1};
+        const double vs[3] = {v0, vm, v1};
+        const double ts[3] = {t0, tm, t1};
+        for (std::size_t iu = 0; iu < 3; ++iu) {
+            for (std::size_t iv = 0; iv < 3; ++iv) {
+                for (std::size_t it = 0; it < 3; ++it) {
+                    LatticeVoxel &node = lattice[lattice_index(iu, iv, it)];
+                    if (iu != 1 && iv != 1 && it != 1) {
+                        const int corner = (iu == 2 ? 1 : 0)
+                            | (iv == 2 ? 2 : 0)
+                            | (it == 2 ? 4 : 0);
+                        node = corners[corner];
+                    } else {
+                        node = evaluate_lattice_voxel(
+                            rays, us[iu], vs[iv], ts[it], transforms, profile
+                        );
+                    }
                 }
             }
-            if (all_same && first_valid) {
-                weights.push_back({first_voxel, cell.weight});
-                continue;
+        }
+        const double child_weight = weight / 8.0;
+        for (int child = 0; child < 8; ++child) {
+            const std::size_t base_u = (child & 1) != 0 ? 1 : 0;
+            const std::size_t base_v = (child & 2) != 0 ? 1 : 0;
+            const std::size_t base_t = (child & 4) != 0 ? 1 : 0;
+            LatticeVoxel child_corners[8];
+            for (int corner = 0; corner < 8; ++corner) {
+                child_corners[corner] = lattice[
+                    lattice_index(
+                        base_u + ((corner & 1) != 0 ? 1 : 0),
+                        base_v + ((corner & 2) != 0 ? 1 : 0),
+                        base_t + ((corner & 4) != 0 ? 1 : 0)
+                    )
+                ];
             }
-            if (cell.depth >= max_depth_) {
-                std::uint64_t voxel = 0;
-                const bool valid = cached_voxel_key(
-                    rays,
-                    0.5 * (cell.u0 + cell.u1),
-                    0.5 * (cell.v0 + cell.v1),
-                    0.5 * (cell.t0 + cell.t1),
-                    transforms,
-                    cache,
-                    voxel,
-                    profile
-                );
-                if (valid) {
-                    weights.push_back({voxel, cell.weight});
+            subdivide_moving(
+                rays,
+                transforms,
+                (child & 1) != 0 ? um : u0,
+                (child & 1) != 0 ? u1 : um,
+                (child & 2) != 0 ? vm : v0,
+                (child & 2) != 0 ? v1 : vm,
+                (child & 4) != 0 ? tm : t0,
+                (child & 4) != 0 ? t1 : tm,
+                child_weight,
+                depth + 1,
+                child_corners,
+                weights,
+                profile
+            );
+        }
+    }
+
+    LatticeVoxel evaluate_stationary_voxel(
+        const PixelRays &rays,
+        const double u,
+        const double v,
+        const CoordinateTransform &transform,
+        BlockProfile *profile
+    ) const {
+        // The stationary counterpart of evaluate_lattice_voxel: t is pinned,
+        // so the rotation is chosen once by the caller instead of being
+        // recovered from t on every node. transforms[size / 2] and
+        // llround(0.5 * (size - 1)) are the same index for every depth --
+        // both are 2^max_depth -- so this is bit-for-bit what the cached
+        // path evaluated.
+        LatticeVoxel result;
+        if (profile != nullptr) {
+            ++profile->coordinate_evaluations;
+        }
+        result.valid = voxel_id(
+            coordinate_at(rays, u, v, transform),
+            result.voxel
+        );
+        return result;
+    }
+
+    void subdivide_stationary(
+        const PixelRays &rays,
+        const CoordinateTransform &transform,
+        const double u0,
+        const double u1,
+        const double v0,
+        const double v1,
+        const double weight,
+        const int depth,
+        const LatticeVoxel (&corners)[4],
+        std::vector<VoxelWeight> &weights,
+        BlockProfile *profile
+    ) const {
+        bool all_same = true;
+        for (int corner = 1; corner < 4; ++corner) {
+            if (
+                corners[corner].valid != corners[0].valid
+                || (corners[0].valid && corners[corner].voxel != corners[0].voxel)
+            ) {
+                all_same = false;
+                break;
+            }
+        }
+        if (all_same && corners[0].valid) {
+            weights.push_back({corners[0].voxel, weight});
+            return;
+        }
+        const double um = 0.5 * (u0 + u1);
+        const double vm = 0.5 * (v0 + v1);
+        if (depth >= max_depth_) {
+            const LatticeVoxel centre = evaluate_stationary_voxel(
+                rays, um, vm, transform, profile
+            );
+            if (centre.valid) {
+                weights.push_back({centre.voxel, weight});
+            }
+            return;
+        }
+        // Four children share one 3x3 lattice whose corners are already in
+        // hand: five new nodes per split, exactly what
+        // split_pixel_stationary_depth2 materializes by hand at depth two.
+        LatticeVoxel lattice[9];
+        const double us[3] = {u0, um, u1};
+        const double vs[3] = {v0, vm, v1};
+        for (std::size_t iu = 0; iu < 3; ++iu) {
+            for (std::size_t iv = 0; iv < 3; ++iv) {
+                LatticeVoxel &node = lattice[iu * 3 + iv];
+                if (iu != 1 && iv != 1) {
+                    const int corner = (iu == 2 ? 1 : 0) | (iv == 2 ? 2 : 0);
+                    node = corners[corner];
+                } else {
+                    node = evaluate_stationary_voxel(
+                        rays, us[iu], vs[iv], transform, profile
+                    );
                 }
-                continue;
             }
-            const double um = 0.5 * (cell.u0 + cell.u1);
-            const double vm = 0.5 * (cell.v0 + cell.v1);
-            const double tm = 0.5 * (cell.t0 + cell.t1);
-            const int child_count = stationary ? 4 : 8;
-            const double child_weight =
-                cell.weight / static_cast<double>(child_count);
-            for (int child = child_count - 1; child >= 0; --child) {
-                stack.push_back({
-                    (child & 1) != 0 ? um : cell.u0,
-                    (child & 1) != 0 ? cell.u1 : um,
-                    (child & 2) != 0 ? vm : cell.v0,
-                    (child & 2) != 0 ? cell.v1 : vm,
-                    stationary
-                        ? 0.5
-                        : ((child & 4) != 0 ? tm : cell.t0),
-                    stationary
-                        ? 0.5
-                        : ((child & 4) != 0 ? cell.t1 : tm),
-                    child_weight,
-                    cell.depth + 1,
-                });
+        }
+        const double child_weight = weight / 4.0;
+        for (int child = 0; child < 4; ++child) {
+            const std::size_t base_u = (child & 1) != 0 ? 1 : 0;
+            const std::size_t base_v = (child & 2) != 0 ? 1 : 0;
+            LatticeVoxel child_corners[4];
+            for (int corner = 0; corner < 4; ++corner) {
+                child_corners[corner] = lattice[
+                    (base_u + ((corner & 1) != 0 ? 1 : 0)) * 3
+                    + base_v + ((corner & 2) != 0 ? 1 : 0)
+                ];
             }
+            subdivide_stationary(
+                rays,
+                transform,
+                (child & 1) != 0 ? um : u0,
+                (child & 1) != 0 ? u1 : um,
+                (child & 2) != 0 ? vm : v0,
+                (child & 2) != 0 ? v1 : vm,
+                child_weight,
+                depth + 1,
+                child_corners,
+                weights,
+                profile
+            );
+        }
+    }
+
+    void split_pixel_stationary(
+        const PixelRays &rays,
+        const CoordinateTransform &transform,
+        std::vector<VoxelWeight> &weights,
+        BlockProfile *profile
+    ) const {
+        LatticeVoxel corners[4];
+        for (int corner = 0; corner < 4; ++corner) {
+            corners[corner] = evaluate_stationary_voxel(
+                rays,
+                (corner & 1) != 0 ? 1.0 : 0.0,
+                (corner & 2) != 0 ? 1.0 : 0.0,
+                transform,
+                profile
+            );
+        }
+        subdivide_stationary(
+            rays,
+            transform,
+            0.0,
+            1.0,
+            0.0,
+            1.0,
+            1.0,
+            0,
+            corners,
+            weights,
+            profile
+        );
+    }
+
+    void split_pixel_moving(
+        const PixelRays &rays,
+        const std::vector<CoordinateTransform> &transforms,
+        std::vector<VoxelWeight> &weights,
+        BlockProfile *profile
+    ) const {
+        LatticeVoxel corners[8];
+        for (int corner = 0; corner < 8; ++corner) {
+            corners[corner] = evaluate_lattice_voxel(
+                rays,
+                (corner & 1) != 0 ? 1.0 : 0.0,
+                (corner & 2) != 0 ? 1.0 : 0.0,
+                (corner & 4) != 0 ? 1.0 : 0.0,
+                transforms,
+                profile
+            );
         }
+        subdivide_moving(
+            rays,
+            transforms,
+            0.0,
+            1.0,
+            0.0,
+            1.0,
+            0.0,
+            1.0,
+            1.0,
+            0,
+            corners,
+            weights,
+            profile
+        );
     }
 
     std::vector<Record> accumulate_block(
@@ -1653,12 +1773,14 @@ class ReconstructionKernel {
         arena.release();
         std::pmr::map<RecordKey, RecordAccum> tree(&arena);
         std::vector<VoxelWeight> weights;
-        PixelCoordinateCache coordinate_cache;
-        std::vector<Cell> stack;
-        const std::size_t reserve_leaves = static_cast<std::size_t>(1)
-            << std::min((stationary ? 2 : 3) * max_depth_, 9);
-        weights.reserve(reserve_leaves);
-        stack.reserve(reserve_leaves);
+        // Subdivision now carries its cell state on the recursion stack --
+        // nine LatticeVoxel per level stationary, twenty-seven moving, so
+        // at most about four kilobytes at the deepest setting -- so the
+        // only scratch a block still needs is the leaf list.
+        weights.reserve(
+            static_cast<std::size_t>(1)
+            << std::min((stationary ? 2 : 3) * max_depth_, 9)
+        );
         // Counts per-pixel-merged (voxel, weight) accumulation events --
         // the same quantity the old push_back-one-Record-per-event code
         // counted via records.size() before its later block-end reduce.
@@ -1758,20 +1880,31 @@ class ReconstructionKernel {
             }
             weights.clear();
             if (stationary && max_depth_ == 2) {
+                // Kept in preference to the general recursion, which measured
+                // 10% slower here: one flat 5x5 lattice per pixel shares
+                // nodes between sibling subtrees, which corner-passing gives
+                // up (22.02 against 20.82 evaluations per pixel). That
+                // sharing is only affordable because a stationary depth-two
+                // lattice is 25 nodes; see the design note on generalizing
+                // it.
                 split_pixel_stationary_depth2(
                     prepared_rays,
                     transforms[transforms.size() / 2],
                     weights,
                     profile
                 );
+            } else if (stationary) {
+                split_pixel_stationary(
+                    prepared_rays,
+                    transforms[transforms.size() / 2],
+                    weights,
+                    profile
+                );
             } else {
-                split_pixel(
+                split_pixel_moving(
                     prepared_rays,
                     transforms,
-                    stationary,
                     weights,
-                    coordinate_cache,
-                    stack,
                     profile
                 );
             }

From ec1d79bfcd03af2e0e6f211e054b77093382400f Mon Sep 17 00:00:00 2001
From: "T. Fuchs" <tfuchs@cornell.edu>
Date: Mon, 10 Aug 2026 13:55:25 -0400
Subject: [PATCH 03/33] docs: record the reciprocal-space locality and
 subdivision investigation

Adjacent frames sit as close in reciprocal space as adjacent pixels, so
a (row, column, frame) brick measures 42% fewer records. Conditional on
per-frame displacement staying near one voxel; not implemented.
---
 ..._space_mapping_locality_and_subdivision.md | 628 ++++++++++++++++++
 1 file changed, 628 insertions(+)
 create mode 100644 doc/design/reciprocal_space_mapping_locality_and_subdivision.md

diff --git a/doc/design/reciprocal_space_mapping_locality_and_subdivision.md b/doc/design/reciprocal_space_mapping_locality_and_subdivision.md
new file mode 100644
index 0000000..cbc37b4
--- /dev/null
+++ b/doc/design/reciprocal_space_mapping_locality_and_subdivision.md
@@ -0,0 +1,628 @@
+# Reciprocal-space mapping: locality, subdivision cost, and the scheduling that would carry them
+
+> **Status: idea for later. Nothing here is implemented, and no decision has
+> been taken.** The measurements are real and were taken against real beamtime
+> data; the designs sketched from them are proposals. Treat this as a record of
+> an investigation, not a plan of record.
+
+Third companion to `reciprocal_space_scratch_architecture.md` (what the pipeline
+*is*) and `reciprocal_space_performance_findings.md` (what has been measured
+about it). This one holds two findings that neither of those settles, the
+places where they meet, and what the scheduler and the memory budget would have
+to do to carry them.
+
+- **A — cross-frame locality.** The native work block deduplicates voxels within
+  one image, mostly by accident. Extending it across adjacent images cuts
+  emitted records by ~42% and *shrinks* the cache working set. **Not started.**
+- **B — subdivision cost.** Above depth 3 the shared-corner cache switched off
+  and the kernel did several times more ray work than it needed to. A
+  corner-passing recursion removes that, bit-for-bit. **Phase 1 is implemented**
+  — 1.65-2.98x stationary and 1.85-2.85x continuous at depths 1-4 on real data,
+  every output array bit-identical. See the phase 1 steps for what landed and
+  the one case that did not.
+
+B landed first: it is smaller, it changes one file, its output is bit-for-bit
+identical, and it is what makes A viable at depth > 0 at all — see "Where the
+two meet". The measurements in "Finding B" below are the *pre-change* state,
+kept as the record of why the work was done.
+
+## How this was measured
+
+All numbers are from the `39_1-rsmap` job used throughout the findings document
+(La3Ni2O7, CHESS QM2, Pilatus 6M, 2527 x 2463, 3651 frames, omega rotation
+0.1 deg/frame over 365 deg, `center` accuracy = depth 0, 2000^3 hkl grid with
+steps [0.00457, 0.00208, 0.00265]).
+
+Which voxel a pixel reaches depends only on geometry and the mask, never on the
+recorded intensity, so the depth-0 pixel-to-voxel map was reproduced in NumPy
+with the affine ray-to-coordinate transform extracted from `kernel.coordinate()`
+itself — every sign and frame convention comes from the kernel rather than from
+a re-derivation. Validated against a real `accumulate()` call on frame 1800:
+
+| | native kernel | model |
+|---|---|---|
+| block records, 16384-pixel runs | 2,690,636 | 2,698,000 (+0.3%) |
+| records after the cross-block merge | 2,577,418 | 2,577,295 (-0.005%) |
+
+Depth scaling (finding B) was measured directly through the kernel's own
+`profile=True` counters, not modelled.
+
+## Test configuration: continuous exposures
+
+Use `angle_fallback="midpoint"` as the standard configuration for this work.
+QM2 rotates continuously, so it is the physically right model, and it is free at
+depth 0: `_midpoint_bounds` centres each sweep on the stationary angle to within
+1.3e-7 rad, and depth 0 samples only `transforms[size/2]`. Re-running the whole
+locality measurement under continuous bounds reproduced the stationary result to
+one part in 10^6 (43,087,060 records against 43,087,014 over a 16-frame window;
+32x32x16 relative 0.5749 either way).
+
+It stops being free from depth 1 up, where `stationary=false` switches the
+kernel from a 4-corner quadtree to an 8-corner octree. That is exactly the path
+with no hand-written fast case and the most to gain from finding B, so testing
+under continuous bounds exercises the harder half.
+
+## Finding A: adjacent frames are as close as adjacent pixels
+
+Median displacement of a pixel's reciprocal-space coordinate, in voxels:
+
+| next column | next row | **next frame** |
+|---|---|---|
+| 0.49 | 0.80 | **0.71** (p10 0.22, p90 1.48) |
+
+So (row, column, frame) is a roughly isotropic sampling lattice, and the natural
+work block is a **brick in all three**. The current block is a flat run of 16384
+pixels — on a 2463-column detector, a 6.65 x 2463 x 1 strip, aspect ratio near
+370:1 through a volume that wants 1:1:1.
+
+Exhaustive — every block of a real edge-inclusive partition of a 16-frame window
+over full frames, with the job's real mask. No sampling.
+
+| block shape | records / valid sample | vs. current | live map |
+|---|---|---|---|
+| 7 x 2463 x 1 (**current flat run**) | 0.4741 | 1.000 | 522 KiB |
+| 128 x 128 x 1 | 0.4580 | 0.966 | 467 KiB |
+| 64 x 64 x 4 | 0.3056 | 0.645 | 307 KiB |
+| 32 x 64 x 8 | 0.2809 | 0.593 | 294 KiB |
+| **32 x 32 x 16** | **0.2725** | **0.575** | **282 KiB** |
+| union over the whole window (floor) | 0.2602 | 0.549 | — |
+
+Stable to three decimals at scan positions 0, 900 and 2700.
+
+- **Fixing the block shape within one image is worth 3.4%.** A flat run does
+  waste its perimeter, but a square detector tile is not where the volume is.
+- **A 32x32x16 brick captures 94% of what a 16-frame window offers.** The
+  redundancy is entirely local in frame index: after the first frame each
+  additional one contributes ~1.404 M new voxels against its own 2.577 M, and
+  that ratio is flat from F = 2. It needs a cache-sized brick, not a buffer.
+- **The working set falls.** Same 16384 samples, fewer distinct voxels, about
+  half the map nodes. The brick does not compete with the L2-sized work-block
+  calibration; it relaxes it.
+
+### The condition the whole idea rests on
+
+Striding the frame window stands in for a coarser scan:
+
+| deg/frame | shift (voxels) | 32x32x16 vs. flat run |
+|---|---|---|
+| 0.10 (this job) | 0.69 | 0.65 |
+| 0.20 | 1.38 | 1.00 |
+| 0.40 | 2.75 | 1.02 |
+| 0.80 | 5.51 | 1.02 |
+
+**Past roughly one voxel of travel per frame the gain is gone**, and a brick
+becomes marginally worse than the current layout because it trades in-image
+block area for frame extent that no longer overlaps. This is a statement about
+grid step versus angular step: refine the grid and you move into the regime;
+coarsen the scan and you leave it. `F = 1` — bit-for-bit today's behaviour —
+must stay a reachable outcome, not a degenerate case of new machinery.
+
+## Finding B: the subdivision is adaptive, but has a cliff at depth 3
+
+A cell is split only if its corners disagree on the voxel, so work tracks the
+length of the voxel-boundary curves crossing the pixel footprint: O(2^depth),
+not O(4^depth). Frame 1800, 512x512 tile, one thread,
+`work_block_pixels = 16384 >> depth`:
+
+| depth | evals/px | x prev | leaves/px | rec/px | ns/px | x prev |
+|---|---|---|---|---|---|---|
+| 0 | 1.00 | — | 1.0 | 0.489 | 83.8 | — |
+| 1 | 11.43 | 11.43 | 3.9 | 0.570 | 543.2 | 6.48 |
+| 2 | 26.30 | 2.30 | 11.8 | 0.611 | 615.7 | 1.13 |
+| 3 | 58.55 | 2.23 | 30.2 | 0.632 | 3090.4 | 5.02 |
+| **4** | **398.21** | **6.80** | 69.8 | 0.642 | 9215.8 | 2.98 |
+| 5 | 863.88 | 2.17 | 151.8 | 0.647 | 19341.4 | 2.10 |
+| 6 | 1810.67 | 2.10 | 318.7 | 0.650 | 39801.6 | 2.06 |
+
+The steady trend is x2.1-2.3 per level. **Except at 3 to 4, where it is x6.80** —
+`begin_pixel`'s `if (max_depth > 3) { side = 0; return; }`. Above depth 3 the
+shared-corner cache switches off and every corner of every cell is recomputed.
+Redundancy factor 6.80 / 2.15 ~ **3.2x**, which is what you expect when each
+dyadic lattice point is shared by up to four cells. `ACCURACY_DEPTHS` maps
+`very_high` to 4 and `maximum` to 5: both run in the uncached regime.
+
+### Continuous exposures are far worse
+
+The table above is the stationary quadtree. Under `midpoint` bounds the kernel
+runs the 8-corner octree instead, and both terms get worse: the tree grows x4
+per level rather than x2 (a 3D domain cut by 2D surfaces), and up to eight cells
+share a lattice point rather than four. Same real frame, 256x256 tile, one
+thread, both arms measured by `benchmarks/benchmark_reconstruction_subdivision.py`:
+
+| depth | stationary evals/px | ns/px | continuous evals/px | ns/px |
+|---|---|---|---|---|
+| 0 | 1.00 | 78.9 | 1.00 | 71.5 |
+| 1 | 11.37 | 553.2 | 33.84 | 1467.2 |
+| 2 | 26.05 | 600.4 | 150.81 | 8466.3 |
+| 3 | 57.87 | 3086.8 | 672.92 | 41343.1 |
+| 4 | 393.06 | 8952.2 | **13777.53** | **281743.5** |
+
+The cliff at 3 to 4 is **x20.5** against a x4.5 trend — redundancy factor ~4.6,
+consistent with eight-way sharing. In absolute terms depth 4 with continuous
+exposures costs 282 us per pixel: 29 minutes of single-thread CPU for one frame,
+74 hours of CPU for this job. **Continuous exposures above depth 3 are not
+usable today**, which matters because continuous is the physically right model
+for a rotation scan.
+
+Two side observations from the same sweep. A swept exposure genuinely reaches
+more voxels — 0.886 records/pixel at depth 4 against 0.605 stationary, and
+0.684 against 0.538 already at depth 1 — so switching a job to `midpoint`
+bounds raises record volume by up to ~46%, which the finding-A memory
+accounting would have to carry. And the depth-4 moving maximum is 2325 leaves
+for a single pixel against the `8^4 = 4096` the precheck assumes, so the bound
+is nearer the truth in 3D than in 2D but the mean (1432) is still far below it.
+
+Three things are wrong with the cache, in increasing order of importance.
+
+**It is 3D even for stationary exposures.** `side^3` entries, but with `t` pinned
+only the plane `it = (side-1)/2` is touched — 289 of 4913 at depth 3. The index
+`(iu*side + iv)*side + it` strides 408 B, so every access lands on its own cache
+line: 289 points cost 289 lines where a contiguous 2D array would need ~110. Per
+worker thread the array is 3.0 / 17.1 / 115.2 KiB at depths 1 / 2 / 3 — past L1d
+(48 KiB) at depth 3, and 842 KiB / 6.3 MiB at depths 4 / 5, which is why it is
+capped at all.
+
+**It is dense, but the walk is sparse.** ~20% of the lattice is touched at depth
+3, ~11% at depth 6. Density is the wrong bet and gets worse with depth.
+
+**The generic path pays for its machinery regardless.** Depth 2 (hand-written
+`split_pixel_stationary_depth2`) does **2.3x the evaluations of depth 1 in 1.13x
+the time**. Backing a raw evaluation out at ~23 ns from the uncached depths 4-6,
+the generic path carries roughly 250 ns/pixel of stack traffic, cache indexing
+and generation bookkeeping that the depth-2 special case simply does not have.
+
+The cache is still a net win where enabled — about 28% at depth 3 — but it buys
+far less than the structure it displaces could.
+
+### Sub-pixel geometry is not the problem
+
+For the record, since it comes up: `ray_at` bilinearly blends the four corner
+unit rays rather than calling the detector model at the sub-pixel position.
+Measured against exact `primBeamPoints` over 21,605 pixels spanning the frame,
+the error is **2.1e-5 voxels median, 2.7e-5 max**, against a one-pixel step of
+0.79 voxels — one part in 30,000 of a pixel. Replacing it would cost ~61 ns per
+point (`verify_geometry_cost.py`) against ~7 ns, i.e. ~3.6x slower at depth 4,
+to fix nothing measurable. The motor angles, meanwhile, are already exact:
+`frame_rotation` interpolates the *angle* and takes its sine and cosine, so
+every sub-sample gets a true rotation, and the dyadic `t` lattice is hit exactly
+by `llround`. Keep the bilinear rays.
+
+## Where the two meet: the per-pixel state budget
+
+`ray_at(u, v)` is frame-invariant — bilinear interpolation of detector corner
+rays plus a normalise. Only `M_f . ray + b_f` and `voxel_id` depend on the frame.
+So in principle a frame-group brick could evaluate each lattice ray once and
+reuse it across all F frames.
+
+In practice, only at depth 0:
+
+- *Which* lattice points are needed is frame-dependent — the tree follows the
+  voxel boundaries, and those rotate. Only the shallow levels are reliably
+  shared.
+- A per-pixel dyadic ray cache costs `(2^(depth+1)+1)^2 x 24 B` **per pixel**:
+  6.9 KiB at depth 3, 26 KiB at depth 4, 101 KiB at depth 5. Per pixel, so it can
+  only be alive for one pixel at a time. That forces **pixel-outer, frame-inner**
+  iteration at depth > 0 — the opposite of the frames-outer/pixels-inner order
+  that makes depth 0 stream well. The brick loop needs two modes.
+- The payoff is small anyway: the unit ray is ~30% of an evaluation, so
+  cross-frame caching saves at most ~28%, against ~2.5x from finding B.
+
+This is the dependency between the two findings. Corner-passing keeps the
+per-pixel subdivision state at 1.2-3.5 KiB of stack instead of a 6.9-101 KiB
+dense array, which is what makes pixel-outer/frame-inner iteration affordable
+inside a brick at all. **At depth 0, cache centre rays for the brick (24 KiB for
+32x32) and iterate frames-outer. Above depth 0, iterate pixels-outer and do not
+build a cross-frame ray cache.**
+
+## Memory
+
+### What finding B costs and frees
+
+Frees, per worker thread: the whole `PixelCoordinateCache` — 3.0 / 17.1 /
+115.2 KiB at depths 1 / 2 / 3, and the 842 KiB / 6.3 MiB it would need at depths
+4 / 5 if it were not disabled. Costs: recursion frames of 144 B (stationary) or
+432 B (moving) per level, bounded by `max_depth <= 8`, so 1.2 / 3.5 KiB.
+
+Nothing else moves. `work_block_memory_cap`, `_RESERVED_RECORDS_PER_PIXEL` and
+the arena are untouched.
+
+### The `worst_leaves` bound is what actually caps depth
+
+Separate from both findings, and now well-evidenced. `native_bytes_per_pixel =
+128 + 2 * children**depth * 40` appears twice, deliberately mirrored: in the
+kernel's `accumulate` precheck and in `_frame_parallelism`. Measured maximum
+leaves for any *single* pixel:
+
+| depth | 3 | 4 | 5 | 6 |
+|---|---|---|---|---|
+| measured max | 55 | 142 | 319 | 679 |
+| measured mean | 30.2 | 69.8 | 151.8 | 318.7 |
+| `4^depth` assumed | 64 | 256 | 1024 | 4096 |
+
+The real bound grows ~2.1x per level, because a pixel's leaf count is set by
+boundary length, not area. Two compounding conservatisms: `4^depth` (or `8^depth`
+on the Python side, which cannot know whether a range is stationary), and
+assuming *every* pixel simultaneously hits its worst case. At depth 3 that is
+5248 B/px, so a full frame "needs" 32.7 GB and the layout is forced into tiny
+tiles; a 512x512 tile at depth 4 could not run at all until
+`memory_budget_bytes` was raised to 2^50 for the sweep above.
+
+Because `largest_tile_pixels` feeds `worker_memory`, which divides into
+`image_workers`, this bound throttles concurrency across the whole depth range.
+This is findings open item 2, it is a **memory** fix rather than a compute one,
+and it should be its own commit — not folded into either finding.
+
+### What finding A costs and frees
+
+Per-pixel constants, for the 6,224,001-pixel detector:
+
+| term | B/px | per frame |
+|---|---|---|
+| raw decoded image (`_RAW_IMAGE_BYTES_PER_PIXEL`) | 8 | 49.8 MB |
+| corrected intensity + variance + mask | 17 | 105.8 MB |
+| `_FRAME_RECORD_BYTES_PER_PIXEL` (batch + merge copy) | 80 | 497.9 MB |
+| **`python_memory` per in-flight frame** | **105** | **653.5 MB** |
+
+A frame group changes the shape of this:
+
+- **Resident group buffer: F x 105.8 MB.** Only the corrected arrays need to
+  stay; the raw image is released after correction. 846 MB at F = 8, 1.69 GB at
+  F = 16. This is per *group*, not per worker — a group has ~6000 bricks and can
+  occupy every thread on its own.
+- **Correction transient: (correction workers) x 155.6 MB**, unchanged in
+  character from today.
+- **`_FRAME_RECORD_BYTES_PER_PIXEL` does not multiply by F.** One merged batch
+  per group replaces F per-frame batches, and it is 0.573x the size of their
+  sum.
+
+The term that *does* grow is level 3 of the findings document's subdivision
+table — every block's sorted `std::vector<Record>` live at once. Today ~108 MB
+per concurrent frame; for a 16-frame group over full frames it would be 991 MB.
+
+**Bound it with the detector tiling that already exists.** Bricks covering one
+row band for all F frames, emitted and routed before the next band. A 256-row
+band at F = 16 is 630,528 x 16 x 0.2725 x 40 B = **110 MB** — the same figure as
+today. Frames stay resident across bands (you cannot load a band without the
+frame), so the group buffer is unaffected; only the record footprint is banded.
+Tiling therefore acquires a real purpose it does not have today, where the
+findings measured it flat at 9.6-10.1 GiB across 1/6/12 tiles.
+
+Net for the job: records through the checkpoint layer fall from 9.41e9 to
+5.39e9, 376 GB to 216 GB. The checkpoint accumulator is the dominant memory
+term and what caps concurrent frames at nine, so either the same
+`CHECKPOINT_MEMORY_SHARE` holds ~1.75x as many frames per checkpoint — fewer
+parts, cheaper finalization, which speaks to findings open item 5 — or the share
+can shrink and hand the budget to the pipeline.
+
+## Scheduling
+
+Frame grouping does not fit the current Sec7 pipeline; it replaces the question
+that pipeline exists to answer.
+
+**Today.** `image_workers` compute workers each run a kernel with
+`kernel_threads` native threads, `image_workers x kernel_threads ~ spec.threads`,
+and `_map_pending_ranges` rebalances the pair live against the reader pool's
+measured delivery rate, with `_kernel_threads_sweep` and asymmetric hysteresis.
+The pair exists because one frame's kernel call cannot use all the threads
+profitably and the memory budget caps how many frames can be in flight.
+
+**With groups.** One `accumulate_group()` call over a 16-frame group has ~6000
+bricks — enough to saturate 24 threads by itself. So `kernel_threads =
+spec.threads` and one group at a time becomes the natural layout, and the joint
+balancing problem largely dissolves. What replaces it is a simpler question:
+**how many prefetch/correct workers keep one all-threads group call fed?**
+
+The measured single-thread frame cost gives the answer directly. Python work is
+load 68.9 ms + correct 30.1 ms = 99.0 ms per frame, GIL-held. Native work is
+~490 ms per frame of block wall, GIL-released, and finding A should take the
+record-dependent part of it down. At 24 threads a group's kernel occupies
+`490 / 24 ~ 20 ms` of wall time per frame, so keeping it fed needs
+`ceil(99.0 / 20.4) = 5` concurrent load-and-correct workers. Their CPU share is
+only the correction part — 5 x 30/99 ~ 1.5 cores of the 24.
+
+Consequences worth stating plainly:
+
+- **The GIL-serialised fraction stops competing with the parallel region.**
+  Findings open item 1 predicts a 6.1x ceiling at 16 workers from a 10.5% serial
+  fraction and measured 4.2x. Double-buffering groups against a correct pool
+  attacks that structurally rather than by tuning the split.
+- **`image_workers = min(len(pending_ranges), ...)` dissolves.** Findings open
+  item 4 — the tail of every job running with fewer workers by construction —
+  goes away, because parallelism now lives inside one group rather than across
+  scheduling ranges.
+- **`_BoundedGate` retargets** from `image_workers + _PREFETCH_QUEUE_SLACK` to
+  `F + slack`: it bounds frames in the group buffer.
+- **The live rebalance survives, with a different variable.** `kernel_threads`
+  is pinned at the budget, so what remains adaptive is `F` and the reader pool
+  size, against memory pressure and delivery rate. The existing hysteresis and
+  backoff machinery (`_REBALANCE_INITIAL_SECONDS`, `_REBALANCE_BACKOFF`) applies
+  unchanged; only the quantity it steers changes.
+- **A group is a synchronisation point.** Group N+1's kernel cannot start until
+  all F of its frames are loaded and corrected. Double-buffering hides this only
+  if the correct pool keeps up; if it does not, the pipeline stalls once per
+  group rather than degrading smoothly. This is the main scheduling risk, and it
+  argues for the smaller F (8) until measured.
+
+## Plan
+
+### Phase 1 — Corner-passing subdivision (finding B)
+
+Independent of everything else, bit-for-bit, one function.
+
+**Mechanism.** A cell carries its own corner values; splitting evaluates only
+the points the split creates. Stationary (quadtree over u, v): 4 corners in,
+3x3 lattice out, **5 new evaluations** shared by 4 children. Moving (octree over
+u, v, t): 8 corners in, 3x3x3 lattice, **19 new** shared by 8 children. Recursive
+descent rather than the explicit stack, so the 9 or 27 lattice values live in the
+caller's frame and children read them directly instead of each pushed cell
+carrying a copy. This is `split_pixel_stationary_depth2` generalised — the
+fastest per-evaluation path in the sweep, and an existing proof the structure
+works.
+
+What is given up is sharing across *parents*: a point on the boundary between
+two cells with different parents is evaluated twice. Bounded, and the depth-2
+path already accepts it.
+
+**Lazy corners.** Neither `split_pixel` nor `same_lattice_voxel` exits early on
+the first disagreeing corner. Under corner-passing that costs nothing for a
+cell's inherited corners, which are already in hand, but it does mean the 5 (or
+19) new points of a split are all materialised even when the first two settle a
+child. Evaluate them on demand, as `split_pixel_stationary_depth2` already does
+with its `evaluated` bitmask.
+
+**Predicted, as acceptance criteria.** With `S` splits, the stationary quadtree
+has `4S+1` cells: uncached probing costs `4 x cells ~ 16S`, corner-passing `5S`,
+perfect sharing `~4S`. The octree has `8S+1` cells: `64S`, `19S`, `~8S`.
+
+| exposure | depth | evals/px now | predicted | note |
+|---|---|---|---|---|
+| stationary | 3 | 57.87 (misses only) | ~72 | count *rises*, time falls |
+| stationary | 4 | 393.06 (all probes) | ~155 | 2.5x |
+| continuous | 3 | 672.92 (misses only) | ~1600 | count rises 2.4x |
+| continuous | 4 | 13777.53 (all probes) | ~4100 | 3.4x |
+
+Note the asymmetry: corner-passing recovers less of the ideal in 3D (19S against
+~8S) than in 2D (5S against ~4S), because 19 new points serve 8 children rather
+than 5 serving 4. It should still win below depth 4 in both, because the dense
+cache's per-probe cost is paid on *all* `16S`/`64S` probes while corner-passing
+pays only for evaluations — but **the depth <= 3 continuous case is the one that
+could plausibly regress**, and it is the case to measure first rather than last.
+
+Time: **2-2.5x at stationary depth >= 4**, **~3x at continuous depth >= 4**,
+and 1.5-2x at depth 3 where the probe overhead disappears. Outside those bands
+the model is wrong and that is worth understanding before proceeding.
+
+**Bit-for-bit is the acceptance criterion.** Same lattice points, same all-same
+test, same centre fallback, same weights; `voxel_id` is deterministic, so
+re-evaluating a shared point yields the identical value. This is a pure
+performance change with no `phys` impact. One thing to preserve deliberately:
+leaf push order. The current stack pushes children 3->0 and pops from the back,
+visiting 0,1,2,3 depth-first; `weights` is later sorted by voxel with a
+non-stable `std::sort` and summed in vector order, so a different visit order
+would change FP association. Recursing children in index order reproduces it.
+
+**Steps.**
+
+1. *Scaffolding, before touching the kernel.* **Done.**
+
+   Golden digests turned out to be the wrong thing to check in: floating-point
+   contraction differs between compilers, so a digest captured on one build
+   fails on another for reasons that have nothing to do with the subdivision.
+   Split in two instead.
+
+   - `orgui/datautils/xrayutils/test/test_reconstruction_subdivision.py` —
+     an independent Python implementation of the adaptive rule, which obtains
+     every voxel through the kernel's own `coordinate()` so it cannot drift on
+     frame conventions or rounding, and reproduces only the *algorithm*: which
+     lattice points are visited, which cells resolve, what weight each leaf
+     carries. It matches the kernel exactly at depths 0-4 stationary and 0-3
+     moving, covering the centre-only fast path, `split_pixel_stationary_depth2`
+     and generic `split_pixel` in both its cached and uncached regimes.
+     Alongside it: weight/intensity conservation at every depth, thread- and
+     block-size independence on full output arrays, sorted-unique output keys,
+     and depth convergence. Portable, and it catches an algorithmic error rather
+     than merely a change. 32 tests, 1.4 s.
+   - `benchmarks/benchmark_reconstruction_subdivision.py` — the depth/exposure
+     cost sweep plus XXH3 fingerprints of all six output arrays, with a
+     `--compare` mode that reports bitwise agreement and per-case speed-up
+     against a captured baseline. Runs on the synthetic geometry or a real job.
+     Baselines live in `benchmarks/baselines/` (gitignored — they are per build
+     and per machine).
+
+   One thing the reference must *not* assert, and now documents: the emitted
+   voxel count is not monotone in depth. A cell still unresolved at `max_depth`
+   contributes at its centre, so raising the depth replaces one centre sample
+   with four elsewhere and a voxel the coarser sample happened to hit can go
+   unvisited. Real frames average that away; a small tile does not.
+2. *Stationary recursion.* **Done.** `subdivide_stationary` /
+   `split_pixel_stationary`: the 2D special case of the same shape — 9-node
+   lattice, 5 new per split, 4 children — with `t` pinned so the rotation is
+   chosen once by the caller rather than recovered from `t` per node.
+
+   | exposure | depth | evals/px was | now | ns/px was | now | speed-up |
+   |---|---|---|---|---|---|---|
+   | stationary | 1 | 11.32 | 11.32 | 707.7 | 430.0 | **1.65x** |
+   | stationary | 2 | 25.89 | 25.89 | 652.0 | 601.5 | 1.08x |
+   | stationary | 3 | 57.44 | 64.61 | 3061.8 | 1502.7 | **2.04x** |
+   | stationary | 4 | 389.67 | 142.87 | 9378.9 | 3147.0 | **2.98x** |
+
+   **`split_pixel_stationary_depth2` is kept, not deleted.** Routing depth 2
+   through the general recursion measured a consistent 10% *regression*
+   (0.90x, best-of-9, on the default `balanced` preset), because that path
+   holds one flat 5x5 lattice for the whole pixel and so shares nodes between
+   *sibling subtrees* — which corner-passing gives up. 22.02 evaluations per
+   pixel against 20.82. Depth 2 therefore still routes to it and the row above
+   is unchanged behaviour.
+
+   That result is worth more than the code it saves: it says the flat lazy
+   lattice is the better structure *where it fits*. The reason to abandon it
+   was never the idea, it was the shape — the removed cache was 3D even for
+   stationary exposures, so at depth 3 it spent 115 KiB to use 289 scattered
+   entries. A flat **2D** lattice is 4.6 KiB contiguous at depth 3, 17 KiB at
+   depth 4, 68 KiB at depth 5. Generalising `split_pixel_stationary_depth2` to
+   arbitrary depth might beat corner-passing across the stationary range the
+   way it does at depth 2. Worth measuring; not attempted here.
+3. *Moving recursion.* **Done, and taken first** — it is where the prize is,
+   and it was the case that might have regressed. `subdivide_moving` /
+   `split_pixel_moving`, 27-node lattice indexed `(iu*3 + iv)*3 + it`, 19 new
+   per split, children visited in index order so leaf emission and therefore
+   floating-point association are unchanged. `accumulate_block` routes
+   `!stationary` to it; `split_pixel` and the dense cache remain, now used only
+   by the stationary path.
+
+   Measured on a real frame, 128x128 tile, one thread, against a baseline
+   captured from the same checkout immediately before the change:
+
+   | exposure | depth | evals/px was | now | ns/px was | now | speed-up |
+   |---|---|---|---|---|---|---|
+   | continuous | 1 | 33.91 | 33.91 | 1510.2 | 815.0 | **1.85x** |
+   | continuous | 2 | 152.12 | 194.43 | 8913.9 | 4291.3 | **2.08x** |
+   | continuous | 3 | 682.52 | 1009.03 | 42748.9 | 22161.5 | **1.93x** |
+   | continuous | 4 | 14031.56 | 4662.31 | 283696.2 | 99668.8 | **2.85x** |
+
+   All fingerprints identical. Stationary unchanged (0.97-1.10x, noise).
+
+   Three things this settled. The feared depth <= 3 regression did not happen:
+   evaluations *rise* by 24-48% from cross-parent duplication, yet time still
+   nearly halves, because the dense cache pays its per-probe cost on all `64S`
+   probes while corner-passing pays only per evaluation. Depth 1 is the clean
+   case — one split, so corner-passing is exactly optimal and the evaluation
+   count is unchanged; the entire 1.85x there is cache overhead removed. And
+   the cross-parent loss is far smaller in practice than the worst-case `19S`
+   against `~8S` model suggests, because not every cell splits.
+
+   Deliberately deferred: lazy corner materialisation. All 19 new points are
+   computed eagerly; only leaf children that disagree early would benefit, and
+   the structural change was worth measuring on its own first.
+4. *Delete.* **Done.** `PixelCoordinateCache`, `CachedVoxel`,
+   `cached_voxel_key`, `begin_pixel`, `struct Cell`, generic `split_pixel`, the
+   `max_depth > 3` cutoff and the per-block `stack`/cache scratch are all gone.
+   `split_pixel_stationary_depth2` stays, for the reason in step 2. Net +133
+   lines (329 added, 196 removed).
+
+   A block's only remaining scratch is the leaf vector: subdivision state now
+   lives on the recursion stack, 9 `LatticeVoxel` per level stationary and 27
+   moving, so ~1.2 KiB / ~3.5 KiB at the deepest setting against a per-worker
+   array that was 115 KiB at depth 3 and would have been 6.3 MiB at depth 5.
+
+5. *Measure.* **Done.** The cliff is gone and `coordinate_evaluations` now
+   means one thing at every depth — distinct evaluations — where before it
+   counted misses below depth 4 and all probes above, which is what made the
+   3-to-4 step read as x6.80. Scaling is now uniformly geometric:
+
+   | depth | stationary evals/px | x prev | continuous evals/px | x prev |
+   |---|---|---|---|---|
+   | 1 | 9.94 | — | 31.05 | — |
+   | 2 | 20.83 | 2.10 | 148.66 | 4.79 |
+   | 3 | 48.16 | 2.31 | 695.15 | 4.68 |
+   | 4 | 102.40 | 2.13 | 3038.42 | 4.37 |
+   | 5 | 212.91 | 2.08 | 12739.36 | 4.19 |
+   | 6 | 435.95 | 2.05 | 52205.21 | 4.10 |
+
+   x2 per level for a 1D boundary set sampled in 2D, x4 for a 2D boundary set
+   sampled in 3D — which is what the adaptive rule should cost and, before
+   this, only did below depth 4.
+
+### Phase 2 — The `worst_leaves` bound (memory, standalone)
+
+Replace `children**depth` in both mirrored sites with a bound derived from
+boundary growth (~2^depth) plus headroom, keeping the two in step. Findings open
+item 2 argues the authoritative cap belongs in the kernel, which knows
+`stationary` at `accumulate()` time while the Python side must assume the worst.
+Unblocks realistic tile sizes — and therefore `image_workers` — from depth 3 up.
+Not required by either finding; do not fold it in.
+
+### Phase 3 — Frame-group bricks at depth 0 (finding A)
+
+1. *Kernel.* `accumulate_group()` taking `(frames, rows, columns)` arrays and
+   `(frames, 2, 4)` angle bounds, returning the same six columnar arrays merged
+   across the group. The block queue becomes a 3D brick index; `block_results`,
+   the per-worker arena and the loser-tree merge are unchanged.
+2. *Brick loop, depth 0.* Precompute the brick's unit centre rays once into a
+   24 KiB stack buffer, iterate frames-outer / pixels-inner. Note separately
+   that at depth 0 the centre ray is frame-invariant for the whole job — pure
+   detector geometry, currently recomputed 3651 times per pixel. A
+   whole-detector centre-ray cache (149 MB, alongside the existing 143 MiB
+   corner cache) removes it entirely, with or without grouping.
+3. *Banding.* Bricks iterate row band by row band so level-3 records stay near
+   today's ~110 MB, as above.
+4. *Choosing F per job.* Extend the calibration probe: run `accumulate_group` on
+   one sample brick at F in {1, 2, 4, 8, 16}, take the smallest F past which the
+   marginal density gain falls below a few per cent, capped by memory. Five small
+   native calls, and it measures the quantity that actually matters rather than
+   deriving it from the angular step. `F = 1` must reproduce today exactly.
+5. *Scheduler.* As above: pin `kernel_threads`, size the correct pool from the
+   measured Python-to-native ratio, retarget `_BoundedGate`, repoint the live
+   rebalance at `F`.
+
+### Phase 4 — Frame-group bricks above depth 0
+
+Only after phases 1 and 3. Pixel-outer / frame-inner, no cross-frame ray cache,
+brick dimensions re-derived against the corrected leaf bound from phase 2. All
+of finding A is measured at depth 0 only; at higher depth a pixel resolves into
+a footprint rather than a point, so overlap should be at least as good, but that
+is an expectation, not a result.
+
+## Risks and open questions
+
+**Interlaced scans.** `orgui/backend/interlacedScanLoader.py` exists, so frames
+adjacent in file order are not always adjacent in angle. Groups must be formed by
+angle adjacency; where that conflicts with sequential I/O or the resume
+frame-range structure, the honest answer is `F = 1`. Check monotonicity of
+`exposure_angle_bounds` per job rather than assuming it.
+
+**Non-uniform angular steps.** The same problem in milder form; the per-job probe
+handles it if it samples more than one place in the scan.
+
+**Bit-for-bit.** Phase 1 must be bit-identical and should be held to that — a
+discrepancy means the lattice or the visit order diverged, and both are bugs.
+Phase 3 is *not* bit-identical: contributions from several frames merge in the
+block map instead of in the tree accumulator. Identical arithmetic, fewer
+intermediate roundings, so slightly better conditioned, but any test pinning an
+XXH3 digest of record values will move. That is a `phys`-scope change and needs
+saying so.
+
+**The group synchronisation point**, as described under Scheduling. Prefer F = 8
+until measured.
+
+**MSVC inlining** of the recursion is an assumption; check the measurement, not
+the intent. The 19-point octree bookkeeping is the fiddly part.
+
+**Depth > 0 for finding A** is unmeasured, as is the interaction between swept
+exposures and cross-frame overlap above depth 0 — each frame then covers a
+0.71-voxel slab rather than a plane, and adjacent slabs tile rather than overlap.
+
+**`excluded_frames`** must be skipped when forming groups, not merely masked
+inside them.
+
+**Records converge at depth 3.** 0.632 -> 0.642 -> 0.647 -> 0.650 across depths
+3-6, while cost rises 3x / 6x / 13x. Everything above depth 3 buys finer
+partial-volume weights, not new voxels. Whether `very_high` and `maximum` change
+a reconstructed intensity measurably is a separate question, but it is worth
+asking before optimising them — it bears directly on how much phases 1 and 2 are
+worth.
+
+## Also
+
+`benchmarks/benchmark_reconstruction_mapping.py` no longer imports: it wants
+`_map_frame_range`, which was replaced by `_map_one_frame`.

From dd6c82cc19bb035364e42a887d9fd9bae46c47de Mon Sep 17 00:00:00 2001
From: "T. Fuchs" <tfuchs@cornell.edu>
Date: Mon, 10 Aug 2026 14:18:25 -0400
Subject: [PATCH 04/33] fix(phys): bound the mapping memory estimate by
 records, not by leaves

Both memory prechecks sized a pixel at its worst-case leaf count,
claiming 32.7 GB for a Pilatus-6M frame at depth 3. That rejected any
useful detector tile and collapsed image_workers to one.

Changes how many frames run concurrently, not any record value.
---
 .../cpp/reciprocal_reconstruction_cpp.cpp     |  49 ++++++---
 .../xrayutils/test/test_reconstruction_job.py | 102 +++++++++++++++++-
 orgui/reconstruction_job.py                   |  38 +++----
 3 files changed, 149 insertions(+), 40 deletions(-)

diff --git a/orgui/datautils/xrayutils/cpp/reciprocal_reconstruction_cpp.cpp b/orgui/datautils/xrayutils/cpp/reciprocal_reconstruction_cpp.cpp
index 4d52c71..3ad51e3 100644
--- a/orgui/datautils/xrayutils/cpp/reciprocal_reconstruction_cpp.cpp
+++ b/orgui/datautils/xrayutils/cpp/reciprocal_reconstruction_cpp.cpp
@@ -514,6 +514,23 @@ struct RecordAccum {
     std::uint32_t contributors = 0;
 };
 
+// Records one pixel is assumed to leave behind, for every purpose that has
+// to size memory before the data is seen: the per-worker arena, and the
+// per-call budget precheck.
+//
+// Sizing by leaf count instead assumes every leaf of every pixel reaches a
+// voxel no other leaf did. Real data does not behave that way at any depth:
+// measured post-dedup density stays between 0.46 and 0.89 records per pixel
+// from depth 0 through 8, and across both exposure models, because deeper
+// subdivision's extra samples overwhelmingly land in voxels a neighbouring
+// sample already reached. Leaves are transient -- they are sorted, merged
+// and discarded per pixel -- while records are what stays resident.
+//
+// Four is therefore several times the density ever observed, and both users
+// below take it as a ceiling on the leaf count rather than a replacement,
+// so depth 0 stays exact: one leaf can only yield one record.
+constexpr std::size_t reserved_records_per_pixel = 4;
+
 struct VoxelWeight {
     std::uint64_t voxel;
     double weight;
@@ -670,8 +687,20 @@ class ReconstructionKernel {
         for (int depth = 0; depth < max_depth_; ++depth) {
             worst_leaves *= subdivision_children;
         }
+        // Bounded by reserved_records_per_pixel for the reason given there.
+        // Unbounded, this term is 4^depth (or 8^depth when the exposure
+        // rotates) and rejects any useful detector tile from depth 3 up: at
+        // depth 3 it claims 5248 bytes for a pixel that really costs about
+        // 106, so a full Pilatus-6M frame "needs" 32.7 GB and the layout is
+        // forced onto tiny tiles -- which then caps how many frames can be
+        // in flight, since one worker's estimate divides into the budget.
+        // Two resident copies of a pixel's records are still allowed for,
+        // covering the per-block vectors and the merged output together.
         const std::size_t estimated_bytes_per_pixel =
-            128 + 2 * worst_leaves * sizeof(Record);
+            128
+            + 2
+                * std::min<std::size_t>(worst_leaves, reserved_records_per_pixel)
+                * sizeof(Record);
         if (
             pixels > 0
             && estimated_bytes_per_pixel
@@ -699,20 +728,10 @@ class ReconstructionKernel {
         // performance risk, never a correctness one.
         const std::size_t bytes_per_node =
             sizeof(std::pair<const RecordKey, RecordAccum>) + 32;
-        // Reserving one node per leaf assumes every leaf of every pixel
-        // reaches a voxel no other leaf did. Real data does not behave
-        // that way at any depth: measured post-dedup density stays
-        // between 0.46 and 0.87 records per pixel from depth 0 through 8,
-        // because deeper subdivision's extra samples overwhelmingly land
-        // in voxels a neighbouring sample already reached. Reserving by
-        // leaf count therefore over-reserves by up to four orders of
-        // magnitude at high depth -- gigabytes per worker thread -- while
-        // buying nothing. A small multiple of the measured density leaves
-        // ample headroom, and the resource falls back to the heap if a
-        // block ever does exceed it, so this bounds performance, never
-        // correctness. Depth 0 is exact: one leaf can only yield one
-        // record.
-        constexpr std::size_t reserved_records_per_pixel = 4;
+        // Sized by reserved_records_per_pixel, as the precheck above is.
+        // Undersizing here is a performance risk and never a correctness
+        // one: the resource falls back to the heap transparently if a
+        // block ever does exceed it.
         const std::size_t arena_bytes =
             block_size
                 * std::min<std::size_t>(worst_leaves, reserved_records_per_pixel)
diff --git a/orgui/datautils/xrayutils/test/test_reconstruction_job.py b/orgui/datautils/xrayutils/test/test_reconstruction_job.py
index e641006..6a024ab 100644
--- a/orgui/datautils/xrayutils/test/test_reconstruction_job.py
+++ b/orgui/datautils/xrayutils/test/test_reconstruction_job.py
@@ -20,6 +20,7 @@
     WORK_BLOCK_PRESETS,
     _FRAME_RECORD_BYTES_PER_PIXEL,
     _PYTHON_CORRECTION_BYTES_PER_PIXEL,
+    _RESERVED_RECORDS_PER_PIXEL,
     ReconstructionGrid,
     ReconstructionJob,
     _frame_parallelism,
@@ -229,7 +230,7 @@ def test_frame_parallelism_accounts_for_a_frames_own_records():
     tiles = [(0, 1024, 0, 1024)]
 
     _workers, _threads, per_worker, _accumulation = _frame_parallelism(
-        spec, tiles, 8 * 1024**3, stationary=True, threads_per_image=1
+        spec, tiles, 8 * 1024**3, threads_per_image=1
     )
 
     detector_pixels = 1024 * 1024
@@ -321,10 +322,10 @@ def test_frame_parallelism_scopes_native_memory_to_largest_tile_not_detector_sum
     ) == sum((r1 - r0) * (c1 - c0) for r0, r1, c0, c1 in four_small_tiles)
 
     single_result = _frame_parallelism(
-        spec, one_big_tile, memory_bytes, stationary=False, threads_per_image=1
+        spec, one_big_tile, memory_bytes, threads_per_image=1
     )
     split_result = _frame_parallelism(
-        spec, four_small_tiles, memory_bytes, stationary=False, threads_per_image=1
+        spec, four_small_tiles, memory_bytes, threads_per_image=1
     )
 
     # The split (4x smaller largest tile) version's per-worker memory
@@ -335,6 +336,99 @@ def test_frame_parallelism_scopes_native_memory_to_largest_tile_not_detector_sum
     assert split_result[0] >= single_result[0]
 
 
+def test_frame_parallelism_native_estimate_does_not_grow_with_depth():
+    """The per-pixel native estimate is bounded by records, not leaves.
+
+    Sized by the worst-case leaf count it grows as ``4**depth`` (or
+    ``8**depth`` when the exposure rotates), which claimed 5248 bytes at
+    depth 3 for a pixel that really costs about 106 and so forced any
+    realistic detector onto tiny tiles. Because one worker's estimate
+    divides into the budget, that also capped how many frames could be in
+    flight. Leaves are transient; records are what stays resident.
+    """
+    grid = _GridSpec(
+        minimum=(-1.0, -1.0, -1.0),
+        maximum=(1.0, 1.0, 1.0),
+        step=(1.0, 1.0, 1.0),
+        frame="lab",
+        chunk_shape=(2, 2, 2),
+    )
+    tiles = _uniform_tiles(2048, 2048)
+    memory_bytes = 10_000 * 1024**2
+    results = {
+        depth: _frame_parallelism(
+            _ReconstructionSpec(grids=(grid,), max_depth=depth, threads=24),
+            tiles,
+            memory_bytes,
+            threads_per_image=1,
+        )
+        for depth in (0, 1, 2, 3, 5, 8)
+    }
+    # Depth 0 is exact -- one leaf can only reach one voxel -- so it stays
+    # below the saturated bound; every deeper setting shares one value.
+    assert results[0][2] < results[1][2]
+    deep = {depth: value[2] for depth, value in results.items() if depth}
+    assert len(set(deep.values())) == 1, deep
+    # And the concurrency that estimate funds must not collapse with depth.
+    assert results[8][0] == results[1][0]
+    assert results[8][0] > 1
+
+
+def test_frame_parallelism_native_estimate_mirrors_the_kernel_precheck():
+    """The Python estimate and the kernel's own guard must agree.
+
+    They are two statements of one bound, and the kernel throws where this
+    side merely sizes a pool, so a tile this side considers affordable must
+    not be one the kernel rejects.
+    """
+    native = pytest.importorskip(
+        "orgui.datautils.xrayutils._reciprocal_reconstruction_cpp"
+    )
+    rows = columns = 512
+    pixels = rows * columns
+    rays = np.zeros((rows + 1, columns + 1, 3), dtype=np.float64)
+    rays[..., 1] = 1.0
+    arguments = (
+        np.ones((rows, columns)),
+        np.ones((rows, columns)),
+        np.zeros((rows, columns), dtype=bool),
+        rays,
+        np.zeros(4),
+        np.zeros(4),
+    )
+
+    def kernel_for(max_depth, memory_budget_bytes):
+        return native.ReconstructionKernel(
+            np.array([-1.0, -1.0, -1.0]),
+            np.array([0.01, 0.01, 0.01]),
+            np.array([256, 256, 256], dtype=np.int64),
+            np.array([16, 16, 16], dtype=np.int64),
+            "lab",
+            1.0,
+            np.eye(3),
+            np.eye(3),
+            max_depth,
+            1,
+            64,
+            memory_budget_bytes,
+        )
+
+    for max_depth in (0, 1, 3, 5, 8):
+        expected = (
+            128 + 2 * _RESERVED_RECORDS_PER_PIXEL * 40
+            if max_depth
+            else 128 + 2 * 40
+        )
+        # Exactly what the Python estimate asks for must be accepted. This
+        # is the direction that pins the two together: a leaf-sized bound
+        # demands 4**depth times more and rejects this outright from depth
+        # 2 up.
+        kernel_for(max_depth, pixels * expected).accumulate(*arguments)
+        # One byte per pixel short of it must not be.
+        with pytest.raises(ValueError, match="native memory budget"):
+            kernel_for(max_depth, pixels * (expected - 1)).accumulate(*arguments)
+
+
 def test_frame_parallelism_accounts_for_python_correction_buffers_and_prefetch():
     """The native-kernel-only estimate misses two real, non-native memory
     costs (design doc Sec7): _correction_pipeline.correct_frame's
@@ -360,7 +454,7 @@ def test_frame_parallelism_accounts_for_python_correction_buffers_and_prefetch()
 
     image_workers, _kernel_threads, per_worker_memory, _accumulation = (
         _frame_parallelism(
-            spec, tiles, memory_bytes, stationary=True, threads_per_image=1
+            spec, tiles, memory_bytes, threads_per_image=1
         )
     )
 
diff --git a/orgui/reconstruction_job.py b/orgui/reconstruction_job.py
index 1d41d77..aba591c 100644
--- a/orgui/reconstruction_job.py
+++ b/orgui/reconstruction_job.py
@@ -1423,7 +1423,6 @@ def _frame_parallelism(
     tiles,
     memory_bytes,
     *,
-    stationary,
     frames_per_task=1,
     threads_per_image=1,
     accumulation_budget_bytes=None,
@@ -1470,8 +1469,6 @@ def _frame_parallelism(
         column_stop)`` tuples.
     :param int memory_bytes:
         Total memory budget in bytes.
-    :param bool stationary:
-        Whether exposure start and end angles are identical.
     :param int frames_per_task:
         Number of images streamed by one resumable task. Images are not
         retained together.
@@ -1501,14 +1498,25 @@ def _frame_parallelism(
     # always full-detector-sized regardless of how many native tiles the
     # frame is split into.
     detector_pixels = sum(tile_pixels)
-    children = 4 if stationary else 8
-    worst_leaves = children**spec.max_depth
     # Mirrors ReconstructionKernel::accumulate's own memory precheck
-    # exactly (a fixed per-pixel baseline plus twice the worst-case leaf
-    # count times one native Record's on-the-wire size), so this
-    # Python-side estimate cannot drift from what the kernel itself
+    # exactly (a fixed per-pixel baseline plus twice the records a pixel is
+    # assumed to leave behind, times one native Record's on-the-wire size),
+    # so this Python-side estimate cannot drift from what the kernel itself
     # actually enforces.
-    native_bytes_per_pixel = 128 + 2 * worst_leaves * _CHECKPOINT_BYTES_PER_ROW
+    #
+    # That estimate used to be the worst-case *leaf* count, 4**depth or
+    # 8**depth, which is where the two could drift: a spec cannot know
+    # whether an individual frame range's exposure rotates, so this side had
+    # to assume the rotating case and reserve twice what the kernel would.
+    # Bounding both by the same record ceiling removes the asymmetry
+    # outright -- above depth 0 the bound saturates, so the estimate no
+    # longer depends on the exposure model at all, and the two sides now
+    # agree by construction rather than by careful mirroring.
+    native_bytes_per_pixel = (
+        128 + 2 * _RESERVED_RECORDS_PER_PIXEL * _CHECKPOINT_BYTES_PER_ROW
+        if spec.max_depth > 0
+        else 128 + 2 * _CHECKPOINT_BYTES_PER_ROW
+    )
     native_memory = largest_tile_pixels * native_bytes_per_pixel
     # The native estimate alone misses real, per-in-flight-frame Python
     # memory: the raw decoded image plus every corrected array
@@ -1645,7 +1653,6 @@ def reconstruction_execution_settings(job, scan=None, config=None):
                 spec,
                 tiles,
                 scheduler_memory,
-                stationary=stationary,
                 frames_per_task=stop - start,
                 threads_per_image=effective_threads_per_image,
                 accumulation_budget_bytes=job.accumulation_budget_bytes,
@@ -2251,16 +2258,11 @@ def layout_for(threads_per_image_seed):
         workers = []
         natives = []
         for frame_range in pending_ranges:
-            task_bounds = bounds[frame_range[0] : frame_range[1]]
-            stationary = bool(
-                np.array_equal(task_bounds[:, 0], task_bounds[:, 1])
-            )
             image_limit, native_threads, _worker_memory, _accumulation_limit = (
                 _frame_parallelism(
                     spec,
                     tiles,
                     scheduler_memory,
-                    stationary=stationary,
                     frames_per_task=frame_range[1] - frame_range[0],
                     threads_per_image=threads_per_image_seed,
                     accumulation_budget_bytes=accumulation_budget_bytes,
@@ -2317,11 +2319,6 @@ def layout_for(threads_per_image_seed):
         sweep_rays = ray_arrays[sweep_tile]
         sweep_angles_start = np.ascontiguousarray(bounds[frame_indices[0], 0])
         sweep_angles_end = np.ascontiguousarray(bounds[frame_indices[0], 1])
-        sweep_stationary = bool(
-            np.array_equal(
-                bounds[frame_indices[0], 0], bounds[frame_indices[0], 1]
-            )
-        )
         # Every tile of a frame is mapped, so a frame's kernel work is the
         # whole detector's pixels -- what the sweep's single sample tile
         # has to be scaled up to.
@@ -2597,7 +2594,6 @@ def compute_loop(retire, pool_cancellation):
                             spec,
                             tiles,
                             scheduler_memory,
-                            stationary=sweep_stationary,
                             threads_per_image=candidate_threads,
                             accumulation_budget_bytes=accumulation_budget_bytes,
                         )

From 6b2c8558a7cd7924d0aa08eecf7fbbdc07fa9cfe Mon Sep 17 00:00:00 2001
From: "T. Fuchs" <tfuchs@cornell.edu>
Date: Mon, 10 Aug 2026 14:18:46 -0400
Subject: [PATCH 05/33] docs: record the phase 2 memory-bound result

What the record-based bound unblocked, and why it resolved the
Python/kernel drift by removing the exposure dependence from both sides.
---
 ..._space_mapping_locality_and_subdivision.md | 34 ++++++++++++++++++-
 1 file changed, 33 insertions(+), 1 deletion(-)

diff --git a/doc/design/reciprocal_space_mapping_locality_and_subdivision.md b/doc/design/reciprocal_space_mapping_locality_and_subdivision.md
index cbc37b4..27d5e17 100644
--- a/doc/design/reciprocal_space_mapping_locality_and_subdivision.md
+++ b/doc/design/reciprocal_space_mapping_locality_and_subdivision.md
@@ -543,7 +543,39 @@ would change FP association. Recursing children in index order reproduces it.
    sampled in 3D — which is what the adaptive rule should cost and, before
    this, only did below depth 4.
 
-### Phase 2 — The `worst_leaves` bound (memory, standalone)
+### Phase 2 — The `worst_leaves` bound (memory, standalone). Done.
+
+Both prechecks are now bounded by `reserved_records_per_pixel`, the constant
+the arena already used, hoisted to file scope in the kernel so the two users
+share one definition. Measured on the real detector against the job's own
+10 GiB budget:
+
+| depth | tiles | before | after |
+|---|---|---|---|
+| 0 | 1 | 5 workers, 1.93 GiB | unchanged |
+| 2 | 1 | 1 worker, 9.63 GiB | 3 workers, 3.21 GiB |
+| 3 | 1 | 1 worker, 31.03 GiB | 3 workers, 3.21 GiB |
+| 5 | 1 | 1 worker, 476.2 GiB | 3 workers, 3.21 GiB |
+| 3 | 4 bands | 1 worker, 9.63 GiB | 7 workers, 1.38 GiB |
+| 5 | 4 bands | 1 worker, 119.6 GiB | 7 workers, 1.38 GiB |
+
+It also closed the drift risk findings open item 2 raised, but not the way
+that item anticipated. The concern was that a spec cannot know whether a
+frame range's exposure rotates, so the Python side had to assume `8**depth`
+where the kernel knew it was `4**depth` — and the conclusion drawn was that
+the authoritative cap therefore belonged in the kernel. With the bound
+saturating above depth 0, neither side depends on the exposure model at all,
+so the asymmetry disappears rather than being relocated. `_frame_parallelism`
+no longer takes `stationary`, and the two call sites that computed it only to
+pass it no longer do.
+
+What was *not* done: a per-thread term for the leaf buffer. It is the one
+genuinely exposure- and depth-dependent allocation left, and it is the reason
+the original bound looked plausible — but it is bounded by thread count
+rather than tile size, and at 24 threads it reaches ~14 MB at moving depth 6
+and ~230 MB at depth 8. Too small to model against a multi-gigabyte budget.
+
+### Phase 2 — original reasoning, retained
 
 Replace `children**depth` in both mirrored sites with a bound derived from
 boundary growth (~2^depth) plus headroom, keeping the two in step. Findings open

From 10e8199084ef21c82d7acead781f98b43ad51c6b Mon Sep 17 00:00:00 2001
From: "T. Fuchs" <tfuchs@cornell.edu>
Date: Mon, 10 Aug 2026 14:35:23 -0400
Subject: [PATCH 06/33] feat(phys): map adjacent frames together in one native
 call

Consecutive frames land as close in reciprocal space as adjacent pixels,
so a (row, column, frame) brick emits 0.575x the records. Not
bit-for-bit, though every voxel and contributor count is unchanged.

accumulate() is untouched and remains the per-image path.
---
 .../cpp/reciprocal_reconstruction_cpp.cpp     | 597 ++++++++++++++++--
 .../test/test_reconstruction_group.py         | 309 +++++++++
 2 files changed, 863 insertions(+), 43 deletions(-)
 create mode 100644 orgui/datautils/xrayutils/test/test_reconstruction_group.py

diff --git a/orgui/datautils/xrayutils/cpp/reciprocal_reconstruction_cpp.cpp b/orgui/datautils/xrayutils/cpp/reciprocal_reconstruction_cpp.cpp
index 3ad51e3..7d8b52a 100644
--- a/orgui/datautils/xrayutils/cpp/reciprocal_reconstruction_cpp.cpp
+++ b/orgui/datautils/xrayutils/cpp/reciprocal_reconstruction_cpp.cpp
@@ -545,6 +545,15 @@ bool operator<(const VoxelWeight &left, const VoxelWeight &right) {
     return left.voxel < right.voxel;
 }
 
+// One work block of a frame group: a rectangle of the detector, taken
+// across every frame of the group.
+struct BrickExtent {
+    std::size_t row_begin;
+    std::size_t row_end;
+    std::size_t column_begin;
+    std::size_t column_end;
+};
+
 struct BlockProfile {
     std::uint64_t pixels_seen = 0;
     std::uint64_t valid_pixels = 0;
@@ -800,53 +809,240 @@ class ReconstructionKernel {
             records = merge_sorted_blocks(block_results);
         }
         const auto merge_finished = std::chrono::steady_clock::now();
-        const auto conversion_started = merge_finished;
         py::dict result = records_to_python(records);
         const auto conversion_finished = std::chrono::steady_clock::now();
         if (profile) {
-            BlockProfile combined;
-            for (const BlockProfile &block : block_profiles) {
-                combined.pixels_seen += block.pixels_seen;
-                combined.valid_pixels += block.valid_pixels;
-                combined.coordinate_evaluations += block.coordinate_evaluations;
-                combined.voxel_weights += block.voxel_weights;
-                combined.maximum_weights_per_pixel = std::max(
-                    combined.maximum_weights_per_pixel,
-                    block.maximum_weights_per_pixel
-                );
-                combined.unreduced_records += block.unreduced_records;
-                combined.reduced_records += block.reduced_records;
-                combined.mapping_nanoseconds += block.mapping_nanoseconds;
-                combined.reduction_nanoseconds += block.reduction_nanoseconds;
+            result["_profile"] = summarize_profile(
+                block_profiles,
+                record_count,
+                records.size(),
+                total_started,
+                blocks_started,
+                blocks_finished,
+                merge_started,
+                merge_finished,
+                conversion_finished
+            );
+        }
+        return result;
+    }
+
+    py::dict accumulate_group(
+        const FloatArray &intensity,
+        const FloatArray &variance,
+        const BoolArray &mask,
+        const FloatArray &corner_rays,
+        const FloatArray &angles_start,
+        const FloatArray &angles_end,
+        const bool profile = false
+    ) const {
+        // Several consecutive images mapped in one call, with the work
+        // block a brick in (row, column, frame) rather than a run of one
+        // image. On a rotation scan two adjacent frames are as close in
+        // reciprocal space as two adjacent pixels are -- measured 0.71
+        // voxels per frame against 0.49 and 0.80 for the next column and
+        // row -- so a brick collapses redundancy the per-image block
+        // cannot see, and it does so in the same cache-resident map.
+        //
+        // The caller chooses the group size by how many frames it passes;
+        // one frame reproduces accumulate() exactly.
+        const auto total_started = std::chrono::steady_clock::now();
+        const py::buffer_info intensity_info = intensity.request();
+        const py::buffer_info variance_info = variance.request();
+        const py::buffer_info mask_info = mask.request();
+        const py::buffer_info ray_info = corner_rays.request();
+        const py::buffer_info start_info = angles_start.request();
+        const py::buffer_info end_info = angles_end.request();
+        validate_group_inputs(
+            intensity_info,
+            variance_info,
+            mask_info,
+            ray_info,
+            start_info,
+            end_info
+        );
+
+        const auto *intensity_data =
+            static_cast<const double *>(intensity_info.ptr);
+        const auto *variance_data =
+            static_cast<const double *>(variance_info.ptr);
+        const auto *mask_data = static_cast<const bool *>(mask_info.ptr);
+        const auto *ray_data = static_cast<const double *>(ray_info.ptr);
+        const auto *start_data = static_cast<const double *>(start_info.ptr);
+        const auto *end_data = static_cast<const double *>(end_info.ptr);
+
+        const std::size_t frames =
+            static_cast<std::size_t>(intensity_info.shape[0]);
+        const std::size_t rows =
+            static_cast<std::size_t>(intensity_info.shape[1]);
+        const std::size_t cols =
+            static_cast<std::size_t>(intensity_info.shape[2]);
+        const std::size_t frame_pixels = rows * cols;
+        const std::size_t samples = frames * frame_pixels;
+
+        // Exposure geometry is per frame: a group may mix stationary and
+        // swept frames, and each needs its own rotation lattice.
+        std::vector<std::vector<CoordinateTransform>> transforms(frames);
+        std::vector<char> stationary(frames, 1);
+        for (std::size_t frame = 0; frame < frames; ++frame) {
+            const double *frame_start = start_data + frame * 4;
+            const double *frame_end = end_data + frame * 4;
+            bool still = true;
+            for (int index = 0; index < 4; ++index) {
+                still = still && frame_start[index] == frame_end[index];
             }
-            const auto seconds = [](const auto start, const auto stop) {
-                return std::chrono::duration<double>(stop - start).count();
-            };
-            py::dict details;
-            details["pixels_seen"] = combined.pixels_seen;
-            details["valid_pixels"] = combined.valid_pixels;
-            details["coordinate_evaluations"] =
-                combined.coordinate_evaluations;
-            details["voxel_weights"] = combined.voxel_weights;
-            details["maximum_weights_per_pixel"] =
-                combined.maximum_weights_per_pixel;
-            details["unreduced_block_records"] =
-                combined.unreduced_records;
-            details["reduced_block_records"] = combined.reduced_records;
-            details["pre_merge_records"] = record_count;
-            details["final_records"] = records.size();
-            details["block_mapping_cpu_seconds"] =
-                static_cast<double>(combined.mapping_nanoseconds) * 1.0e-9;
-            details["block_reduction_cpu_seconds"] =
-                static_cast<double>(combined.reduction_nanoseconds) * 1.0e-9;
-            details["block_wall_seconds"] =
-                seconds(blocks_started, blocks_finished);
-            details["merge_seconds"] =
-                seconds(merge_started, merge_finished);
-            details["python_conversion_seconds"] =
-                seconds(conversion_started, conversion_finished);
-            details["total_seconds"] =
-                seconds(total_started, conversion_finished);
+            stationary[frame] = still ? 1 : 0;
+            for (const FrameRotation &rotation :
+                 frame_rotations(frame_start, frame_end)) {
+                transforms[frame].push_back(coordinate_transform(rotation));
+            }
+        }
+
+        std::size_t worst_leaves = 1;
+        for (int depth = 0; depth < max_depth_; ++depth) {
+            worst_leaves *= 8;
+        }
+        const std::size_t estimated_bytes_per_pixel =
+            128
+            + 2
+                * std::min<std::size_t>(worst_leaves, reserved_records_per_pixel)
+                * sizeof(Record);
+        if (
+            samples > 0
+            && estimated_bytes_per_pixel > memory_budget_bytes_ / samples
+        ) {
+            throw py::value_error(
+                "Frame group exceeds the native memory budget at the "
+                "configured adaptive depth; use fewer frames per group, a "
+                "smaller detector tile, or increase memory_budget_bytes"
+            );
+        }
+
+        // One brick holds the same sample count a single-image block would,
+        // so the map's working set is unchanged; the frame extent is spent
+        // out of the detector-plane extent rather than added to it.
+        const std::size_t samples_per_brick = bounded_block_size();
+        const std::size_t plane_pixels = std::max<std::size_t>(
+            1, samples_per_brick / std::max<std::size_t>(frames, 1)
+        );
+        std::size_t brick_columns = std::max<std::size_t>(
+            1,
+            static_cast<std::size_t>(
+                std::sqrt(static_cast<double>(plane_pixels))
+            )
+        );
+        brick_columns = std::min(brick_columns, cols);
+        std::size_t brick_rows =
+            std::max<std::size_t>(1, plane_pixels / brick_columns);
+        brick_rows = std::min(brick_rows, rows);
+
+        std::vector<BrickExtent> bricks;
+        for (std::size_t row = 0; row < rows; row += brick_rows) {
+            for (std::size_t column = 0; column < cols; column += brick_columns) {
+                bricks.push_back({
+                    row,
+                    std::min(row + brick_rows, rows),
+                    column,
+                    std::min(column + brick_columns, cols),
+                });
+            }
+        }
+        const std::size_t brick_count = bricks.size();
+        std::vector<std::vector<Record>> block_results(brick_count);
+        std::vector<BlockProfile> block_profiles(profile ? brick_count : 0);
+        std::atomic<std::size_t> next_brick{0};
+
+        const std::size_t bytes_per_node =
+            sizeof(std::pair<const RecordKey, RecordAccum>) + 32;
+        const std::size_t arena_bytes =
+            brick_rows * brick_columns * frames
+                * std::min<std::size_t>(worst_leaves, reserved_records_per_pixel)
+                * bytes_per_node
+            + 4096;
+
+        const auto blocks_started = std::chrono::steady_clock::now();
+        {
+            py::gil_scoped_release release;
+            const int worker_count = static_cast<int>(
+                std::min<std::size_t>(
+                    static_cast<std::size_t>(threads_), std::max<std::size_t>(brick_count, 1)
+                )
+            );
+            std::vector<std::thread> workers;
+            workers.reserve(static_cast<std::size_t>(worker_count));
+            for (int worker = 0; worker < worker_count; ++worker) {
+                workers.emplace_back([&, this]() {
+                    std::unique_ptr<std::byte[]> arena_buffer(
+                        new std::byte[arena_bytes]
+                    );
+                    std::pmr::monotonic_buffer_resource arena(
+                        arena_buffer.get(), arena_bytes
+                    );
+                    std::vector<Vec3> centre_rays;
+                    std::vector<VoxelWeight> weights;
+                    weights.reserve(
+                        static_cast<std::size_t>(1)
+                        << std::min(3 * max_depth_, 9)
+                    );
+                    while (true) {
+                        const std::size_t index = next_brick.fetch_add(1);
+                        if (index >= brick_count) {
+                            break;
+                        }
+                        block_results[index] = accumulate_brick(
+                            bricks[index],
+                            frames,
+                            rows,
+                            cols,
+                            intensity_data,
+                            variance_data,
+                            mask_data,
+                            ray_data,
+                            transforms,
+                            stationary,
+                            centre_rays,
+                            weights,
+                            profile ? &block_profiles[index] : nullptr,
+                            arena
+                        );
+                    }
+                });
+            }
+            for (auto &worker : workers) {
+                worker.join();
+            }
+        }
+        const auto blocks_finished = std::chrono::steady_clock::now();
+
+        std::size_t record_count = 0;
+        for (const auto &block : block_results) {
+            record_count += block.size();
+        }
+        std::vector<Record> records;
+        const auto merge_started = std::chrono::steady_clock::now();
+        {
+            py::gil_scoped_release release;
+            records = merge_sorted_blocks(block_results);
+        }
+        const auto merge_finished = std::chrono::steady_clock::now();
+        py::dict result = records_to_python(records);
+        const auto conversion_finished = std::chrono::steady_clock::now();
+        if (profile) {
+            py::dict details = summarize_profile(
+                block_profiles,
+                record_count,
+                records.size(),
+                total_started,
+                blocks_started,
+                blocks_finished,
+                merge_started,
+                merge_finished,
+                conversion_finished
+            );
+            details["frames"] = frames;
+            details["bricks"] = brick_count;
+            details["brick_rows"] = brick_rows;
+            details["brick_columns"] = brick_columns;
             result["_profile"] = std::move(details);
         }
         return result;
@@ -944,6 +1140,110 @@ class ReconstructionKernel {
         );
     }
 
+    static py::dict summarize_profile(
+        const std::vector<BlockProfile> &block_profiles,
+        const std::size_t pre_merge_records,
+        const std::size_t final_records,
+        const std::chrono::steady_clock::time_point total_started,
+        const std::chrono::steady_clock::time_point blocks_started,
+        const std::chrono::steady_clock::time_point blocks_finished,
+        const std::chrono::steady_clock::time_point merge_started,
+        const std::chrono::steady_clock::time_point merge_finished,
+        const std::chrono::steady_clock::time_point conversion_finished
+    ) {
+        BlockProfile combined;
+        for (const BlockProfile &block : block_profiles) {
+            combined.pixels_seen += block.pixels_seen;
+            combined.valid_pixels += block.valid_pixels;
+            combined.coordinate_evaluations += block.coordinate_evaluations;
+            combined.voxel_weights += block.voxel_weights;
+            combined.maximum_weights_per_pixel = std::max(
+                combined.maximum_weights_per_pixel,
+                block.maximum_weights_per_pixel
+            );
+            combined.unreduced_records += block.unreduced_records;
+            combined.reduced_records += block.reduced_records;
+            combined.mapping_nanoseconds += block.mapping_nanoseconds;
+            combined.reduction_nanoseconds += block.reduction_nanoseconds;
+        }
+        const auto seconds = [](const auto start, const auto stop) {
+            return std::chrono::duration<double>(stop - start).count();
+        };
+        py::dict details;
+        details["pixels_seen"] = combined.pixels_seen;
+        details["valid_pixels"] = combined.valid_pixels;
+        details["coordinate_evaluations"] = combined.coordinate_evaluations;
+        details["voxel_weights"] = combined.voxel_weights;
+        details["maximum_weights_per_pixel"] =
+            combined.maximum_weights_per_pixel;
+        details["unreduced_block_records"] = combined.unreduced_records;
+        details["reduced_block_records"] = combined.reduced_records;
+        details["pre_merge_records"] = pre_merge_records;
+        details["final_records"] = final_records;
+        details["block_mapping_cpu_seconds"] =
+            static_cast<double>(combined.mapping_nanoseconds) * 1.0e-9;
+        details["block_reduction_cpu_seconds"] =
+            static_cast<double>(combined.reduction_nanoseconds) * 1.0e-9;
+        details["block_wall_seconds"] = seconds(blocks_started, blocks_finished);
+        details["merge_seconds"] = seconds(merge_started, merge_finished);
+        details["python_conversion_seconds"] =
+            seconds(merge_finished, conversion_finished);
+        details["total_seconds"] = seconds(total_started, conversion_finished);
+        return details;
+    }
+
+    static void validate_group_inputs(
+        const py::buffer_info &intensity,
+        const py::buffer_info &variance,
+        const py::buffer_info &mask,
+        const py::buffer_info &rays,
+        const py::buffer_info &start,
+        const py::buffer_info &end
+    ) {
+        if (intensity.ndim != 3) {
+            throw py::value_error(
+                "intensity must be a three-dimensional (frames, rows, columns) "
+                "array"
+            );
+        }
+        for (const py::buffer_info *other : {&variance, &mask}) {
+            if (
+                other->ndim != 3
+                || other->shape[0] != intensity.shape[0]
+                || other->shape[1] != intensity.shape[1]
+                || other->shape[2] != intensity.shape[2]
+            ) {
+                throw py::value_error(
+                    "variance and mask must match the intensity shape"
+                );
+            }
+        }
+        if (intensity.shape[0] < 1) {
+            throw py::value_error("A frame group needs at least one frame");
+        }
+        if (
+            rays.ndim != 3
+            || rays.shape[0] != intensity.shape[1] + 1
+            || rays.shape[1] != intensity.shape[2] + 1
+            || rays.shape[2] != 3
+        ) {
+            throw py::value_error(
+                "corner_rays must have shape (rows + 1, columns + 1, 3)"
+            );
+        }
+        for (const py::buffer_info *angles : {&start, &end}) {
+            if (
+                angles->ndim != 2
+                || angles->shape[0] != intensity.shape[0]
+                || angles->shape[1] != 4
+            ) {
+                throw py::value_error(
+                    "angles_start and angles_end must have shape (frames, 4)"
+                );
+            }
+        }
+    }
+
     static void validate_inputs(
         const py::buffer_info &intensity,
         const py::buffer_info &variance,
@@ -1768,6 +2068,206 @@ class ReconstructionKernel {
         );
     }
 
+    std::vector<Record> accumulate_brick(
+        const BrickExtent &brick,
+        const std::size_t frames,
+        const std::size_t rows,
+        const std::size_t columns,
+        const double *intensity,
+        const double *variance,
+        const bool *mask,
+        const double *rays,
+        const std::vector<std::vector<CoordinateTransform>> &transforms,
+        const std::vector<char> &stationary,
+        std::vector<Vec3> &centre_rays,
+        std::vector<VoxelWeight> &weights,
+        BlockProfile *profile,
+        std::pmr::monotonic_buffer_resource &arena
+    ) const {
+        (void)rows;
+        arena.release();
+        std::pmr::map<RecordKey, RecordAccum> tree(&arena);
+        std::uint64_t unreduced_count = 0;
+        RecordKey cached_key{};
+        RecordAccum *cached_accumulator = nullptr;
+        const auto accumulator_for = [&tree, &cached_key, &cached_accumulator](
+            const RecordKey key
+        ) -> RecordAccum & {
+            if (cached_accumulator != nullptr && key == cached_key) {
+                return *cached_accumulator;
+            }
+            RecordAccum &accumulator = tree[key];
+            cached_key = key;
+            cached_accumulator = &accumulator;
+            return accumulator;
+        };
+        const std::size_t brick_rows = brick.row_end - brick.row_begin;
+        const std::size_t brick_columns = brick.column_end - brick.column_begin;
+        const std::size_t frame_stride = rows * columns;
+
+        if (max_depth_ == 0) {
+            // A pixel's centre ray is pure detector geometry -- the frame's
+            // rotation enters only through the transform applied after it --
+            // so it is the same for every frame in the group and is computed
+            // once per brick instead of once per (pixel, frame). At 32x32
+            // that buffer is 24 KiB.
+            centre_rays.resize(brick_rows * brick_columns);
+            for (std::size_t row = brick.row_begin; row < brick.row_end; ++row) {
+                for (
+                    std::size_t column = brick.column_begin;
+                    column < brick.column_end;
+                    ++column
+                ) {
+                    centre_rays[
+                        (row - brick.row_begin) * brick_columns
+                        + (column - brick.column_begin)
+                    ] = ray_at(pixel_rays(row, column, columns, rays), 0.5, 0.5);
+                }
+            }
+        }
+
+        const auto mapping_started = std::chrono::steady_clock::now();
+        for (std::size_t frame = 0; frame < frames; ++frame) {
+            const std::vector<CoordinateTransform> &frame_transforms =
+                transforms[frame];
+            const CoordinateTransform &centre_transform =
+                frame_transforms[frame_transforms.size() / 2];
+            for (std::size_t row = brick.row_begin; row < brick.row_end; ++row) {
+                const std::size_t row_offset =
+                    frame * frame_stride + row * columns;
+                for (
+                    std::size_t column = brick.column_begin;
+                    column < brick.column_end;
+                    ++column
+                ) {
+                    const std::size_t flat = row_offset + column;
+                    if (profile != nullptr) {
+                        ++profile->pixels_seen;
+                    }
+                    if (
+                        mask[flat]
+                        || !std::isfinite(intensity[flat])
+                        || !std::isfinite(variance[flat])
+                        || variance[flat] < 0.0
+                    ) {
+                        continue;
+                    }
+                    if (profile != nullptr) {
+                        ++profile->valid_pixels;
+                    }
+                    if (max_depth_ == 0) {
+                        if (profile != nullptr) {
+                            ++profile->coordinate_evaluations;
+                        }
+                        std::uint64_t voxel = 0;
+                        if (
+                            voxel_id(
+                                coordinate_from_unit_ray(
+                                    centre_rays[
+                                        (row - brick.row_begin) * brick_columns
+                                        + (column - brick.column_begin)
+                                    ],
+                                    centre_transform
+                                ),
+                                voxel
+                            )
+                        ) {
+                            if (profile != nullptr) {
+                                ++profile->voxel_weights;
+                                profile->maximum_weights_per_pixel = std::max(
+                                    profile->maximum_weights_per_pixel,
+                                    static_cast<std::uint64_t>(1)
+                                );
+                            }
+                            RecordAccum &accumulator =
+                                accumulator_for(record_key(voxel));
+                            accumulator.weighted_intensity += intensity[flat];
+                            accumulator.weighted_variance += variance[flat];
+                            accumulator.weight += 1.0;
+                            accumulator.contributors += 1;
+                            ++unreduced_count;
+                        }
+                        continue;
+                    }
+                    const PixelRays prepared_rays =
+                        pixel_rays(row, column, columns, rays);
+                    weights.clear();
+                    if (stationary[frame] != 0 && max_depth_ == 2) {
+                        split_pixel_stationary_depth2(
+                            prepared_rays, centre_transform, weights, profile
+                        );
+                    } else if (stationary[frame] != 0) {
+                        split_pixel_stationary(
+                            prepared_rays, centre_transform, weights, profile
+                        );
+                    } else {
+                        split_pixel_moving(
+                            prepared_rays, frame_transforms, weights, profile
+                        );
+                    }
+                    if (profile != nullptr) {
+                        profile->voxel_weights += weights.size();
+                        profile->maximum_weights_per_pixel = std::max(
+                            profile->maximum_weights_per_pixel,
+                            static_cast<std::uint64_t>(weights.size())
+                        );
+                    }
+                    std::sort(weights.begin(), weights.end());
+                    std::size_t offset = 0;
+                    while (offset < weights.size()) {
+                        const std::uint64_t voxel = weights[offset].voxel;
+                        double weight = 0.0;
+                        do {
+                            weight += weights[offset].weight;
+                            ++offset;
+                        } while (
+                            offset < weights.size()
+                            && weights[offset].voxel == voxel
+                        );
+                        RecordAccum &accumulator =
+                            accumulator_for(record_key(voxel));
+                        accumulator.weighted_intensity +=
+                            weight * intensity[flat];
+                        accumulator.weighted_variance +=
+                            weight * weight * variance[flat];
+                        accumulator.weight += weight;
+                        accumulator.contributors += 1;
+                        ++unreduced_count;
+                    }
+                }
+            }
+        }
+        const auto mapping_finished = std::chrono::steady_clock::now();
+        if (profile != nullptr) {
+            profile->unreduced_records = unreduced_count;
+            profile->mapping_nanoseconds = static_cast<std::uint64_t>(
+                std::chrono::duration_cast<std::chrono::nanoseconds>(
+                    mapping_finished - mapping_started
+                ).count()
+            );
+        }
+        std::vector<Record> records;
+        records.reserve(tree.size());
+        for (const auto &entry : tree) {
+            records.push_back({
+                entry.first,
+                entry.second.weighted_intensity,
+                entry.second.weighted_variance,
+                entry.second.weight,
+                entry.second.contributors,
+            });
+        }
+        if (profile != nullptr) {
+            profile->reduced_records = records.size();
+            profile->reduction_nanoseconds = static_cast<std::uint64_t>(
+                std::chrono::duration_cast<std::chrono::nanoseconds>(
+                    std::chrono::steady_clock::now() - mapping_finished
+                ).count()
+            );
+        }
+        return records;
+    }
+
     std::vector<Record> accumulate_block(
         const std::size_t begin,
         const std::size_t end,
@@ -2233,6 +2733,17 @@ PYBIND11_MODULE(_reciprocal_reconstruction_cpp, module) {
             py::arg("angles_end"),
             py::arg("profile") = false
         )
+        .def(
+            "accumulate_group",
+            &ReconstructionKernel::accumulate_group,
+            py::arg("intensity"),
+            py::arg("variance"),
+            py::arg("mask"),
+            py::arg("corner_rays"),
+            py::arg("angles_start"),
+            py::arg("angles_end"),
+            py::arg("profile") = false
+        )
         .def(
             "coordinate",
             &ReconstructionKernel::coordinate,
diff --git a/orgui/datautils/xrayutils/test/test_reconstruction_group.py b/orgui/datautils/xrayutils/test/test_reconstruction_group.py
new file mode 100644
index 0000000..87feecd
--- /dev/null
+++ b/orgui/datautils/xrayutils/test/test_reconstruction_group.py
@@ -0,0 +1,309 @@
+"""Tests for mapping several adjacent frames in one native call.
+
+``accumulate_group`` exists to collapse redundancy that spans frames: on a
+rotation scan two adjacent images land as close together in reciprocal space
+as two adjacent pixels do, so a work block shaped as a brick in (row,
+column, frame) shares voxels the per-image block never sees. What it must
+not change is the answer -- a group has to produce what merging the same
+frames one at a time produces, to floating-point association.
+"""
+
+from __future__ import annotations
+
+import numpy as np
+import pytest
+
+from orgui.datautils.xrayutils.reconstruction import _reduce_batches
+
+
+native = pytest.importorskip(
+    "orgui.datautils.xrayutils._reciprocal_reconstruction_cpp"
+)
+
+
+ROWS, COLUMNS = 24, 28
+WAVEVECTOR = 7.601
+UB = np.array(
+    [
+        [1.6122, -0.5878, 0.0000],
+        [0.0000, 1.4899, -0.1051],
+        [0.0000, 0.0000, 1.2083],
+    ]
+)
+
+
+def _rays(rows=ROWS, columns=COLUMNS):
+    """Unit rays at pixel corners of a flat detector, placed off-axis."""
+    row_edges = np.arange(rows + 1) + 300 - 0.5
+    column_edges = np.arange(columns + 1) + 400 - 0.5
+    x = np.broadcast_to(column_edges[None, :] * 0.172, (rows + 1, columns + 1))
+    z = np.broadcast_to(row_edges[:, None] * 0.172, (rows + 1, columns + 1))
+    rays = np.stack((x, np.full_like(x, 500.0), z), axis=-1).astype(np.float64)
+    rays /= np.linalg.norm(rays, axis=-1, keepdims=True)
+    return np.ascontiguousarray(rays)
+
+
+def _angles(frames, moving):
+    """Per-frame exposure bounds for a 0.1 deg/frame rotation."""
+    step = np.deg2rad(0.1)
+    start = np.zeros((frames, 4))
+    start[:, 0] = 0.10471975511965978
+    start[:, 1] = 0.7 + step * np.arange(frames)
+    end = start.copy()
+    if moving:
+        end[:, 1] += step
+    return np.ascontiguousarray(start), np.ascontiguousarray(end)
+
+
+def _grid(rays, start, end, voxels_per_pixel=0.5):
+    """A grid whose voxel is a chosen multiple of the sample pitch."""
+    probe = native.ReconstructionKernel(
+        np.full(3, -1e12),
+        np.ones(3),
+        np.full(3, 2_000_000, dtype=np.int64),
+        np.ones(3, dtype=np.int64),
+        "hkl",
+        WAVEVECTOR,
+        np.ascontiguousarray(np.linalg.inv(UB)),
+        np.ascontiguousarray(np.eye(3)),
+        0,
+        1,
+        1,
+        1024 * 1024,
+    )
+
+    def at(frame, row, column, t):
+        return np.asarray(
+            probe.coordinate(
+                rays, start[frame], end[frame], row, column, 0.5, 0.5, t
+            )
+        )
+
+    centre = at(0, 0, 0, 0.5)
+    pitch = np.maximum.reduce(
+        [
+            np.abs(at(0, 0, 1, 0.5) - centre),
+            np.abs(at(0, 1, 0, 0.5) - centre),
+            np.abs(at(0, 0, 0, 1.0) - at(0, 0, 0, 0.0)),
+            np.abs(at(len(start) - 1, 0, 0, 0.5) - centre) / max(1, len(start) - 1),
+        ]
+    )
+    step = np.maximum(pitch, np.max(pitch) * 0.2) / voxels_per_pixel
+    corners = np.asarray(
+        [
+            at(frame, row, column, t)
+            for frame in (0, len(start) - 1)
+            for row in (0, ROWS - 1)
+            for column in (0, COLUMNS - 1)
+            for t in (0.0, 1.0)
+        ]
+    )
+    minimum = corners.min(axis=0) - 4.0 * step - 0.31718 * step
+    shape = np.ceil(
+        (corners.max(axis=0) + 4.0 * step - minimum) / step
+    ).astype(np.int64)
+    return minimum, step, shape
+
+
+def _kernel(minimum, step, shape, max_depth, threads, block):
+    return native.ReconstructionKernel(
+        np.ascontiguousarray(minimum),
+        np.ascontiguousarray(step),
+        np.ascontiguousarray(shape),
+        np.array([16, 16, 16], dtype=np.int64),
+        "hkl",
+        WAVEVECTOR,
+        np.ascontiguousarray(np.linalg.inv(UB)),
+        np.ascontiguousarray(np.eye(3)),
+        max_depth,
+        threads,
+        block,
+        1 << 40,
+    )
+
+
+def _payload(frames, seed=0):
+    rng = np.random.default_rng(seed)
+    intensity = rng.uniform(1.0, 100.0, size=(frames, ROWS, COLUMNS))
+    variance = np.abs(rng.normal(4.0, 0.5, size=(frames, ROWS, COLUMNS)))
+    mask = np.zeros((frames, ROWS, COLUMNS), dtype=bool)
+    mask[:, ROWS // 3, COLUMNS // 2] = True
+    mask[frames // 2, 0, 0] = True
+    return intensity, variance, mask
+
+
+def _case(frames, moving, max_depth, threads=1, block=256):
+    rays = _rays()
+    start, end = _angles(frames, moving)
+    minimum, step, shape = _grid(rays, start, end)
+    kernel = _kernel(minimum, step, shape, max_depth, threads, block)
+    intensity, variance, mask = _payload(frames)
+    return kernel, rays, start, end, intensity, variance, mask
+
+
+def _per_frame_reference(kernel, rays, start, end, intensity, variance, mask):
+    """What the same frames produce mapped one at a time, then merged."""
+    batches = [
+        kernel.accumulate(
+            np.ascontiguousarray(intensity[frame]),
+            np.ascontiguousarray(variance[frame]),
+            np.ascontiguousarray(mask[frame]),
+            rays,
+            np.ascontiguousarray(start[frame]),
+            np.ascontiguousarray(end[frame]),
+        )
+        for frame in range(intensity.shape[0])
+    ]
+    return _reduce_batches(batches)
+
+
+def _as_mapping(batch):
+    return dict(
+        zip(
+            zip(
+                batch["chunk_id"].tolist(),
+                batch["local_voxel_id"].tolist(),
+            ),
+            zip(
+                batch["weighted_intensity"].tolist(),
+                batch["weighted_variance"].tolist(),
+                batch["weight"].tolist(),
+                batch["contributors"].tolist(),
+            ),
+        )
+    )
+
+
+@pytest.mark.parametrize("moving", [False, True])
+@pytest.mark.parametrize("max_depth", [0, 1, 2])
+@pytest.mark.parametrize("frames", [1, 2, 8])
+def test_group_matches_frames_mapped_one_at_a_time(frames, max_depth, moving):
+    """The scientific contract: grouping changes cost, never the answer.
+
+    Contributions from several frames now meet in the block map rather than
+    in the checkpoint accumulator, so the sums associate differently and are
+    compared to tolerance -- but every voxel, every contributor count and
+    every total must agree.
+    """
+    case = _case(frames, moving, max_depth)
+    kernel, rays, start, end, intensity, variance, mask = case
+    grouped = _as_mapping(
+        kernel.accumulate_group(intensity, variance, mask, rays, start, end)
+    )
+    expected = _as_mapping(_per_frame_reference(*case))
+
+    assert len(expected) > 1, "geometry must reach more than one voxel"
+    assert set(grouped) == set(expected)
+    for key, (found, variance_, weight, contributors) in grouped.items():
+        reference = expected[key]
+        assert contributors == reference[3], key
+        assert found == pytest.approx(reference[0], rel=1e-12), key
+        assert variance_ == pytest.approx(reference[1], rel=1e-12), key
+        assert weight == pytest.approx(reference[2], rel=1e-12), key
+
+
+def test_single_frame_group_reaches_the_same_voxels_as_accumulate():
+    """A group of one is the fallback for scans a brick cannot help.
+
+    It is scientifically identical to the per-image path but **not**
+    bit-for-bit: a brick partitions the detector into rectangles where a
+    block partitions it into runs of the flattened image, so pixels group
+    into per-block accumulators differently and their sums associate
+    differently. Every voxel and every contributor count still agrees
+    exactly; the totals agree to rounding.
+    """
+    for max_depth in (0, 1, 2, 3):
+        case = _case(1, False, max_depth)
+        kernel, rays, start, end, intensity, variance, mask = case
+        grouped = kernel.accumulate_group(
+            intensity, variance, mask, rays, start, end
+        )
+        single = kernel.accumulate(
+            np.ascontiguousarray(intensity[0]),
+            np.ascontiguousarray(variance[0]),
+            np.ascontiguousarray(mask[0]),
+            rays,
+            np.ascontiguousarray(start[0]),
+            np.ascontiguousarray(end[0]),
+        )
+        for name in ("chunk_id", "local_voxel_id", "contributors"):
+            np.testing.assert_array_equal(grouped[name], single[name], name)
+        for name in ("weighted_intensity", "weighted_variance", "weight"):
+            np.testing.assert_allclose(
+                grouped[name], single[name], rtol=1e-14, err_msg=name
+            )
+
+
+@pytest.mark.parametrize("moving", [False, True])
+def test_group_is_independent_of_threads_and_brick_size(moving):
+    """Splitting the group across workers or bricks changes nothing."""
+    reference = None
+    for threads, block in ((1, 4096), (1, 64), (4, 128), (8, 32)):
+        kernel, rays, start, end, intensity, variance, mask = _case(
+            8, moving, 0, threads=threads, block=block
+        )
+        candidate = kernel.accumulate_group(
+            intensity, variance, mask, rays, start, end
+        )
+        if reference is None:
+            reference = candidate
+            continue
+        for name in ("chunk_id", "local_voxel_id", "contributors"):
+            np.testing.assert_array_equal(
+                candidate[name], reference[name], f"{name} {threads}/{block}"
+            )
+        for name in ("weighted_intensity", "weighted_variance", "weight"):
+            np.testing.assert_allclose(
+                candidate[name],
+                reference[name],
+                rtol=1e-12,
+                err_msg=f"{name} {threads}/{block}",
+            )
+
+
+def test_group_emits_fewer_records_than_the_frames_do_apart():
+    """The point of the exercise, on geometry that has the redundancy.
+
+    Adjacent frames must collapse into shared voxels, so a group emits
+    strictly fewer records than the same frames mapped separately -- while
+    reaching exactly the same set of voxels, which the equivalence test
+    above already pins.
+    """
+    case = _case(8, False, 0)
+    kernel, rays, start, end, intensity, variance, mask = case
+    grouped = kernel.accumulate_group(
+        intensity, variance, mask, rays, start, end, True
+    )
+    apart = sum(
+        kernel.accumulate(
+            np.ascontiguousarray(intensity[frame]),
+            np.ascontiguousarray(variance[frame]),
+            np.ascontiguousarray(mask[frame]),
+            rays,
+            np.ascontiguousarray(start[frame]),
+            np.ascontiguousarray(end[frame]),
+            True,
+        )["_profile"]["reduced_block_records"]
+        for frame in range(intensity.shape[0])
+    )
+    brick_records = grouped["_profile"]["reduced_block_records"]
+    assert brick_records < apart
+    # The pre-merge saving is what reaches the checkpoint layer, so it is
+    # worth pinning that it is a substantial fraction rather than a token.
+    assert brick_records < 0.85 * apart
+
+
+def test_group_rejects_mismatched_shapes():
+    kernel, rays, start, end, intensity, variance, mask = _case(4, False, 0)
+    with pytest.raises(ValueError, match="three-dimensional"):
+        kernel.accumulate_group(
+            intensity[0], variance[0], mask[0], rays, start, end
+        )
+    with pytest.raises(ValueError, match="match the intensity shape"):
+        kernel.accumulate_group(
+            intensity, variance[:2], mask, rays, start, end
+        )
+    with pytest.raises(ValueError, match=r"\(frames, 4\)"):
+        kernel.accumulate_group(
+            intensity, variance, mask, rays, start[:2], end
+        )

From 4dc40045fa51721e9445f9887b0caa5c19a25a82 Mon Sep 17 00:00:00 2001
From: "T. Fuchs" <tfuchs@cornell.edu>
Date: Mon, 10 Aug 2026 14:35:54 -0400
Subject: [PATCH 07/33] docs: record the frame-group kernel result

Measured record and time ratios against the geometric prediction, plus
the correction that a single-frame group is not bit-for-bit.
---
 ..._space_mapping_locality_and_subdivision.md | 42 +++++++++++++++++--
 1 file changed, 38 insertions(+), 4 deletions(-)

diff --git a/doc/design/reciprocal_space_mapping_locality_and_subdivision.md b/doc/design/reciprocal_space_mapping_locality_and_subdivision.md
index 27d5e17..c35c070 100644
--- a/doc/design/reciprocal_space_mapping_locality_and_subdivision.md
+++ b/doc/design/reciprocal_space_mapping_locality_and_subdivision.md
@@ -586,10 +586,44 @@ Not required by either finding; do not fold it in.
 
 ### Phase 3 — Frame-group bricks at depth 0 (finding A)
 
-1. *Kernel.* `accumulate_group()` taking `(frames, rows, columns)` arrays and
-   `(frames, 2, 4)` angle bounds, returning the same six columnar arrays merged
-   across the group. The block queue becomes a 3D brick index; `block_results`,
-   the per-worker arena and the loser-tree merge are unchanged.
+1. *Kernel.* **Done.** `accumulate_group()` takes `(frames, rows, columns)`
+   arrays and `(frames, 4)` angle bounds and returns the same six columnar
+   arrays merged across the group. The block queue is a brick index;
+   `block_results`, the per-worker arena and the loser-tree merge are
+   unchanged. The caller sets the group size purely by how many frames it
+   passes, and one brick holds the same sample count a single-image block did,
+   so the frame extent is spent out of the detector-plane extent rather than
+   added to it.
+
+   Measured on the real job, 512 x 2463 tile, depth 0, against the same frames
+   mapped one at a time:
+
+   | group | brick | records | time |
+   |---|---|---|---|
+   | 1 | 128x128x1 | x0.966 | x0.903 |
+   | 2 | 91x90x2 | x0.752 | x0.752 |
+   | 4 | 64x64x4 | x0.645 | x0.715 |
+   | 8 | 45x45x8 | x0.595 | x0.606 |
+   | 16 | 32x32x16 | **x0.575** | **x0.621** |
+
+   Record ratios at one thread, time ratios at twelve. The record figure
+   matches the exhaustive geometric prediction of 0.5749 made before any of it
+   was implemented, which retires the main open question of finding A.
+
+   It also settles the one the plan flagged as unpredictable: **the kernel
+   itself is 1.6x faster**, not merely the stages downstream. About 42% fewer
+   distinct keys are inserted into the block map, and the pixel centre ray --
+   pure detector geometry, unchanged by the rotation -- is computed once per
+   brick instead of once per (pixel, frame). A single-frame group is already
+   1.1x faster from the block shape alone.
+
+   One correction to the plan: **`F = 1` is not bit-for-bit with today.** A
+   brick partitions the detector into rectangles where a block partitions it
+   into runs of the flattened image, so pixels group into accumulators
+   differently and their sums associate differently. Same voxels, same
+   contributor counts, totals equal to rounding. The fallback is
+   scientifically identical rather than byte-identical — and slightly better,
+   at 3.4% fewer records.
 2. *Brick loop, depth 0.* Precompute the brick's unit centre rays once into a
    24 KiB stack buffer, iterate frames-outer / pixels-inner. Note separately
    that at depth 0 the centre ray is frame-invariant for the whole job — pure

From 8ad06e4451966dd376f35d23f837864bf22c352c Mon Sep 17 00:00:00 2001
From: "T. Fuchs" <tfuchs@cornell.edu>
Date: Mon, 10 Aug 2026 14:48:29 -0400
Subject: [PATCH 08/33] test: add the frame-group benchmark used to measure
 accumulate_group

Compares group bricks against the same frames mapped one at a time,
reporting records before the cross-block merge and wall time.
---
 benchmarks/benchmark_reconstruction_group.py | 255 +++++++++++++++++++
 1 file changed, 255 insertions(+)
 create mode 100644 benchmarks/benchmark_reconstruction_group.py

diff --git a/benchmarks/benchmark_reconstruction_group.py b/benchmarks/benchmark_reconstruction_group.py
new file mode 100644
index 0000000..97ee6f1
--- /dev/null
+++ b/benchmarks/benchmark_reconstruction_group.py
@@ -0,0 +1,255 @@
+"""Frame-group bricks against per-image blocks, on a prepared job.
+
+``accumulate_group`` maps several adjacent images in one call, with the work
+block a brick in (row, column, frame) rather than a run of one flattened
+image. This measures what that is worth at each group size: records emitted
+before the cross-block merge -- which is what every stage downstream of the
+kernel scales with -- and wall time.
+
+The per-image arm is the same frames through ``accumulate``, one at a time,
+merged with ``_reduce_batches``, i.e. exactly what the pipeline does today.
+
+**Run this with ``PYTHONPATH`` pointing at the checkout.** ``python
+benchmarks/<script>.py`` puts ``benchmarks/`` on ``sys.path``, not the
+repository root, so a pip-installed ``orgui`` shadows the working tree and
+the sweep silently measures a binary you did not just build. The resolved
+package path is reported for that reason.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+from pathlib import Path
+from time import perf_counter
+
+import numpy as np
+
+from orgui.datautils.xrayutils.reconstruction import (
+    _detector_corner_rays,
+    _kernel_for_grid,
+    _native_module,
+    _reduce_batches,
+)
+from orgui.reconstruction_job import (
+    _correction_pipeline,
+    _load_assets,
+    read_job,
+)
+
+
+RECORD_FIELDS = (
+    "chunk_id",
+    "local_voxel_id",
+    "weighted_intensity",
+    "weighted_variance",
+    "weight",
+    "contributors",
+)
+
+
+def _arguments():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("job", type=Path)
+    parser.add_argument("--frame", type=int, default=1800)
+    parser.add_argument("--groups", type=int, nargs="+", default=[1, 2, 4, 8, 16])
+    parser.add_argument("--rows", type=int, default=512)
+    parser.add_argument("--row-origin", type=int, default=1000)
+    parser.add_argument("--threads", type=int, default=1)
+    parser.add_argument("--repeat", type=int, default=2)
+    parser.add_argument("--output", type=Path)
+    return parser.parse_args()
+
+
+def _load_window(job, frames, tile):
+    """Corrected intensity, variance and mask for one window of frames."""
+    config = job.config_data
+    scan = job.scan
+    correct = _correction_pipeline(config, scan, _load_assets(job), {})
+    selection = np.s_[tile[0] : tile[1], tile[2] : tile[3]]
+    rows = tile[1] - tile[0]
+    columns = tile[3] - tile[2]
+    intensity = np.empty((len(frames), rows, columns))
+    variance = np.empty_like(intensity)
+    mask = np.empty(intensity.shape, dtype=bool)
+    for slot, frame in enumerate(frames):
+        payload = scan.get_raw_img(frame)
+        corrected = correct.correct_frame(
+            payload, np.asarray(payload.img), frame
+        )
+        intensity[slot] = corrected[0][selection]
+        variance[slot] = corrected[1][selection]
+        mask[slot] = corrected[2][selection]
+    return intensity, variance, mask
+
+
+def _best_of(repeat, call):
+    best = None
+    for _ in range(repeat):
+        started = perf_counter()
+        result = call()
+        elapsed = perf_counter() - started
+        if best is None or elapsed < best[0]:
+            best = (elapsed, result)
+    return best
+
+
+def _per_frame_arm(kernel, rays, start, end, payload, repeat):
+    """Today's path: every frame mapped alone, then merged."""
+    intensity, variance, mask = payload
+    seconds = 0.0
+    block_records = 0
+    batches = []
+    for slot in range(intensity.shape[0]):
+        elapsed, batch = _best_of(
+            repeat,
+            lambda slot=slot: kernel.accumulate(
+                np.ascontiguousarray(intensity[slot]),
+                np.ascontiguousarray(variance[slot]),
+                np.ascontiguousarray(mask[slot]),
+                rays,
+                np.ascontiguousarray(start[slot]),
+                np.ascontiguousarray(end[slot]),
+                True,
+            ),
+        )
+        seconds += elapsed
+        block_records += int(batch["_profile"]["reduced_block_records"])
+        batches.append({name: batch[name] for name in RECORD_FIELDS})
+    merged = _reduce_batches(batches)
+    return {
+        "mode": "per-frame",
+        "frames": intensity.shape[0],
+        "block_records": block_records,
+        "records_after_merge": int(merged["chunk_id"].size),
+        "seconds": seconds,
+    }
+
+
+def _group_arm(kernel, rays, start, end, payload, group, repeat):
+    """Frames mapped in windows of ``group``, covering as many as fit."""
+    intensity, variance, mask = payload
+    frames = intensity.shape[0]
+    windows = max(1, frames // group)
+    seconds = 0.0
+    block_records = 0
+    records = 0
+    brick = None
+    for origin in range(0, windows * group, group):
+        window = slice(origin, origin + group)
+        elapsed, batch = _best_of(
+            repeat,
+            lambda window=window: kernel.accumulate_group(
+                np.ascontiguousarray(intensity[window]),
+                np.ascontiguousarray(variance[window]),
+                np.ascontiguousarray(mask[window]),
+                rays,
+                np.ascontiguousarray(start[window]),
+                np.ascontiguousarray(end[window]),
+                True,
+            ),
+        )
+        seconds += elapsed
+        block_records += int(batch["_profile"]["reduced_block_records"])
+        records += int(batch["chunk_id"].size)
+        brick = [
+            int(batch["_profile"]["brick_rows"]),
+            int(batch["_profile"]["brick_columns"]),
+            int(batch["_profile"]["frames"]),
+        ]
+    return {
+        "mode": "group",
+        "group": group,
+        "frames": windows * group,
+        "brick": brick,
+        "block_records": block_records,
+        "records_after_merge": records,
+        "seconds": seconds,
+    }
+
+
+def main():
+    """Compare frame-group bricks against per-image blocks."""
+    arguments = _arguments()
+    job = read_job(arguments.job)
+    config = job.config_data
+    spec = job.internal_spec()
+    grid = spec.grids[0]
+    bounds = np.asarray(
+        job.scan.exposure_angle_bounds(config, fallback=job.angle_fallback),
+        dtype=np.float64,
+    )
+    _rows, columns = config.detector.detector.shape
+    tile = (
+        arguments.row_origin,
+        arguments.row_origin + arguments.rows,
+        0,
+        columns,
+    )
+    biggest = max(arguments.groups)
+    frames = list(range(arguments.frame, arguments.frame + biggest))
+    payload = _load_window(job, frames, tile)
+    rays = _detector_corner_rays(config.detector, tile)
+    start = np.ascontiguousarray(bounds[frames, 0])
+    end = np.ascontiguousarray(bounds[frames, 1])
+    kernel = _kernel_for_grid(
+        spec, grid, config.ub_calculator, threads=arguments.threads
+    )
+    valid_samples = int((~payload[2]).sum())
+
+    header = {
+        "job": str(arguments.job),
+        "tile": list(tile),
+        "frames": frames[0],
+        "valid_samples": valid_samples,
+        "max_depth": spec.max_depth,
+        "work_block_pixels": spec.work_block_pixels,
+        "threads": arguments.threads,
+        "native": _native_module().__file__,
+    }
+    print(json.dumps(header, sort_keys=True), flush=True)
+
+    baseline = _per_frame_arm(
+        kernel, rays, start, end, payload, arguments.repeat
+    )
+    print(json.dumps(baseline, sort_keys=True), flush=True)
+
+    results = [baseline]
+    for group in arguments.groups:
+        if group > biggest:
+            continue
+        entry = _group_arm(
+            kernel, rays, start, end, payload, group, arguments.repeat
+        )
+        share = entry["frames"] / baseline["frames"]
+        entry["relative_block_records"] = entry["block_records"] / (
+            baseline["block_records"] * share
+        )
+        entry["relative_seconds"] = entry["seconds"] / (
+            baseline["seconds"] * share
+        )
+        results.append(entry)
+        print(json.dumps(entry, sort_keys=True), flush=True)
+
+    print()
+    print(
+        f"{'group':>6} {'brick':>16} {'block records':>14} "
+        f"{'vs per-frame':>13} {'time':>8}"
+    )
+    for entry in results[1:]:
+        brick = "x".join(str(value) for value in entry["brick"])
+        print(
+            f"{entry['group']:>6} {brick:>16} {entry['block_records']:>14,} "
+            f"{entry['relative_block_records']:>13.3f} "
+            f"{entry['relative_seconds']:>8.3f}"
+        )
+
+    if arguments.output:
+        arguments.output.write_text(
+            json.dumps({"header": header, "results": results}, indent=2),
+            encoding="utf-8",
+        )
+
+
+if __name__ == "__main__":
+    main()

From dfe47572be55d51645b54c1217055ded601fd595 Mon Sep 17 00:00:00 2001
From: "T. Fuchs" <tfuchs@cornell.edu>
Date: Mon, 10 Aug 2026 16:22:28 -0400
Subject: [PATCH 09/33] feat(phys)!: map frames in groups through one native
 call

The pipeline now calls accumulate_group, making a work block a brick in
(row, column, frame) rather than a run of one image. Records reaching
the checkpoint router fall to 0.598x at eight frames per group.

BREAKING CHANGE: reconstructed values move in the last bits for every
job, including those mapping one frame at a time, since contributions
now merge inside the kernel rather than in the checkpoint accumulator.
Which voxels are reached and how many samples reach each are unchanged.
Checkpoints stay resumable, but a job resumed across this change holds
parts from both association orders.
---
 orgui/datautils/xrayutils/reconstruction.py   | 177 +++++++++++----
 .../xrayutils/test/test_reconstruction.py     | 139 ++++++++++--
 .../xrayutils/test/test_reconstruction_job.py | 166 +++++++++++++-
 orgui/reconstruction_job.py                   | 212 +++++++++++++++---
 4 files changed, 605 insertions(+), 89 deletions(-)

diff --git a/orgui/datautils/xrayutils/reconstruction.py b/orgui/datautils/xrayutils/reconstruction.py
index 796d3c0..06183d6 100644
--- a/orgui/datautils/xrayutils/reconstruction.py
+++ b/orgui/datautils/xrayutils/reconstruction.py
@@ -11,7 +11,7 @@
 
 from __future__ import annotations
 
-from collections.abc import Callable, Iterable, Mapping
+from collections.abc import Callable, Iterable, Mapping, Sequence
 from dataclasses import asdict, dataclass, field
 from hashlib import sha256
 import bisect
@@ -186,6 +186,15 @@ class _ReconstructionSpec:
     #: (see ReconstructionJob.internal_spec, which derives it); 4096 is the
     #: measured optimum for this class's own default depth of 2.
     work_block_pixels: int = 4096
+    #: Frames mapped in one native call. On a rotation scan two adjacent
+    #: frames are as close in reciprocal space as two adjacent pixels are
+    #: (measured 0.71 voxels per frame against 0.49 and 0.80 for the next
+    #: column and row), so a work block that is a brick in (row, column,
+    #: frame) collapses redundancy a per-image block cannot see. One frame
+    #: per call is the degenerate case of the same machinery, not a
+    #: separate path. Past roughly one voxel of travel per frame the gain
+    #: is gone, so this is a per-job quantity rather than a constant.
+    frames_per_group: int = 1
     memory_budget_bytes: int = 512 * 1024 * 1024
     checkpoint_count: int = 10
     compression: str = "bitshuffle-lz4"
@@ -209,6 +218,8 @@ def __post_init__(self):
             raise ValueError("threads must be positive")
         if self.work_block_pixels < 1:
             raise ValueError("work_block_pixels must be positive")
+        if self.frames_per_group < 1:
+            raise ValueError("frames_per_group must be positive")
         if self.memory_budget_bytes < 1024 * 1024:
             raise ValueError("memory_budget_bytes must be at least 1 MiB")
         if self.checkpoint_count < 1:
@@ -1202,8 +1213,23 @@ def checkpoint_index_for_frame(self, grid_name, frame_index) -> int:
             )
         return position
 
-    def route(self, grid_name, frame_index, batch) -> None:
-        """Insert one frame's reduced batch into its checkpoint accumulator.
+    def route(self, grid_name, frame_index, batch, *, frames=1) -> None:
+        """Insert one reduced batch into its checkpoint accumulator.
+
+        ``frames`` is how many frames the batch covers, starting at
+        ``frame_index``: one for a single-frame batch, ``F`` for a frame
+        group mapped in one native call, whose contributions merge inside
+        the kernel and can no longer be told apart afterwards. The
+        checkpoint's own remaining-frame countdown and the
+        ``frames_covered`` attribute that makes a part resumable both
+        count frames, not ``route()`` calls, so a group must declare its
+        own size here or its checkpoint would never reach zero and could
+        never be resumed from.
+
+        A group must lie entirely inside one planned checkpoint range and
+        must be contiguous in frame index; both are checked, since getting
+        either wrong corrupts the resume bookkeeping silently rather than
+        loudly.
 
         A no-op if that ``(grid, checkpoint)`` was already resumed from a
         previous run. May flush the checkpoint (or a part of it, under the
@@ -1226,7 +1252,18 @@ def route(self, grid_name, frame_index, batch) -> None:
         running -- which would otherwise silently lose that frame's
         contribution to a ``finalize()``-reset accumulator.
         """
+        frames = max(1, int(frames))
         index = self.checkpoint_index_for_frame(grid_name, frame_index)
+        if frames > 1:
+            last = self.checkpoint_index_for_frame(
+                grid_name, int(frame_index) + frames - 1
+            )
+            if last != index:
+                raise ValueError(
+                    f"Frame group [{int(frame_index)}, "
+                    f"{int(frame_index) + frames}) straddles grid "
+                    f"{grid_name!r}'s planned checkpoint boundaries"
+                )
         key = (grid_name, index)
         with self._lock:
             if key in self._resumed:
@@ -1236,12 +1273,14 @@ def route(self, grid_name, frame_index, batch) -> None:
                 accumulator = _CheckpointAccumulator()
                 self._accumulators[key] = accumulator
             self._inflight[key] = self._inflight.get(key, 0) + 1
-            self._remaining[key] -= 1
+            self._remaining[key] -= frames
         accumulator.insert(batch)
         to_flush = None
         with self._lock:
             self._inflight[key] -= 1
-            self._frames_in_part[key] = self._frames_in_part.get(key, 0) + 1
+            self._frames_in_part[key] = (
+                self._frames_in_part.get(key, 0) + frames
+            )
             # Gate both flush triggers on no-other-insert-in-flight-for-this-
             # key, not just "done": an over-budget flush racing a still-
             # running sibling insert() for the same key would tear down
@@ -1444,29 +1483,43 @@ def _build_kernels(spec, ub_calculator, *, threads=None, memory_budget_bytes=Non
     }
 
 
-def _map_one_frame(
+def _map_frame_group(
     spec: _ReconstructionSpec,
     kernels: Mapping[str, object],
     ray_arrays: Mapping[tuple[int, int, int, int], np.ndarray],
     detector_tiles: Iterable[tuple[int, int, int, int]],
     correction_pipeline: Callable,
-    image_payload: object,
-    frame_index: int,
+    image_payloads: Sequence[object],
+    frame_indices: Sequence[int],
     angles_start: np.ndarray,
     angles_end: np.ndarray,
     router: _CheckpointRouter,
 ) -> None:
-    """Process one already-loaded frame: correction, per-tile-per-grid
-    kernel accumulate, merge, and route.
-
-    This is the compute-side half of what ``_map_frame_range``'s loop
-    body used to do in one serial per-frame step (design doc Sec7):
-    image loading (I/O) happens separately, in the prefetch pipeline, so
-    ``image_payload`` here is already loaded -- this function does no
-    I/O of its own and is safe to call concurrently for different frames
-    from multiple compute workers, provided each worker uses its own
-    ``kernels`` (see :func:`_build_kernels`) and every worker shares the
-    same read-only ``ray_arrays``.
+    """Process one already-loaded group of frames: correction, per-tile-
+    per-grid kernel accumulate, merge, and route.
+
+    This is the compute-side half of the mapping pipeline (design doc
+    Sec7): image loading (I/O) happens separately, in the prefetch
+    pipeline, so ``image_payloads`` here are already loaded -- this
+    function does no I/O of its own and is safe to call concurrently for
+    different groups from multiple compute workers, provided each worker
+    uses its own ``kernels`` (see :func:`_build_kernels`) and every
+    worker shares the same read-only ``ray_arrays``.
+
+    ``frame_indices`` must be contiguous and ascending, and must lie
+    inside one planned checkpoint range for every grid: the group's
+    frames merge inside the kernel and cannot be told apart afterwards,
+    so they must all belong to the same checkpoint. The caller forms
+    groups under those rules; :meth:`_CheckpointRouter.route` re-checks
+    the checkpoint one.
+
+    A single-frame group is the degenerate case of the same call, not a
+    separate path -- but note it is *not* bit-for-bit with a per-image
+    ``accumulate()``: a brick partitions the detector into rectangles
+    where a block partitions it into runs of the flattened image, so
+    pixels group into accumulators differently and their sums associate
+    differently. Same voxels, same contributor counts, totals equal to
+    rounding, and about 3.4% fewer records from the better block shape.
 
     The correction pipeline's whole-frame ``correct_frame`` step (pixel
     repair, static factors) runs here, in the compute worker, not in the
@@ -1474,41 +1527,78 @@ def _map_one_frame(
     ``image_workers``/``kernel_threads`` thread budget, not hidden inside
     I/O-rate bookkeeping the reader pool's blocked-fraction signal
     depends on staying clean.
+
+    :param image_payloads:
+        Loaded payloads, one per frame in the group, in frame order.
+    :param frame_indices:
+        The group's frame indices: contiguous, ascending, all inside one
+        checkpoint range per grid.
+    :param angles_start:
+        ``(frames, 4)`` exposure-start diffractometer angles, radians.
+    :param angles_end:
+        ``(frames, 4)`` exposure-end angles, radians. Equal to
+        ``angles_start`` for a stationary exposure; the kernel decides
+        per frame, so a group may mix the two.
     """
-    image = np.asarray(image_payload.img)
+    frame_indices = [int(index) for index in frame_indices]
+    images = [np.asarray(payload.img) for payload in image_payloads]
     frame_correction = getattr(correction_pipeline, "correct_frame", None)
-    corrected_frame = (
-        frame_correction(image_payload, image, frame_index)
+    # The whole group's corrected frames stay resident until every tile
+    # has been mapped: a tile is a row band of the detector, so each
+    # frame is revisited once per band. This is the group buffer, and it
+    # is what _frame_parallelism sizes a compute worker against.
+    corrected_frames = (
+        [
+            frame_correction(payload, image, frame_index)
+            for payload, image, frame_index in zip(
+                image_payloads, images, frame_indices
+            )
+        ]
         if callable(frame_correction)
         else None
     )
     # Collect every tile's contribution per grid, then route exactly once
-    # per (frame, grid) -- the router's remaining-frame countdown
-    # (Sec9/Sec10) counts frames, not tile-level route() calls, so a
-    # multi-tile frame must be merged down to one routed batch per grid
-    # before reaching it.
+    # per (group, grid) -- the router's remaining-frame countdown
+    # (Sec9/Sec10) counts frames, so a multi-tile group must be merged
+    # down to one routed batch per grid before reaching it.
     tile_batches: dict[str, list[Mapping[str, np.ndarray]]] = {
         grid.grid_name: [] for grid in spec.grids
     }
     for detector_tile in detector_tiles:
         row_start, row_stop, column_start, column_stop = detector_tile
         selection = np.s_[row_start:row_stop, column_start:column_stop]
-        if corrected_frame is None:
-            intensity, variance, mask = correction_pipeline(
-                image_payload,
-                image[selection],
-                frame_index,
-                detector_tile,
-            )
+        if corrected_frames is None:
+            tile_values = [
+                correction_pipeline(
+                    payload,
+                    image[selection],
+                    frame_index,
+                    detector_tile,
+                )
+                for payload, image, frame_index in zip(
+                    image_payloads, images, frame_indices
+                )
+            ]
         else:
-            intensity, variance, mask = (
-                values[selection] for values in corrected_frame
-            )
-        intensity = np.ascontiguousarray(intensity, dtype=np.float64)
-        variance = np.ascontiguousarray(variance, dtype=np.float64)
-        mask = np.ascontiguousarray(mask, dtype=bool)
+            tile_values = [
+                tuple(values[selection] for values in corrected)
+                for corrected in corrected_frames
+            ]
+        # One contiguous (frames, rows, columns) buffer per array, which
+        # is the copy the per-tile ascontiguousarray already made for a
+        # single frame -- the group spends it over F frames instead of 1.
+        intensity = np.ascontiguousarray(
+            np.stack([values[0] for values in tile_values]), dtype=np.float64
+        )
+        variance = np.ascontiguousarray(
+            np.stack([values[1] for values in tile_values]), dtype=np.float64
+        )
+        mask = np.ascontiguousarray(
+            np.stack([values[2] for values in tile_values]), dtype=bool
+        )
+        del tile_values
         for grid in spec.grids:
-            batch = kernels[grid.grid_name].accumulate(
+            batch = kernels[grid.grid_name].accumulate_group(
                 intensity,
                 variance,
                 mask,
@@ -1519,7 +1609,12 @@ def _map_one_frame(
             tile_batches[grid.grid_name].append(batch)
     for grid in spec.grids:
         merged = _reduce_batches(tile_batches[grid.grid_name])
-        router.route(grid.grid_name, frame_index, merged)
+        router.route(
+            grid.grid_name,
+            frame_indices[0],
+            merged,
+            frames=len(frame_indices),
+        )
 
 
 def _compression_kwargs(name):
diff --git a/orgui/datautils/xrayutils/test/test_reconstruction.py b/orgui/datautils/xrayutils/test/test_reconstruction.py
index ea949a7..045e662 100644
--- a/orgui/datautils/xrayutils/test/test_reconstruction.py
+++ b/orgui/datautils/xrayutils/test/test_reconstruction.py
@@ -24,7 +24,7 @@
     _sobol_tile_origins,
     _files_per_job,
     _kernel_threads_sweep,
-    _map_one_frame,
+    _map_frame_group,
     _merge_sorted_batches,
     _reduce_batches,
     _tile_ray_arrays,
@@ -1229,7 +1229,7 @@ def _router(
     )
 
 
-def test_map_one_frame_routes_batches_and_skips_resumed_checkpoints(tmp_path):
+def test_map_frame_group_routes_batches_and_skips_resumed_checkpoints(tmp_path):
     grid = _GridSpec(
         minimum=(-1.0, -1.0, -1.0),
         maximum=(1.0, 1.0, 1.0),
@@ -1263,16 +1263,16 @@ def correction(payload, raw, frame, tile):
 
     for frame_index in range(2):
         payload = scan.get_raw_img(frame_index)
-        _map_one_frame(
+        _map_frame_group(
             spec,
             kernels,
             ray_arrays,
             detector_tiles,
             correction,
-            payload,
-            frame_index,
-            bounds[frame_index, 0],
-            bounds[frame_index, 1],
+            [payload],
+            [frame_index],
+            bounds[frame_index : frame_index + 1, 0],
+            bounds[frame_index : frame_index + 1, 1],
             router,
         )
 
@@ -1286,7 +1286,7 @@ def correction(payload, raw, frame, tile):
     assert written_indices == {0}
 
 
-def test_map_one_frame_corrects_once_before_tiling(tmp_path):
+def test_map_frame_group_corrects_once_before_tiling(tmp_path):
     class Detector:
         detector = type("PyfaiDetector", (), {"shape": (1, 2)})()
 
@@ -1326,18 +1326,131 @@ def correct_frame(payload, raw, frame):
     kernels = _build_kernels(spec, _FakeUB())
     payload = h5_Image(np.array([[10.0, 20.0]]))
 
-    _map_one_frame(
+    _map_frame_group(
         spec,
         kernels,
         ray_arrays,
         tiles,
         correction,
-        payload,
-        0,
-        np.zeros(4, dtype=np.float64),
-        np.zeros(4, dtype=np.float64),
+        [payload],
+        [0],
+        np.zeros((1, 4), dtype=np.float64),
+        np.zeros((1, 4), dtype=np.float64),
         router,
     )
 
     assert calls == {"frame": 1, "tile": 0}
     assert len(router.written) == 1
+
+
+class _RecordingRouter:
+    """Collects routed batches instead of writing checkpoints."""
+
+    def __init__(self):
+        self.batches = []
+        self.frames = []
+
+    def route(self, grid_name, frame_index, batch, *, frames=1):
+        self.batches.append(
+            {name: np.asarray(values) for name, values in batch.items()}
+        )
+        self.frames.append((grid_name, int(frame_index), int(frames)))
+
+
+def _totals(batches):
+    """Per-voxel contributor counts and weight, summed over batches.
+
+    Independent of how the frames were split into calls, which is exactly
+    the property a frame group must not disturb.
+    """
+    contributors: dict[tuple[int, int], int] = {}
+    weight: dict[tuple[int, int], float] = {}
+    for batch in batches:
+        for chunk, voxel, count, mass in zip(
+            batch["chunk_id"],
+            batch["local_voxel_id"],
+            batch["contributors"],
+            batch["weight"],
+        ):
+            key = (int(chunk), int(voxel))
+            contributors[key] = contributors.get(key, 0) + int(count)
+            weight[key] = weight.get(key, 0.0) + float(mass)
+    return contributors, weight
+
+
+def test_map_frame_group_reaches_the_same_voxels_as_single_frame_calls():
+    """Grouping frames must not change the answer.
+
+    It is deliberately not bit-for-bit -- several frames merge in the
+    block map rather than in the tree accumulator, so sums associate
+    differently -- so what is pinned here is what must not move: which
+    voxels were reached, and how many samples reached each of them.
+    """
+    grid = _GridSpec(
+        minimum=(-4.0, -4.0, -4.0),
+        maximum=(4.0, 4.0, 4.0),
+        step=(0.25, 0.25, 0.25),
+        frame="lab",
+        chunk_shape=(4, 4, 4),
+    )
+    frames = 4
+    scan = _FakeScan([10.0, 20.0, 30.0, 40.0])
+    detector_tiles = ((0, 1, 0, 1),)
+    ray_arrays = _tile_ray_arrays(
+        _FakeDetector(), detector_tiles, {(0, 1, 0, 1): _constant_rays(1, 1)}
+    )
+    # A rotation, so the frames land at different places in the volume and
+    # a group has something to merge rather than nothing.
+    bounds = np.zeros((frames, 2, 4), dtype=np.float64)
+    bounds[:, :, 1] = np.deg2rad(0.1) * np.arange(frames)[:, None]
+
+    def correction(payload, raw, frame, tile):
+        return (
+            raw.astype(np.float64),
+            np.maximum(raw, 0.0).astype(np.float64),
+            np.zeros(raw.shape, dtype=bool),
+        )
+
+    results = {}
+    for frames_per_group in (1, 2, 4):
+        spec = _ReconstructionSpec(
+            grids=(grid,), max_depth=0, frames_per_group=frames_per_group
+        )
+        kernels = _build_kernels(spec, _FakeUB())
+        router = _RecordingRouter()
+        for origin in range(0, frames, frames_per_group):
+            group = list(range(origin, origin + frames_per_group))
+            _map_frame_group(
+                spec,
+                kernels,
+                ray_arrays,
+                detector_tiles,
+                correction,
+                [scan.get_raw_img(index) for index in group],
+                group,
+                np.ascontiguousarray(bounds[group, 0]),
+                np.ascontiguousarray(bounds[group, 1]),
+                router,
+            )
+        # One route() call per group, each declaring its own frame count,
+        # so the checkpoint countdown still sees every frame exactly once.
+        assert sum(entry[2] for entry in router.frames) == frames
+        assert len(router.frames) == frames // frames_per_group
+        results[frames_per_group] = (
+            _totals(router.batches),
+            sum(int(batch["chunk_id"].size) for batch in router.batches),
+        )
+
+    (reference_contributors, reference_weight), reference_rows = results[1]
+    assert reference_contributors, "the fixture must reach at least one voxel"
+    for frames_per_group in (2, 4):
+        (contributors, weight), rows = results[frames_per_group]
+        assert contributors == reference_contributors
+        assert set(weight) == set(reference_weight)
+        for key, value in weight.items():
+            assert value == pytest.approx(reference_weight[key], rel=1e-12)
+        # The point of grouping: the same samples leave the kernel as
+        # fewer records, because frames sharing a voxel now merge inside
+        # one call. Without this the test would still pass if grouping
+        # silently degraded to mapping each frame on its own.
+        assert rows < reference_rows
diff --git a/orgui/datautils/xrayutils/test/test_reconstruction_job.py b/orgui/datautils/xrayutils/test/test_reconstruction_job.py
index 6a024ab..bb9edf3 100644
--- a/orgui/datautils/xrayutils/test/test_reconstruction_job.py
+++ b/orgui/datautils/xrayutils/test/test_reconstruction_job.py
@@ -13,7 +13,11 @@
 from orgui.app.database import config_data_to_json
 from orgui.backend.scans import ScanReference, SimulationScan
 from orgui.datautils.xrayutils import CTRcalc, DetectorCalibration, HKLVlieg
-from orgui.datautils.xrayutils.reconstruction import _GridSpec, _ReconstructionSpec
+from orgui.datautils.xrayutils.reconstruction import (
+    _CheckpointRouter,
+    _GridSpec,
+    _ReconstructionSpec,
+)
 import orgui.reconstruction_job as reconstruction_job_module
 from orgui.reconstruction_job import (
     MAX_CONCURRENT_ACTIVE_CHECKPOINTS,
@@ -23,6 +27,8 @@
     _RESERVED_RECORDS_PER_PIXEL,
     ReconstructionGrid,
     ReconstructionJob,
+    _angles_advance_monotonically,
+    _frame_groups,
     _frame_parallelism,
     _node_checkpoint_plan,
     _node_excluded_frames,
@@ -820,3 +826,161 @@ def test_cluster_array_task_count_never_mutates_shared_job_json(tmp_path):
     )
 
     assert job_path.read_bytes() == frozen_job
+
+
+def test_frame_groups_cut_at_the_group_size_and_at_checkpoint_boundaries():
+    """A group is bounded by two things at once.
+
+    Its own size, and the planned checkpoint it belongs to: the frames of
+    a group merge inside the kernel and cannot be separated afterwards, so
+    a group spanning two checkpoints could not be routed to either.
+    """
+    router = _CheckpointRouter(
+        {"hkl": [(0, 5), (5, 12)]},
+        spec_digest="test",
+        checkpoint_dir="unused",
+        active_budget_bytes=1024**2,
+    )
+    groups = _frame_groups((0, 12), router, ["hkl"], 4)
+
+    assert groups == [[0, 1, 2, 3], [4], [5, 6, 7, 8], [9, 10, 11]]
+    assert [frame for group in groups for frame in group] == list(range(12))
+    assert all(len(group) <= 4 for group in groups)
+
+
+def test_frame_groups_respect_every_grid_not_only_the_first():
+    """Grids may be checkpointed on different boundaries, and a group has
+    to be routable to all of them."""
+    router = _CheckpointRouter(
+        {"hkl": [(0, 8)], "q_lab": [(0, 3), (3, 8)]},
+        spec_digest="test",
+        checkpoint_dir="unused",
+        active_budget_bytes=1024**2,
+    )
+    groups = _frame_groups((0, 8), router, ["hkl", "q_lab"], 8)
+
+    assert groups == [[0, 1, 2], [3, 4, 5, 6, 7]]
+
+
+def test_frame_groups_of_one_reproduce_the_per_frame_schedule():
+    router = _CheckpointRouter(
+        {"hkl": [(0, 4)]},
+        spec_digest="test",
+        checkpoint_dir="unused",
+        active_budget_bytes=1024**2,
+    )
+
+    assert _frame_groups((0, 4), router, ["hkl"], 1) == [[0], [1], [2], [3]]
+
+
+def test_angles_advance_monotonically_rejects_an_interlaced_order():
+    """Grouping assumes frames adjacent in index are adjacent in angle.
+
+    An interlaced scan (orgui/backend/interlacedScanLoader.py) breaks
+    that: every other frame first, then the ones between them.
+    """
+    bounds = np.zeros((8, 2, 4), dtype=np.float64)
+    sequential = np.deg2rad(0.1) * np.arange(8)
+    bounds[:, 0, 1] = sequential
+    bounds[:, 1, 1] = sequential
+
+    assert _angles_advance_monotonically(bounds, list(range(8)))
+
+    interlaced = np.concatenate([sequential[::2], sequential[1::2]])
+    bounds[:, 0, 1] = interlaced
+    bounds[:, 1, 1] = interlaced
+
+    assert not _angles_advance_monotonically(bounds, list(range(8)))
+
+
+def test_angles_advance_monotonically_ignores_axes_that_never_move():
+    """A scan rotating one motor is judged on that motor alone; the three
+    stationary angles carry floating-point noise, not direction."""
+    bounds = np.zeros((6, 2, 4), dtype=np.float64)
+    bounds[:, 0, 1] = np.deg2rad(0.1) * np.arange(6)
+    bounds[:, 1, 1] = bounds[:, 0, 1]
+    bounds[:, :, 3] = 1e-17 * np.array([1.0, -1.0, 1.0, -1.0, 1.0, -1.0])[:, None]
+
+    assert _angles_advance_monotonically(bounds, list(range(6)))
+
+
+def test_router_counts_a_group_as_all_of_its_frames(tmp_path):
+    """The checkpoint countdown and ``frames_covered`` count frames, not
+    route() calls -- a group that declared one frame would leave its
+    checkpoint permanently short and unresumable."""
+    router = _CheckpointRouter(
+        {"hkl": [(0, 6)]},
+        spec_digest="test",
+        checkpoint_dir=tmp_path / "checkpoints",
+        active_budget_bytes=1024**3,
+    )
+    batch = {
+        "chunk_id": np.array([0], dtype=np.uint32),
+        "local_voxel_id": np.array([0], dtype=np.uint32),
+        "weighted_intensity": np.array([1.0]),
+        "weighted_variance": np.array([1.0]),
+        "weight": np.array([1.0]),
+        "contributors": np.array([1], dtype=np.uint32),
+    }
+
+    router.route("hkl", 0, dict(batch), frames=3)
+    assert not router.written, "half the checkpoint's frames are still missing"
+
+    router.route("hkl", 3, dict(batch), frames=3)
+    assert len(router.written) == 1
+
+    with h5py.File(router.written[0], "r") as handle:
+        assert int(handle.attrs["frames_covered"]) == 6
+
+
+def test_router_rejects_a_group_straddling_a_checkpoint_boundary(tmp_path):
+    router = _CheckpointRouter(
+        {"hkl": [(0, 4), (4, 8)]},
+        spec_digest="test",
+        checkpoint_dir=tmp_path / "checkpoints",
+        active_budget_bytes=1024**3,
+    )
+    batch = {name: np.zeros(0) for name in ("weighted_intensity",)}
+    batch["chunk_id"] = np.zeros(0, dtype=np.uint32)
+
+    with pytest.raises(ValueError, match="straddles"):
+        router.route("hkl", 2, batch, frames=4)
+
+
+def test_frame_parallelism_charges_a_group_for_its_resident_frames():
+    """A group holds every one of its corrected frames at once, and its
+    native call covers frames x tile samples -- but it emits one merged
+    record batch, not one per frame, so only the correction term
+    multiplies."""
+    grid = _GridSpec(
+        minimum=(0.0, 0.0, 0.0),
+        maximum=(1.0, 1.0, 1.0),
+        step=(0.1, 0.1, 0.1),
+        frame="lab",
+        chunk_shape=(8, 8, 8),
+    )
+    tiles = [(0, 512, 0, 512)]
+    budget = 32 * 1024**3
+
+    def worker_bytes(frames_per_group):
+        spec = _ReconstructionSpec(
+            grids=(grid,),
+            max_depth=0,
+            threads=8,
+            frames_per_group=frames_per_group,
+        )
+        _workers, _threads, per_worker, _accumulation = _frame_parallelism(
+            spec, tiles, budget, threads_per_image=1
+        )
+        return per_worker
+
+    pixels = 512 * 512
+    native_per_pixel = 128 + 2 * 40
+    record_term = pixels * _FRAME_RECORD_BYTES_PER_PIXEL
+    for frames_per_group in (1, 2, 8):
+        expected = (
+            pixels * frames_per_group * native_per_pixel
+            + pixels * frames_per_group * _PYTHON_CORRECTION_BYTES_PER_PIXEL
+            + record_term
+        )
+        assert worker_bytes(frames_per_group) >= expected
diff --git a/orgui/reconstruction_job.py b/orgui/reconstruction_job.py
index aba591c..6e844ca 100644
--- a/orgui/reconstruction_job.py
+++ b/orgui/reconstruction_job.py
@@ -2,7 +2,7 @@
 
 from __future__ import annotations
 
-from dataclasses import asdict, dataclass, field
+from dataclasses import asdict, dataclass, field, replace
 from hashlib import sha256
 import json
 import math
@@ -32,7 +32,7 @@
     _files_per_job,
     _finalize_reconstruction,
     _kernel_threads_sweep,
-    _map_one_frame,
+    _map_frame_group,
     _tile_ray_arrays,
     _validate_mapping_setup,
 )
@@ -1517,12 +1517,30 @@ def _frame_parallelism(
         if spec.max_depth > 0
         else 128 + 2 * _CHECKPOINT_BYTES_PER_ROW
     )
-    native_memory = largest_tile_pixels * native_bytes_per_pixel
+    # A frame group is one native call over (frames, tile rows, tile
+    # columns) samples, and the kernel's own precheck is written against
+    # that sample count -- so a worker's native working set scales with
+    # the group size exactly as the tile size, and this side must scale
+    # with it too or the two would disagree in the one direction that
+    # matters (this side permissive, the kernel throwing).
+    frames_per_group = max(1, int(getattr(spec, "frames_per_group", 1)))
+    native_memory = (
+        largest_tile_pixels * frames_per_group * native_bytes_per_pixel
+    )
     # The native estimate alone misses real, per-in-flight-frame Python
     # memory: the raw decoded image plus every corrected array
     # (_PYTHON_CORRECTION_BYTES_PER_PIXEL), full-detector-sized.
+    #
+    # Only the correction half multiplies by the group size. A group
+    # holds all F corrected frames at once -- each tile is a row band, so
+    # every frame is revisited once per band and none can be released
+    # early -- but it produces one merged record batch for the whole
+    # group, not F of them, and that batch is measured at 0.575x the sum
+    # of the F it replaces. Charging the record term per frame would
+    # reserve nearly twice what a group actually holds.
     python_memory = detector_pixels * (
-        _PYTHON_CORRECTION_BYTES_PER_PIXEL + _FRAME_RECORD_BYTES_PER_PIXEL
+        frames_per_group * _PYTHON_CORRECTION_BYTES_PER_PIXEL
+        + _FRAME_RECORD_BYTES_PER_PIXEL
     )
     image_memory = native_memory + python_memory
     worker_memory = max(
@@ -1541,8 +1559,13 @@ def _frame_parallelism(
     # off the top before dividing the remaining budget among workers, the
     # same way the ray-corner cache is reserved by callers before this
     # function ever sees the budget.
+    # The queue holds groups, not frames, so a slack slot now costs a
+    # whole group's raw images.
     prefetch_reserve_bytes = (
-        _PREFETCH_QUEUE_SLACK * detector_pixels * _RAW_IMAGE_BYTES_PER_PIXEL
+        _PREFETCH_QUEUE_SLACK
+        * frames_per_group
+        * detector_pixels
+        * _RAW_IMAGE_BYTES_PER_PIXEL
     )
     usable_memory_bytes = max(
         minimum_accumulation, memory_bytes - prefetch_reserve_bytes
@@ -2179,6 +2202,80 @@ def _kernel_threads_candidates(total_threads, include=()):
     return sorted(candidates)
 
 
+def _angles_advance_monotonically(bounds, frame_indices):
+    """Whether frames adjacent in index are also adjacent in angle.
+
+    Frame grouping merges several images inside one native call on the
+    premise that consecutive frames sit next to each other in reciprocal
+    space. An interlaced scan (``orgui/backend/interlacedScanLoader.py``)
+    breaks that premise: frames adjacent in file order can be half a
+    rotation apart, and grouping them would collapse nothing while still
+    costing the group buffer. Rather than assume the scan is sequential,
+    check the angles the job actually resolved.
+
+    Every axis that moves at all across the sampled frames must move in
+    one direction only. An axis that never moves is ignored, so a scan
+    that rotates a single motor is judged on that motor alone.
+
+    :param bounds:
+        ``(len(scan), 2, 4)`` exposure angle bounds in radians.
+    :param frame_indices:
+        Frame indices in the order they would be grouped.
+    :returns:
+        ``True`` when grouping is sound for these frames.
+    :rtype: bool
+    """
+    if len(frame_indices) < 3:
+        return True
+    starts = np.asarray(bounds, dtype=np.float64)[list(frame_indices), 0]
+    steps = np.diff(starts, axis=0)
+    for axis in range(starts.shape[1]):
+        column = steps[:, axis]
+        span = float(np.ptp(starts[:, axis]))
+        if span <= 1e-9:
+            continue
+        if not (np.all(column >= -1e-12) or np.all(column <= 1e-12)):
+            return False
+    return True
+
+
+def _frame_groups(frame_range, router, grid_names, frames_per_group):
+    """Split one scheduling range into the groups mapped in one call each.
+
+    A group must be contiguous in frame index and must lie inside one
+    planned checkpoint range for *every* grid: its frames merge inside the
+    kernel and can no longer be told apart, so they have to share a
+    checkpoint. Scheduling ranges are already contiguous and free of
+    excluded frames (:func:`_included_ranges` splits on both), so the only
+    cut this has to make beyond the group size is at a checkpoint
+    boundary -- which need not align with the scheduling ranges, since the
+    two are sized independently.
+
+    :returns:
+        List of frame-index lists, each at most ``frames_per_group`` long,
+        together covering the range in order.
+    :rtype: list[list[int]]
+    """
+    start, stop = frame_range
+    size = max(1, int(frames_per_group))
+    groups = []
+    current: list[int] = []
+    previous_keys = None
+    for frame in range(start, stop):
+        keys = tuple(
+            router.checkpoint_index_for_frame(grid_name, frame)
+            for grid_name in grid_names
+        )
+        if current and (keys != previous_keys or len(current) >= size):
+            groups.append(current)
+            current = []
+        previous_keys = keys
+        current.append(frame)
+    if current:
+        groups.append(current)
+    return groups
+
+
 def _map_pending_ranges(
     spec,
     scan,
@@ -2201,9 +2298,17 @@ def _map_pending_ranges(
     A producer/consumer pipeline (design doc Sec7): a prefetch pool of
     reader threads loads images and feeds a bounded-backpressure queue; a
     pool of compute workers drains it, correcting and accumulating each
-    frame via :func:`_map_one_frame`. The reader pool adapts continuously
-    via a blocked-fraction signal (grow eagerly when compute is starved
-    for images, shrink cautiously when it isn't).
+    group of frames via :func:`_map_frame_group`. The reader pool adapts
+    continuously via a blocked-fraction signal (grow eagerly when compute
+    is starved for images, shrink cautiously when it isn't).
+
+    The unit of work is ``spec.frames_per_group`` consecutive frames
+    mapped in one native call -- one frame per group by default, which is
+    the degenerate case of the same machinery rather than a separate
+    path. Groups are cut at scheduling-range and checkpoint boundaries
+    (:func:`_frame_groups`), and dropped to one frame per group entirely
+    when the scan's angles do not advance monotonically
+    (:func:`_angles_advance_monotonically`).
 
     ``threads_per_image=None`` (Sec7 Phase 4b, "automatic") starts from an
     I/O-optimistic seed (``kernel_threads=1``, all budget as
@@ -2233,6 +2338,21 @@ def _map_pending_ranges(
         scan, config.detector, tiles, bounds, correction_pipeline
     )
 
+    frame_indices = [
+        frame_index
+        for start, stop in pending_ranges
+        for frame_index in range(start, stop)
+    ]
+    # Grouping rests on frames adjacent in index being adjacent in angle.
+    # Where they are not -- an interlaced scan -- the honest answer is one
+    # frame per call, decided here rather than assumed anywhere below.
+    frames_per_group = max(1, int(getattr(spec, "frames_per_group", 1)))
+    if frames_per_group > 1 and not _angles_advance_monotonically(
+        bounds, frame_indices
+    ):
+        frames_per_group = 1
+    spec = replace(spec, frames_per_group=frames_per_group)
+
     ray_cache_bytes = sum(
         (tile[1] - tile[0] + 1)
         * (tile[3] - tile[2] + 1)
@@ -2297,10 +2417,13 @@ def layout_for(threads_per_image_seed):
     # bounds the aggregate worker working set above.
     kernel_memory_budget = effective_memory
 
-    frame_indices = [
-        frame_index
-        for start, stop in pending_ranges
-        for frame_index in range(start, stop)
+    grid_names = [grid.grid_name for grid in spec.grids]
+    frame_groups = [
+        group
+        for frame_range in pending_ranges
+        for group in _frame_groups(
+            frame_range, router, grid_names, frames_per_group
+        )
     ]
 
     if automatic:
@@ -2340,8 +2463,14 @@ def layout_for(threads_per_image_seed):
     mapped_images = completed_images
 
     work_lock = threading.Lock()
-    work_iterator = iter(frame_indices)
-    remaining = [len(frame_indices)]
+    # The unit of work is a group of frames mapped in one native call --
+    # one frame per group when frames_per_group is 1. The gate therefore
+    # bounds in-flight *groups* rather than frames: a reader loads a whole
+    # group before queueing it, so a per-frame gate would let several
+    # readers each hold a partial group and none of them make progress.
+    # _frame_parallelism reserves the queue slack in whole groups to match.
+    work_iterator = iter(frame_groups)
+    remaining = [len(frame_groups)]
     remaining_lock = threading.Lock()
     readers_done = threading.Event()
     # Successfully-read frames since the last rebalance check -- the
@@ -2362,11 +2491,19 @@ def record_exception(exc):
     def should_stop():
         return bool(first_exception)
 
-    def mark_frame_delivered(*, succeeded=True):
+    def mark_group_delivered(group_frames, *, succeeded=True):
+        """One group off the reader pool's plate.
+
+        ``remaining`` counts groups (the work unit), but
+        ``delivered_in_window`` stays in frames: it feeds the rebalance's
+        Little's-law rule against a sweep that reports a per-frame time,
+        and mixing the two units there would silently misprice every
+        candidate thread count.
+        """
         with remaining_lock:
             remaining[0] -= 1
             if succeeded:
-                delivered_in_window[0] += 1
+                delivered_in_window[0] += group_frames
             if remaining[0] <= 0:
                 readers_done.set()
 
@@ -2410,24 +2547,26 @@ def abandon_should_stop():
         try:
             while not local_should_stop():
                 with work_lock:
-                    frame_index = next(work_iterator, None)
-                if frame_index is None:
+                    group = next(work_iterator, None)
+                if group is None:
                     return
                 if not gate.acquire(
                     poll_timeout=_POLL_TIMEOUT_SECONDS,
                     should_stop=abandon_should_stop,
                 ):
-                    mark_frame_delivered(succeeded=False)
+                    mark_group_delivered(len(group), succeeded=False)
                     return
                 try:
-                    image_payload = scan.get_raw_img(frame_index)
+                    image_payloads = [
+                        scan.get_raw_img(frame_index) for frame_index in group
+                    ]
                 except BaseException as exc:  # noqa: BLE001
                     gate.release()
                     record_exception(exc)
-                    mark_frame_delivered(succeeded=False)
+                    mark_group_delivered(len(group), succeeded=False)
                     return
-                ready_queue.put((frame_index, image_payload))
-                mark_frame_delivered()
+                ready_queue.put((group, image_payloads))
+                mark_group_delivered(len(group))
         except BaseException as exc:  # noqa: BLE001 -- must reach the coordinator
             record_exception(exc)
 
@@ -2464,7 +2603,7 @@ def compute_loop(retire, pool_cancellation):
                     if retire.is_set() or pool_cancellation.is_set():
                         return
                     try:
-                        frame_index, image_payload = ready_queue.get(
+                        group, image_payloads = ready_queue.get(
                             timeout=_POLL_TIMEOUT_SECONDS
                         )
                     except Empty:
@@ -2478,22 +2617,22 @@ def compute_loop(retire, pool_cancellation):
                     counts[1] += 1
                     gate.release()
                     try:
-                        _map_one_frame(
+                        _map_frame_group(
                             spec,
                             kernels,
                             ray_arrays,
                             detector_tiles,
                             correction_pipeline,
-                            image_payload,
-                            frame_index,
-                            np.ascontiguousarray(bounds[frame_index, 0]),
-                            np.ascontiguousarray(bounds[frame_index, 1]),
+                            image_payloads,
+                            group,
+                            np.ascontiguousarray(bounds[group, 0]),
+                            np.ascontiguousarray(bounds[group, 1]),
                             router,
                         )
                     except BaseException as exc:  # noqa: BLE001
                         record_exception(exc)
                         return
-                    progress_events.put(frame_index)
+                    progress_events.put(len(group))
             except BaseException as exc:  # noqa: BLE001
                 record_exception(exc)
         finally:
@@ -2516,19 +2655,24 @@ def compute_loop(retire, pool_cancellation):
             deadline = time.monotonic() + _COORDINATOR_TICK_SECONDS
             while time.monotonic() < deadline:
                 try:
-                    progress_events.get(
+                    mapped_in_group = progress_events.get(
                         timeout=max(0.0, deadline - time.monotonic())
                     )
                 except Empty:
                     break
-                mapped_images += 1
+                mapped_images += mapped_in_group
                 if progress is not None:
+                    grouping = (
+                        f" x {frames_per_group} frames"
+                        if frames_per_group > 1
+                        else ""
+                    )
                     progress(
                         mapped_images,
                         total_images + 1,
                         (
                             f"Mapping images {mapped_images}/{total_images} "
-                            f"({compute_pool.size} image workers, "
+                            f"({compute_pool.size} image workers{grouping}, "
                             f"{current_kernel_threads[0]} native threads/image, "
                             f"{reader_pool.size} prefetch readers)"
                         ),
@@ -2632,7 +2776,7 @@ def compute_loop(retire, pool_cancellation):
                             )
                             # Blocks until every straggler on the retired
                             # generation finishes its in-flight
-                            # _map_one_frame call -- a deliberate, rare
+                            # _map_frame_group call -- a deliberate, rare
                             # stall (this whole block runs at most once per
                             # rebalance interval), never abandons
                             # in-flight work.

From ad77f79268974c5cdc0a8904f9898aa4fd8088de Mon Sep 17 00:00:00 2001
From: "T. Fuchs" <tfuchs@cornell.edu>
Date: Mon, 10 Aug 2026 16:22:40 -0400
Subject: [PATCH 10/33] test: add the end-to-end mapping pipeline benchmark

Times the whole mapping phase rather than one kernel call, which is what
grouping has to actually improve. It reports records entering the
checkpoint layer, not rows on disk, where the saving never shows.
---
 .../benchmark_reconstruction_pipeline.py      | 377 ++++++++++++++++++
 1 file changed, 377 insertions(+)
 create mode 100644 benchmarks/benchmark_reconstruction_pipeline.py

diff --git a/benchmarks/benchmark_reconstruction_pipeline.py b/benchmarks/benchmark_reconstruction_pipeline.py
new file mode 100644
index 0000000..eeda093
--- /dev/null
+++ b/benchmarks/benchmark_reconstruction_pipeline.py
@@ -0,0 +1,377 @@
+"""End-to-end reciprocal-space mapping throughput on a prepared job.
+
+``benchmark_reconstruction_group.py`` times one kernel call. This times the
+whole mapping phase around it: the real ``_map_pending_ranges`` pipeline --
+reader pool, whole-frame correction, kernel, checkpoint router, checkpoint
+writes -- over a bounded window of frames. That is the thing frame grouping
+has to actually improve, and it is the thing a faster kernel can fail to
+improve, because the GIL-held Python work per frame does not get cheaper
+when the native call does.
+
+Checkpoint boundaries are synthesised over the measured window rather than
+taken from the job's own plan: a partial run against the real plan would
+leave every checkpoint short of its frame count, so nothing would flush and
+there would be no output to compare. Each synthesised checkpoint is a whole
+number of the job's own scheduling ranges, so it stays contiguous and
+exclusion-free exactly as a planned one is.
+
+The written checkpoint parts are read back and summarised. Frame grouping is
+deliberately *not* bit-for-bit -- contributions from several frames merge in
+the block map instead of in the tree accumulator -- so the summary reports
+what must hold instead: the same voxel keys, the same contributor counts,
+and totals equal to rounding.
+
+**Run this with ``PYTHONPATH`` pointing at the checkout.** ``python
+benchmarks/<script>.py`` puts ``benchmarks/`` on ``sys.path``, not the
+repository root, so a pip-installed ``orgui`` shadows the working tree and
+the run silently measures a binary you did not just build. The resolved
+package path is reported for that reason.
+"""
+
+from __future__ import annotations
+
+import argparse
+from dataclasses import replace
+import json
+from pathlib import Path
+import shutil
+import tempfile
+from time import perf_counter
+
+import h5py
+import numpy as np
+
+from orgui.datautils.xrayutils.reconstruction import (
+    _CheckpointRouter,
+    _native_module,
+    _PARTIAL_COLUMNS,
+)
+from orgui.reconstruction_job import (
+    _base_provenance,
+    _correction_pipeline,
+    _execution_layout,
+    _frame_parallelism,
+    _load_assets,
+    _map_pending_ranges,
+    read_job,
+    split_memory_budget,
+)
+
+
+def _arguments():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("job", type=Path)
+    parser.add_argument("--start", type=int, default=1800)
+    parser.add_argument("--count", type=int, default=160)
+    parser.add_argument(
+        "--ranges-per-checkpoint",
+        type=int,
+        default=2,
+        help="Scheduling ranges per synthesised checkpoint boundary.",
+    )
+    parser.add_argument(
+        "--group",
+        type=int,
+        default=1,
+        help="Frames per kernel call (spec.frames_per_group).",
+    )
+    parser.add_argument(
+        "--threads-per-image",
+        type=int,
+        default=None,
+        help="Fixed kernel_threads. Omitted means the job's own setting.",
+    )
+    parser.add_argument(
+        "--memory-gib",
+        type=float,
+        default=None,
+        help="Override the job's own memory budget, for both arms alike.",
+    )
+    parser.add_argument(
+        "--tile-rows",
+        type=int,
+        default=None,
+        help=(
+            "Override the detector band height. Smaller bands cut a "
+            "worker's native working set, so they buy image_workers back "
+            "at the same total budget rather than by raising it."
+        ),
+    )
+    parser.add_argument("--scratch", type=Path)
+    parser.add_argument("--output", type=Path)
+    return parser.parse_args()
+
+
+def _window_ranges(ranges, start, stop, ranges_per_checkpoint):
+    """Scheduling ranges inside ``[start, stop)``, and the checkpoint
+    boundaries covering exactly those frames.
+
+    A boundary may only merge scheduling ranges that are adjacent: a gap
+    between them is an excluded frame, and a checkpoint spanning one would
+    wait forever for a frame that is never routed.
+    """
+    kept = [
+        frame_range
+        for frame_range in ranges
+        if start <= frame_range[0] and frame_range[1] <= stop
+    ]
+    if not kept:
+        raise SystemExit(
+            f"No whole scheduling range lies inside frames [{start}, {stop})"
+        )
+    boundaries = []
+    run = [kept[0]]
+    for frame_range in kept[1:]:
+        adjacent = frame_range[0] == run[-1][1]
+        if adjacent and len(run) < max(1, ranges_per_checkpoint):
+            run.append(frame_range)
+            continue
+        boundaries.append((run[0][0], run[-1][1]))
+        run = [frame_range]
+    boundaries.append((run[0][0], run[-1][1]))
+    return kept, boundaries
+
+
+_SUMMARY_CHUNK_ROWS = 1 << 22
+
+
+def _summarize_checkpoints(checkpoint_dir, grid_names):
+    """Contributor counts, totals and a key fingerprint over every part.
+
+    These are the quantities that must agree across group sizes, in place
+    of a digest: grouping changes floating-point association, so sums move
+    in the last bits, while which voxels were reached and how many samples
+    reached each of them must not move at all.
+
+    ``voxel_fingerprint`` is ``sum over rows of h(key) * contributors``,
+    accumulated in ``uint64`` and allowed to wrap. Because it is a sum, it
+    does not care what order rows arrive in or how they were split across
+    parts -- it equals the same sum taken over distinct voxels with their
+    total contributor counts, which is exactly the invariant being
+    claimed. Two runs reaching different voxels, or apportioning samples
+    differently between them, disagree here even when every total below
+    matches.
+
+    Streamed in row chunks. A whole window's parts concatenated in memory
+    is tens of gigabytes on a real job -- and would be paid on top of the
+    mapping pipeline's own peak, which is already sized to the job's full
+    memory budget.
+    """
+    summary = {}
+    for grid_name in grid_names:
+        parts = sorted((Path(checkpoint_dir) / grid_name).glob("ckpt*.h5"))
+        frames_covered = 0
+        rows = 0
+        contributors = 0
+        totals = dict.fromkeys(
+            ("weight", "weighted_intensity", "weighted_variance"), 0.0
+        )
+        fingerprint = np.uint64(0)
+        for part in parts:
+            with h5py.File(part, "r") as handle:
+                frames_covered += int(handle.attrs["frames_covered"])
+                part_rows = int(handle["chunk_id"].shape[0])
+                rows += part_rows
+                for begin in range(0, part_rows, _SUMMARY_CHUNK_ROWS):
+                    end = min(begin + _SUMMARY_CHUNK_ROWS, part_rows)
+                    block = {
+                        name: handle[name][begin:end]
+                        for name in _PARTIAL_COLUMNS
+                    }
+                    contributors += int(block["contributors"].sum())
+                    for name in totals:
+                        totals[name] += float(block[name].sum())
+                    keys = (
+                        block["chunk_id"].astype(np.uint64)
+                        * np.uint64(1000003)
+                        + block["local_voxel_id"].astype(np.uint64)
+                    )
+                    with np.errstate(over="ignore"):
+                        fingerprint = np.uint64(
+                            fingerprint
+                            + (
+                                keys * block["contributors"].astype(np.uint64)
+                            ).sum()
+                        )
+        summary[grid_name] = {
+            "parts": len(parts),
+            "frames_covered": frames_covered,
+            "rows": rows,
+            "contributors": contributors,
+            "voxel_fingerprint": int(fingerprint),
+            **totals,
+        }
+    return summary
+
+
+def main():
+    """Time the real mapping pipeline over a bounded window of frames."""
+    arguments = _arguments()
+    job = read_job(arguments.job)
+    config = job.config_data
+    scan = job.scan
+    spec = job.internal_spec()
+    if arguments.group != 1:
+        if not hasattr(spec, "frames_per_group"):
+            raise SystemExit(
+                "This checkout has no spec.frames_per_group; --group only "
+                "works once frame grouping is wired in"
+            )
+        spec = replace(spec, frames_per_group=arguments.group)
+    bounds = scan.exposure_angle_bounds(config, fallback=job.angle_fallback)
+    ranges, tiles = _execution_layout(job, scan, config)
+    if arguments.tile_rows:
+        detector_rows, detector_columns = config.detector.detector.shape
+        tiles = [
+            (row, min(row + arguments.tile_rows, detector_rows), 0, detector_columns)
+            for row in range(0, detector_rows, arguments.tile_rows)
+        ]
+    kept, boundaries = _window_ranges(
+        ranges,
+        arguments.start,
+        arguments.start + arguments.count,
+        arguments.ranges_per_checkpoint,
+    )
+    grid_names = [grid.grid_name for grid in spec.grids]
+    plan = {grid_name: list(boundaries) for grid_name in grid_names}
+
+    effective_memory = (
+        int(arguments.memory_gib * 1024**3)
+        if arguments.memory_gib
+        else job.memory_override_bytes or job.runtime_memory_bytes
+    )
+    active_budget_bytes, effective_memory = split_memory_budget(
+        effective_memory, max(1, len(spec.grids))
+    )
+
+    scratch = Path(
+        arguments.scratch
+        or tempfile.mkdtemp(prefix="orgui-rsmap-bench-")
+    )
+    checkpoint_dir = scratch / "checkpoints"
+    if checkpoint_dir.exists():
+        shutil.rmtree(checkpoint_dir)
+    router = _CheckpointRouter(
+        plan,
+        spec_digest=job.digest,
+        checkpoint_dir=checkpoint_dir,
+        active_budget_bytes=active_budget_bytes,
+    )
+    # Records *entering* the checkpoint layer, which is what frame
+    # grouping is meant to reduce and what every stage downstream of the
+    # kernel scales with. Rows on disk are the wrong measure: the
+    # checkpoint accumulator already merges across frames, so it collapses
+    # the same redundancy later and more expensively, and the saving never
+    # shows up there.
+    routed = {"records": 0, "calls": 0}
+    original_route = router.route
+
+    def counting_route(grid_name, frame_index, batch, **kwargs):
+        routed["records"] += int(batch["chunk_id"].size)
+        routed["calls"] += 1
+        return original_route(grid_name, frame_index, batch, **kwargs)
+
+    router.route = counting_route
+
+    correct = _correction_pipeline(
+        config, scan, _load_assets(job), _base_provenance(job, config)
+    )
+    total_images = sum(stop - start for start, stop in kept)
+    seed_workers, seed_threads, worker_bytes, _accumulation = (
+        _frame_parallelism(
+            spec,
+            tiles,
+            effective_memory,
+            threads_per_image=1,
+            accumulation_budget_bytes=job.accumulation_budget_bytes,
+        )
+    )
+
+    header = {
+        "job": str(arguments.job),
+        "frames": [kept[0][0], kept[-1][1]],
+        "total_images": total_images,
+        "scheduling_ranges": len(kept),
+        "checkpoints": len(boundaries),
+        "tiles": len(tiles),
+        "largest_tile_pixels": max(
+            (row_stop - row_start) * (column_stop - column_start)
+            for row_start, row_stop, column_start, column_stop in tiles
+        ),
+        "frames_per_group": getattr(spec, "frames_per_group", 1),
+        "max_depth": spec.max_depth,
+        "work_block_pixels": spec.work_block_pixels,
+        "threads": spec.threads,
+        "angle_fallback": job.angle_fallback,
+        "effective_memory_bytes": int(effective_memory),
+        "active_budget_bytes": int(active_budget_bytes),
+        # What the memory budget affords at the I/O-optimistic seed, which
+        # is where automatic mode starts and what a larger group shrinks.
+        "seed_image_workers": int(seed_workers),
+        "seed_kernel_threads": int(seed_threads),
+        "worker_memory_mb": round(worker_bytes / 1e6, 1),
+        "native": _native_module().__file__,
+        "scratch": str(scratch),
+    }
+    print(json.dumps(header, sort_keys=True), flush=True)
+
+    started = perf_counter()
+    _map_pending_ranges(
+        spec,
+        scan,
+        config,
+        bounds,
+        tiles,
+        kept,
+        router,
+        correction_pipeline=correct,
+        effective_memory=effective_memory,
+        threads_per_image=(
+            arguments.threads_per_image
+            if arguments.threads_per_image is not None
+            else job.threads_per_image
+        ),
+        accumulation_budget_bytes=job.accumulation_budget_bytes,
+        total_images=total_images,
+        completed_images=0,
+        progress=None,
+    )
+    elapsed = perf_counter() - started
+
+    result = {
+        "seconds": elapsed,
+        "frames_per_second": total_images / elapsed,
+        "seconds_per_frame": elapsed / total_images,
+        "routed_records": routed["records"],
+        "routed_records_per_frame": routed["records"] / total_images,
+        "route_calls": routed["calls"],
+        "checkpoints_written": len(router.written),
+        "grids": _summarize_checkpoints(checkpoint_dir, grid_names),
+    }
+    print(json.dumps(result, sort_keys=True), flush=True)
+
+    print()
+    print(f"{total_images} frames in {elapsed:.1f} s "
+          f"({total_images / elapsed:.3f} frames/s, "
+          f"{elapsed / total_images * 1000:.1f} ms/frame)")
+    print(f"  routed {routed['records']:,} records in {routed['calls']} calls "
+          f"({routed['records'] / total_images:,.0f} per frame); "
+          f"seed layout {seed_workers} workers x {seed_threads} threads")
+    for grid_name, values in result["grids"].items():
+        print(
+            f"  {grid_name}: {values['rows']:,} rows, "
+            f"{values['contributors']:,} contributors, "
+            f"weight {values['weight']:.6e}"
+        )
+
+    if arguments.output:
+        arguments.output.write_text(
+            json.dumps({"header": header, "result": result}, indent=2),
+            encoding="utf-8",
+        )
+    if arguments.scratch is None:
+        shutil.rmtree(scratch, ignore_errors=True)
+
+
+if __name__ == "__main__":
+    main()

From 26d36b0c740138101dcf3f3e3db2d5754e895a8f Mon Sep 17 00:00:00 2001
From: "T. Fuchs" <tfuchs@cornell.edu>
Date: Mon, 10 Aug 2026 16:22:52 -0400
Subject: [PATCH 11/33] docs: record the frame-group pipeline result and
 reorder what remains

The record prediction held at 0.598x; the throughput one did not, since
image_workers collapses when a worker is charged for its resident
frames. The scheduler must come before per-job group sizing.
---
 CHANGELOG.md                                  |  15 ++
 ..._space_mapping_locality_and_subdivision.md | 158 ++++++++++++++++--
 2 files changed, 157 insertions(+), 16 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 1a07610..b8c92a0 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -64,6 +64,21 @@ Scientific correctness and performance fixes:
   corrected atomic-coordinate stacking when splitting unit cells into layers.
 - Added a C++ electron-density backend and bounded caches for atomic form
   factors, anomalous scattering factors, and accelerated form-factor lookup.
+- Reciprocal-space mapping now maps detector images through a work block
+  shaped as a brick in (row, column, frame) rather than a run of one
+  flattened image, and can map several consecutive images in a single native
+  call. On a rotation scan two adjacent images land as close together in
+  reciprocal space as two adjacent pixels do, so a brick merges
+  contributions the per-image block could not see: on a 3651-frame Pilatus
+  6M scan, mapping eight images together emits 0.60x the records for the
+  same samples. **This changes reconstructed values in the last bits, for
+  every job, including jobs that map one image at a time.** Contributions
+  now merge inside the kernel rather than in the checkpoint accumulator, so
+  the same sums associate differently — with fewer intermediate roundings,
+  so slightly better conditioned. Which voxels are reached, and how many
+  detector samples reach each of them, are unchanged. Existing checkpoint
+  files remain resumable, but a job resumed across this change will contain
+  parts from both association orders.
 
 A ***critical bug*** was fixed that affects bulk CTR calculations:
 
diff --git a/doc/design/reciprocal_space_mapping_locality_and_subdivision.md b/doc/design/reciprocal_space_mapping_locality_and_subdivision.md
index c35c070..5d71897 100644
--- a/doc/design/reciprocal_space_mapping_locality_and_subdivision.md
+++ b/doc/design/reciprocal_space_mapping_locality_and_subdivision.md
@@ -1,9 +1,10 @@
 # Reciprocal-space mapping: locality, subdivision cost, and the scheduling that would carry them
 
-> **Status: idea for later. Nothing here is implemented, and no decision has
-> been taken.** The measurements are real and were taken against real beamtime
-> data; the designs sketched from them are proposals. Treat this as a record of
-> an investigation, not a plan of record.
+> **Status: partly implemented.** Phases 1 and 2 have landed, and so have
+> phase 3's kernel and pipeline wiring; phase 3's scheduler and phase 4 have
+> not. Each step below says which it is and what it measured. Everything
+> here was measured against real beamtime data, and where a prediction was
+> made before implementing, the step records whether it held.
 
 Third companion to `reciprocal_space_scratch_architecture.md` (what the pipeline
 *is*) and `reciprocal_space_performance_findings.md` (what has been measured
@@ -13,7 +14,11 @@ to do to carry them.
 
 - **A — cross-frame locality.** The native work block deduplicates voxels within
   one image, mostly by accident. Extending it across adjacent images cuts
-  emitted records by ~42% and *shrinks* the cache working set. **Not started.**
+  emitted records by ~42% and *shrinks* the cache working set. **The kernel and
+  the pipeline wiring are implemented; the scheduler is not.** Records through
+  the router fall to 0.598x at F = 8 on the real job, matching the geometric
+  prediction — but end-to-end throughput *regresses* until the scheduler stops
+  paying for group size out of the worker count. See phase 3.
 - **B — subdivision cost.** Above depth 3 the shared-corner cache switched off
   and the kernel did several times more ray work than it needed to. A
   corner-passing recursion removes that, bit-for-bit. **Phase 1 is implemented**
@@ -630,16 +635,120 @@ Not required by either finding; do not fold it in.
    detector geometry, currently recomputed 3651 times per pixel. A
    whole-detector centre-ray cache (149 MB, alongside the existing 143 MiB
    corner cache) removes it entirely, with or without grouping.
-3. *Banding.* Bricks iterate row band by row band so level-3 records stay near
-   today's ~110 MB, as above.
-4. *Choosing F per job.* Extend the calibration probe: run `accumulate_group` on
-   one sample brick at F in {1, 2, 4, 8, 16}, take the smallest F past which the
-   marginal density gain falls below a few per cent, capped by memory. Five small
-   native calls, and it measures the quantity that actually matters rather than
-   deriving it from the angular step. `F = 1` must reproduce today exactly.
-5. *Scheduler.* As above: pin `kernel_threads`, size the correct pool from the
-   measured Python-to-native ratio, retarget `_BoundedGate`, repoint the live
+3. *Banding.* **Done, and it needed no new code.** The detector tiling that
+   already exists is the banding: `_execution_layout` gives this job six
+   full-width bands of ~421 x 2463 (1.04 Mpx), and `_map_frame_group` maps
+   one band across all F frames before moving to the next. At F = 8 a band
+   is 8.3 M samples, which the kernel's own precheck passes comfortably
+   (208 B/sample against 10 GiB / 8.3 M = 1290 B available), and its record
+   footprint lands near 93 MB — the same place as the ~110 MB figure above.
+   Tiling has acquired the real purpose this document predicted it would.
+4. *Wiring it into the mapping path.* **Done.** `_map_one_frame` is gone;
+   `_map_frame_group` replaces it and is the only path, with F = 1 the
+   degenerate case rather than a separate branch. `_ReconstructionSpec`
+   carries `frames_per_group`. The pieces that had to move with it:
+
+   - `_CheckpointRouter.route()` takes `frames=`, because the checkpoint
+     countdown and the `frames_covered` attribute that makes a part
+     resumable both count frames, not `route()` calls. A group declaring
+     one frame would leave its checkpoint permanently short and
+     unresumable. It also rejects a group straddling a checkpoint
+     boundary, which is otherwise a silent corruption of that bookkeeping.
+   - `_frame_groups` cuts groups at the group size and at every grid's
+     checkpoint boundaries. Scheduling ranges are already contiguous and
+     exclusion-free, so `excluded_frames` needs no separate handling —
+     which is a weaker statement than this document's risk item assumed,
+     and worth having checked rather than assumed.
+   - `_angles_advance_monotonically` drops the job to F = 1 when the
+     angles do not advance monotonically, which is the interlaced-scan
+     answer this document called for.
+   - The `_BoundedGate` bounds in-flight *groups*, not frames. A reader
+     loads a whole group before queueing it, so a per-frame gate would let
+     several readers each hold a partial group with none able to finish.
+   - `_frame_parallelism` charges a worker F x the native tile working set
+     (mirroring the kernel's own precheck, which is written against
+     `frames * tile_pixels`) and F x the corrected-frame buffer, but only
+     1 x the record term — a group emits one merged batch, not F of them.
+
+   **`F = 1` is not bit-for-bit with the old per-image path**, as step 1
+   already established. Same voxels, same contributor counts, totals equal
+   to rounding. That is a `phys`-scope change to every job, not only to
+   jobs that opt into grouping.
+5. *Scheduler.* **Not done, and now the blocker** — see below. Pin
+   `kernel_threads`, size the correct pool from the measured
+   Python-to-native ratio, retarget `_BoundedGate`, repoint the live
    rebalance at `F`.
+6. *Choosing F per job.* **Not done, and deliberately moved behind step 5.**
+   Extend the calibration probe: run `accumulate_group` on one sample brick
+   at F in {1, 2, 4, 8, 16}, take the smallest F past which the marginal
+   density gain falls below a few per cent, capped by memory. Five small
+   native calls, and it measures the quantity that actually matters rather
+   than deriving it from the angular step. There is no point choosing F
+   against a scheduler that punishes every value above 1.
+
+### What the wired pipeline measured
+
+`benchmarks/benchmark_reconstruction_pipeline.py` runs the real
+`_map_pending_ranges` — reader pool, correction, kernel, router, checkpoint
+writes — over a bounded window and reports both throughput and the records
+*entering* the checkpoint layer. 234 frames of the `39_1-rsmap` job (six
+scheduling ranges, three checkpoints, depth 0, stationary bounds, 24
+threads, the job's own 10 GB budget), against a baseline captured from the
+same checkout immediately before the change:
+
+| F | records/frame | vs F=1 | ms/frame | vs baseline | workers |
+|---|---|---|---|---|---|
+| baseline (`accumulate`) | 2,584,531 | 1.000 | 236.8 | 1.00 | 8 |
+| 1 | 2,584,531 | 1.000 | 263.1 | 1.11 | 8 |
+| 2 | 2,005,606 | 0.776 | 245.8 | 1.04 | 6 |
+| 4 | 1,699,144 | 0.657 | 248.1 | 1.05 | 3 |
+| 8 | 1,545,463 | **0.598** | 347.4 | **1.47** | **1** |
+
+**The record prediction held.** 0.598 at F = 8 against the 0.575 the kernel
+benchmark measured on a single tile and the 0.5749 the exhaustive geometric
+model predicted before any of it existed. Finding A survives contact with
+the whole pipeline.
+
+**The throughput prediction did not.** The band stated before implementing
+was 0.7-1.0x; F = 8 landed at 1.47x, outside it and on the wrong side. The
+cause is visible in the last column: `_frame_parallelism` charges a worker
+for F resident frames, so `image_workers` collapses from 8 to 1, and at one
+worker a group's GIL-held correction has nothing to overlap with. This is
+the synchronisation point this document flagged as the main scheduling
+risk, arriving exactly where it was predicted to.
+
+It is a scheduling problem and not a kernel one, which the same instrument
+can show directly. Cutting the detector bands from ~421 rows to 160 lowers
+a worker's native working set, so the same budget affords more workers —
+no extra memory, just less per worker:
+
+| F | bands | workers | ms/frame | records/frame |
+|---|---|---|---|---|
+| 1 | 160-row | 10 | 274.4 | 2,584,531 |
+| 8 | 160-row | 2 | **233.3** | 1,545,463 |
+
+One extra worker takes F = 8 from 347.4 to 233.3 ms/frame — **0.85x against
+like-for-like F = 1**, with byte-identical record volume. So the kernel's
+gain does reach the pipeline; what stands between them is `image_workers`
+being the wrong lever for a work unit that is no longer one frame.
+
+**This retires the question of what order the remaining steps go in.**
+Step 5 must precede step 6: `kernel_threads` pinned at the budget, one
+group at a time, and a prefetch/correct pool sized from the measured
+Python-to-native ratio, so that F stops competing with worker count for
+the same memory. Until then `frames_per_group` defaults to 1 and grouping
+is opt-in.
+
+Two smaller things the measurement settled. Records *written* to checkpoint
+parts barely move with F (0.97x at F = 8) because the checkpoint
+accumulator already merges across frames — it collapses the same
+redundancy later and more expensively, so the saving never appears there.
+Records entering the router is the quantity that matters, and it is what
+the table above reports. And F = 1 through `accumulate_group` routes
+*exactly* the same record count as `accumulate` (604,780,339, to the
+record): the 3.4% block-shape gain is real in block records before the
+cross-block merge, but one call's merged output is the set of voxels that
+frame reached, which does not depend on how the frame was partitioned.
 
 ### Phase 4 — Frame-group bricks above depth 0
 
@@ -679,7 +788,10 @@ exposures and cross-frame overlap above depth 0 — each frame then covers a
 0.71-voxel slab rather than a plane, and adjacent slabs tile rather than overlap.
 
 **`excluded_frames`** must be skipped when forming groups, not merely masked
-inside them.
+inside them. *Resolved, and more cheaply than expected:* `_included_ranges`
+already splits scheduling ranges at every excluded frame, so a range is
+contiguous and exclusion-free by construction and `_frame_groups` only has
+to cut at checkpoint boundaries.
 
 **Records converge at depth 3.** 0.632 -> 0.642 -> 0.647 -> 0.650 across depths
 3-6, while cost rises 3x / 6x / 13x. Everything above depth 3 buys finer
@@ -691,4 +803,18 @@ worth.
 ## Also
 
 `benchmarks/benchmark_reconstruction_mapping.py` no longer imports: it wants
-`_map_frame_range`, which was replaced by `_map_one_frame`.
+`_map_frame_range` and the Parquet-era writers, both of which predate the
+checkpoint router. `benchmarks/benchmark_reconstruction_pipeline.py` is the
+replacement and measures the pipeline that actually exists.
+
+Read back the checkpoint parts in row chunks, never all at once. A 234-frame
+window is ~350 M rows, and concatenating them costs more than the mapping
+run whose peak is already sized to the job's whole memory budget — the
+first version of the pipeline benchmark swapped the machine doing exactly
+that. The comparable quantity is `sum over rows of h(key) * contributors`
+accumulated in wrapping `uint64`: it is a sum, so it is independent of row
+order and of how the accumulator happened to split its parts, and it pins
+both the voxel set and the contributor apportionment. Two baseline runs
+with visibly different flush partitions (352.3 M against 350.1 M rows)
+agreed on it exactly, which is what makes it usable as an acceptance
+criterion where a digest is not.

From aee12bddb526ac931043a6620ecaf5810d23e2f2 Mon Sep 17 00:00:00 2001
From: "T. Fuchs" <tfuchs@cornell.edu>
Date: Mon, 10 Aug 2026 18:52:40 -0400
Subject: [PATCH 12/33] perf(phys): schedule frame groups with a prepare pool

Grouping was not worth turning on while the per-frame scheduler charged
a worker for every resident frame. Moving correction into a prepare pool
takes mapping to 0.866x the per-frame pipeline at four frames per call.

A different number of concurrent calls changes how sums associate.
---
 orgui/datautils/xrayutils/reconstruction.py   |  34 +-
 .../xrayutils/test/test_reconstruction_job.py | 133 +++++
 orgui/reconstruction_job.py                   | 492 +++++++++++++++++-
 3 files changed, 634 insertions(+), 25 deletions(-)

diff --git a/orgui/datautils/xrayutils/reconstruction.py b/orgui/datautils/xrayutils/reconstruction.py
index 06183d6..1fbeeb4 100644
--- a/orgui/datautils/xrayutils/reconstruction.py
+++ b/orgui/datautils/xrayutils/reconstruction.py
@@ -1494,6 +1494,8 @@ def _map_frame_group(
     angles_start: np.ndarray,
     angles_end: np.ndarray,
     router: _CheckpointRouter,
+    *,
+    corrected_frames: Sequence[tuple] | None = None,
 ) -> None:
     """Process one already-loaded group of frames: correction, per-tile-
     per-grid kernel accumulate, merge, and route.
@@ -1522,17 +1524,24 @@ def _map_frame_group(
     rounding, and about 3.4% fewer records from the better block shape.
 
     The correction pipeline's whole-frame ``correct_frame`` step (pixel
-    repair, static factors) runs here, in the compute worker, not in the
-    prefetch reader -- it is CPU-bound work that belongs inside the
-    ``image_workers``/``kernel_threads`` thread budget, not hidden inside
-    I/O-rate bookkeeping the reader pool's blocked-fraction signal
-    depends on staying clean.
+    repair, static factors) runs here by default, in the compute worker.
+    Pass ``corrected_frames`` to have it run somewhere else instead: the
+    grouped scheduler corrects in its prepare pool, so that the GIL-held
+    Python work of group N+1 overlaps group N's native call rather than
+    queueing behind it. A pipeline with no ``correct_frame`` corrects per
+    tile here either way, since there is nothing whole-frame to hoist.
 
     :param image_payloads:
-        Loaded payloads, one per frame in the group, in frame order.
+        Loaded payloads, one per frame in the group, in frame order. May
+        be ``None`` when ``corrected_frames`` supplies the arrays and the
+        pipeline has a ``correct_frame`` step, since the raw images are
+        then no longer needed and are better released.
     :param frame_indices:
         The group's frame indices: contiguous, ascending, all inside one
         checkpoint range per grid.
+    :param corrected_frames:
+        Optional pre-corrected ``(intensity, variance, mask)`` triples,
+        one per frame, each full-detector sized.
     :param angles_start:
         ``(frames, 4)`` exposure-start diffractometer angles, radians.
     :param angles_end:
@@ -1541,22 +1550,23 @@ def _map_frame_group(
         per frame, so a group may mix the two.
     """
     frame_indices = [int(index) for index in frame_indices]
-    images = [np.asarray(payload.img) for payload in image_payloads]
+    images = (
+        [np.asarray(payload.img) for payload in image_payloads]
+        if image_payloads is not None
+        else [None] * len(frame_indices)
+    )
     frame_correction = getattr(correction_pipeline, "correct_frame", None)
     # The whole group's corrected frames stay resident until every tile
     # has been mapped: a tile is a row band of the detector, so each
     # frame is revisited once per band. This is the group buffer, and it
     # is what _frame_parallelism sizes a compute worker against.
-    corrected_frames = (
-        [
+    if corrected_frames is None and callable(frame_correction):
+        corrected_frames = [
             frame_correction(payload, image, frame_index)
             for payload, image, frame_index in zip(
                 image_payloads, images, frame_indices
             )
         ]
-        if callable(frame_correction)
-        else None
-    )
     # Collect every tile's contribution per grid, then route exactly once
     # per (group, grid) -- the router's remaining-frame countdown
     # (Sec9/Sec10) counts frames, so a multi-tile group must be merged
diff --git a/orgui/datautils/xrayutils/test/test_reconstruction_job.py b/orgui/datautils/xrayutils/test/test_reconstruction_job.py
index bb9edf3..c70f1fe 100644
--- a/orgui/datautils/xrayutils/test/test_reconstruction_job.py
+++ b/orgui/datautils/xrayutils/test/test_reconstruction_job.py
@@ -1,5 +1,6 @@
 """End-to-end tests for the centralized reconstruction job."""
 
+import dataclasses
 from hashlib import sha256
 import json
 import threading
@@ -30,6 +31,7 @@
     _angles_advance_monotonically,
     _frame_groups,
     _frame_parallelism,
+    _group_pipeline_layout,
     _node_checkpoint_plan,
     _node_excluded_frames,
     job_status,
@@ -984,3 +986,134 @@ def worker_bytes(frames_per_group):
             + record_term
         )
         assert worker_bytes(frames_per_group) >= expected
+
+
+def _map_with_group_size(tmp_path, frames_per_group, frame_count=8):
+    """Map one range end to end at a given group size; return its output.
+
+    Goes through _map_pending_ranges, so above one frame per group this
+    exercises the streamed scheduler in full -- dispatcher, prepare pool,
+    compute pool and gate -- rather than _map_frame_group alone.
+    """
+    root = tmp_path / f"group{frames_per_group}"
+    root.mkdir()
+    scan, job = _multi_frame_job(root, "result.h5", frame_count)
+    # One checkpoint over every frame, so groups are cut by their own size
+    # rather than by a boundary between every pair of frames.
+    job.checkpoint_plan = {"q_lab": [[0, frame_count]]}
+    config = job.config_data
+    spec = dataclasses.replace(
+        job.internal_spec(), frames_per_group=frames_per_group
+    )
+    boundaries = {"q_lab": [(0, frame_count)]}
+    checkpoint_dir = root / "checkpoints"
+    router = _CheckpointRouter(
+        boundaries,
+        spec_digest=job.digest,
+        checkpoint_dir=checkpoint_dir,
+        active_budget_bytes=64 * 1024**2,
+    )
+    reconstruction_job_module._map_pending_ranges(
+        spec,
+        scan,
+        config,
+        scan.exposure_angle_bounds(config, fallback=job.angle_fallback),
+        [(0, 2, 0, 2)],
+        [(0, frame_count)],
+        router,
+        correction_pipeline=reconstruction_job_module._correction_pipeline(
+            config, scan, reconstruction_job_module._load_assets(job), {}
+        ),
+        effective_memory=64 * 1024**2,
+        threads_per_image=1,
+        accumulation_budget_bytes=None,
+        total_images=frame_count,
+        completed_images=0,
+        progress=None,
+    )
+    contributors: dict[tuple[int, int], int] = {}
+    frames_covered = 0
+    for path in sorted(checkpoint_dir.glob("q_lab/ckpt*.h5")):
+        with h5py.File(path, "r") as handle:
+            frames_covered += int(handle.attrs["frames_covered"])
+            for chunk, voxel, count in zip(
+                handle["chunk_id"][:],
+                handle["local_voxel_id"][:],
+                handle["contributors"][:],
+            ):
+                key = (int(chunk), int(voxel))
+                contributors[key] = contributors.get(key, 0) + int(count)
+    return frames_covered, contributors
+
+
+def test_streamed_group_scheduler_matches_the_per_frame_pipeline(tmp_path):
+    """The grouped scheduler is a different pipeline -- one all-threads
+    call at a time, correction hoisted into the prepare pool -- so it has
+    to be shown to produce the per-frame pipeline's answer, not just to
+    run."""
+    frames_covered, reference = _map_with_group_size(tmp_path, 1)
+    assert frames_covered == 8
+    assert reference, "the fixture must reach at least one voxel"
+
+    for frames_per_group in (2, 4):
+        covered, contributors = _map_with_group_size(
+            tmp_path, frames_per_group
+        )
+        # Every frame is still counted exactly once, which is what makes
+        # the checkpoint resumable.
+        assert covered == 8
+        assert contributors == reference
+
+
+def test_group_pipeline_layout_trades_concurrency_for_group_size():
+    """A group buffer and a group's native working set both scale with the
+    group size, so a larger group buys read-ahead out of concurrency. The
+    thread budget is then split between whatever concurrency survives."""
+    grid = _GridSpec(
+        minimum=(0.0, 0.0, 0.0),
+        maximum=(1.0, 1.0, 1.0),
+        step=(0.1, 0.1, 0.1),
+        frame="lab",
+        chunk_shape=(8, 8, 8),
+    )
+    tiles = [(0, 256, 0, 1024), (256, 512, 0, 1024)]
+    budget = 8 * 1024**3
+    previous_workers = None
+    for frames_per_group in (1, 2, 4, 8, 16):
+        spec = _ReconstructionSpec(
+            grids=(grid,),
+            max_depth=0,
+            threads=24,
+            frames_per_group=frames_per_group,
+        )
+        workers, threads, depth = _group_pipeline_layout(spec, tiles, budget, 4)
+        assert workers >= 1
+        assert threads >= 1
+        assert workers * threads <= 24
+        # Always at least one group of read-ahead beyond the calls in
+        # flight, or there is no double-buffering and every call stalls at
+        # its group boundary.
+        assert depth > workers
+        if previous_workers is not None:
+            assert workers <= previous_workers
+        previous_workers = workers
+
+
+def test_group_pipeline_layout_never_returns_zero_workers_on_a_tiny_budget():
+    grid = _GridSpec(
+        minimum=(0.0, 0.0, 0.0),
+        maximum=(1.0, 1.0, 1.0),
+        step=(0.1, 0.1, 0.1),
+        frame="lab",
+        chunk_shape=(8, 8, 8),
+    )
+    spec = _ReconstructionSpec(
+        grids=(grid,), max_depth=0, threads=8, frames_per_group=16
+    )
+
+    workers, threads, depth = _group_pipeline_layout(
+        spec, [(0, 2048, 0, 2048)], 1024**2, 4
+    )
+
+    assert (workers, threads) == (1, 8)
+    assert depth > workers
diff --git a/orgui/reconstruction_job.py b/orgui/reconstruction_job.py
index 6e844ca..9681b8d 100644
--- a/orgui/reconstruction_job.py
+++ b/orgui/reconstruction_job.py
@@ -2202,6 +2202,129 @@ def _kernel_threads_candidates(total_threads, include=()):
     return sorted(candidates)
 
 
+_GROUP_PIPELINE_MINIMUM_READAHEAD = 1
+"""Groups prepared *beyond* the ones being mapped.
+
+At least one, or there is no double-buffering at all: every concurrent
+call would stop dead at its group boundary while the next group is loaded
+and corrected, which is the synchronisation point that makes grouping
+lose."""
+_GROUP_PIPELINE_MAXIMUM_READAHEAD = 3
+"""More than a few groups of read-ahead buys nothing and costs the largest
+resident term in the pipeline, one full group buffer each."""
+
+
+class _GroupAssembly:
+    """One frame group being loaded and corrected, frame by frame.
+
+    Prepare workers cooperate on a single group rather than each taking a
+    whole group of their own. A group buffer is ``F`` full-detector
+    corrected frames -- the largest resident term in the pipeline -- so
+    per-worker groups would multiply it by the pool size. Frame
+    granularity keeps the number of live buffers down to the pipeline
+    depth while still spreading a group's ``F`` loads and corrections
+    across every worker in the pool.
+    """
+
+    __slots__ = ("frames", "payloads", "corrected", "_remaining", "_lock", "failed")
+
+    def __init__(self, frames):
+        self.frames = list(frames)
+        self.payloads = [None] * len(self.frames)
+        self.corrected = [None] * len(self.frames)
+        self._remaining = len(self.frames)
+        self._lock = threading.Lock()
+        self.failed = False
+
+    def complete_one(self, *, failed=False) -> bool:
+        """Record one frame finished. ``True`` when the group is whole."""
+        with self._lock:
+            if failed:
+                self.failed = True
+            self._remaining -= 1
+            return self._remaining <= 0
+
+
+def _group_pipeline_layout(spec, tiles, memory_bytes, prepare_workers):
+    """Concurrent group calls, native threads each, and read-ahead depth.
+
+    The grouped scheduler spends its memory differently from the
+    per-frame one. There, every concurrent frame carried its own native
+    working set *and* its own record batch, so the budget bought
+    ``image_workers`` directly. Here the budget is split three ways:
+    concurrent group calls, the read-ahead buffer in front of them, and
+    the prepare pool's own transients.
+
+    One concurrent call with every thread was the design's first guess
+    and it measured badly -- three calls of eight threads beat one call
+    of twenty-four by 1.38x on real data at four frames per group. A
+    group call has thousands of bricks, so the thread budget is not
+    limited by work available to hand out; it is limited by what the call
+    does around the parallel region. Concurrency therefore still has to
+    be bought, exactly as it did before grouping, and this decides how
+    much of it the budget affords.
+
+    :param int prepare_workers:
+        Prepare pool size. Each worker holds one raw image and one
+        corrected frame while it works, independently of the group
+        buffers those frames land in.
+    :returns:
+        ``(compute_workers, kernel_threads, pipeline_depth)``.
+    :rtype: tuple[int, int, int]
+    """
+    tiles = list(tiles)
+    tile_pixels = [
+        (row_stop - row_start) * (column_stop - column_start)
+        for row_start, row_stop, column_start, column_stop in tiles
+    ]
+    if not tile_pixels:
+        return (
+            1,
+            max(1, int(spec.threads)),
+            1 + _GROUP_PIPELINE_MINIMUM_READAHEAD,
+        )
+    detector_pixels = sum(tile_pixels)
+    frames_per_group = max(1, int(getattr(spec, "frames_per_group", 1)))
+    native_bytes_per_pixel = (
+        128 + 2 * _RESERVED_RECORDS_PER_PIXEL * _CHECKPOINT_BYTES_PER_ROW
+        if spec.max_depth > 0
+        else 128 + 2 * _CHECKPOINT_BYTES_PER_ROW
+    )
+    # Per concurrent group call: its native working set over the largest
+    # tile times the group size, plus the merged record batch it builds.
+    call_bytes = (
+        max(tile_pixels) * frames_per_group * native_bytes_per_pixel
+        + detector_pixels * _FRAME_RECORD_BYTES_PER_PIXEL
+    )
+    group_buffer_bytes = max(
+        1, frames_per_group * detector_pixels * _PYTHON_CORRECTION_BYTES_PER_PIXEL
+    )
+    prepare_bytes = max(1, int(prepare_workers)) * detector_pixels * (
+        _RAW_IMAGE_BYTES_PER_PIXEL + _PYTHON_CORRECTION_BYTES_PER_PIXEL
+    )
+    spare = max(0, int(memory_bytes) - prepare_bytes)
+    # Every concurrent call needs a group buffer to read from, and the
+    # read-ahead sits on top of that -- so a worker costs both terms.
+    per_worker_bytes = call_bytes + group_buffer_bytes
+    compute_workers = max(
+        1,
+        min(
+            max(1, int(spec.threads)),
+            spare // max(1, per_worker_bytes),
+        ),
+    )
+    kernel_threads = max(1, int(spec.threads) // compute_workers)
+    remaining = spare - compute_workers * per_worker_bytes
+    pipeline_depth = compute_workers + max(
+        _GROUP_PIPELINE_MINIMUM_READAHEAD,
+        min(
+            _GROUP_PIPELINE_MAXIMUM_READAHEAD,
+            remaining // group_buffer_bytes,
+        ),
+    )
+    return compute_workers, kernel_threads, pipeline_depth
+
+
 def _angles_advance_monotonically(bounds, frame_indices):
     """Whether frames adjacent in index are also adjacent in angle.
 
@@ -2276,6 +2399,315 @@ def _frame_groups(frame_range, router, grid_names, frames_per_group):
     return groups
 
 
+def _map_frame_groups_streamed(
+    spec,
+    scan,
+    config,
+    bounds,
+    detector_tiles,
+    ray_arrays,
+    frame_groups,
+    router,
+    *,
+    correction_pipeline,
+    kernel_memory_budget,
+    compute_workers,
+    kernel_threads,
+    pipeline_depth,
+    prepare_workers,
+    maximum_prepare_workers,
+    total_images,
+    completed_images,
+    progress,
+):
+    """Map frame groups with one all-threads native call at a time.
+
+    The scheduler the per-frame pipeline uses answers a question frame
+    grouping dissolves. There, one frame's kernel call could not
+    profitably use every thread, so the budget was split jointly between
+    ``image_workers`` and ``kernel_threads`` and rebalanced live. A group
+    call has thousands of bricks and saturates the thread budget on its
+    own, so there is nothing left to split: ``kernel_threads`` is pinned
+    at the budget and exactly one group is mapped at a time.
+
+    What replaces the joint balance is a simpler question -- how many
+    workers it takes to keep that one call fed. Loading and correcting a
+    frame is GIL-held Python; mapping is not. So correction moves out of
+    the compute worker and into the prepare pool, where group N+1's
+    Python work overlaps group N's native call instead of queueing behind
+    it. Without that move a group is a hard synchronisation point and the
+    pipeline stalls once per group, which is what made grouping lose
+    end-to-end even while it cut records by 40%.
+
+    The pool is sized the same way the reader pool always was, by the
+    blocked fraction the compute worker measures: grow eagerly when it is
+    starved, shrink cautiously when it is not. That signal now steers
+    load *and* correction together, which is the whole per-frame Python
+    cost rather than only its I/O half.
+
+    Mutates ``router`` in place; returns nothing.
+
+    :param int compute_workers:
+        Concurrent group calls.
+    :param int kernel_threads:
+        Native threads per call.
+    :param int pipeline_depth:
+        Groups that may be resident at once
+        (:func:`_group_pipeline_layout`).
+    :param int prepare_workers:
+        Initial prepare pool size.
+    :param int maximum_prepare_workers:
+        Ceiling for the pool, one when the scan cannot be read
+        concurrently.
+    """
+    frame_correction = getattr(correction_pipeline, "correct_frame", None)
+    corrects_whole_frame = callable(frame_correction)
+
+    progress_events = SimpleQueue()
+    frame_queue = SimpleQueue()
+    ready_queue = SimpleQueue()
+    gate = _BoundedGate(pipeline_depth)
+    dispatch_done = threading.Event()
+    mapped_images = completed_images
+
+    counter_lock = threading.Lock()
+    dispatched = [0]
+    completed = [0]
+
+    exception_lock = threading.Lock()
+    first_exception: list[BaseException] = []
+
+    def record_exception(exc):
+        with exception_lock:
+            if not first_exception:
+                first_exception.append(exc)
+
+    def should_stop():
+        return bool(first_exception)
+
+    def all_groups_retired():
+        with counter_lock:
+            return dispatch_done.is_set() and completed[0] >= dispatched[0]
+
+    def dispatch_loop(cancellation):
+        """Hand out one group's frames at a time, under the gate.
+
+        Issuing every group's frames up front would let the pool spread
+        itself over all of them and hold a group buffer for each. The
+        gate is acquired per group and released when that group has been
+        mapped, so the number of live buffers is the pipeline depth and
+        nothing else.
+        """
+        def stop_waiting():
+            return cancellation.is_set() or should_stop()
+
+        try:
+            for group in frame_groups:
+                if stop_waiting():
+                    break
+                if not gate.acquire(
+                    poll_timeout=_POLL_TIMEOUT_SECONDS,
+                    should_stop=stop_waiting,
+                ):
+                    break
+                assembly = _GroupAssembly(group)
+                with counter_lock:
+                    dispatched[0] += 1
+                for slot in range(len(group)):
+                    frame_queue.put((assembly, slot))
+        except BaseException as exc:  # noqa: BLE001 -- must reach the coordinator
+            record_exception(exc)
+        finally:
+            dispatch_done.set()
+
+    def prepare_loop(retire, cancellation):
+        """Load and correct one frame per item, into its group's slot.
+
+        Exits only between items, never mid-frame: a claimed slot must be
+        completed one way or the other, since its group can never become
+        whole otherwise and the coordinator would wait forever for a
+        group that no one is still working on.
+        """
+        try:
+            while True:
+                if retire.is_set() or cancellation.is_set():
+                    return
+                try:
+                    assembly, slot = frame_queue.get(
+                        timeout=_POLL_TIMEOUT_SECONDS
+                    )
+                except Empty:
+                    if dispatch_done.is_set() and frame_queue.empty():
+                        return
+                    continue
+                failed = False
+                if should_stop():
+                    # Drain without doing the work: the group still has to
+                    # complete so the compute side can retire it and
+                    # release its gate slot.
+                    failed = True
+                else:
+                    frame_index = assembly.frames[slot]
+                    try:
+                        payload = scan.get_raw_img(frame_index)
+                        if corrects_whole_frame:
+                            assembly.corrected[slot] = frame_correction(
+                                payload, np.asarray(payload.img), frame_index
+                            )
+                        else:
+                            assembly.payloads[slot] = payload
+                    except BaseException as exc:  # noqa: BLE001
+                        record_exception(exc)
+                        failed = True
+                if assembly.complete_one(failed=failed):
+                    ready_queue.put(assembly)
+        except BaseException as exc:  # noqa: BLE001
+            record_exception(exc)
+
+    blocked_counts: dict[int, list[int]] = {}
+    blocked_counts_lock = threading.Lock()
+
+    def compute_loop(retire, cancellation):
+        counts = [0, 0]  # [blocked, total], only this thread ever writes
+        with blocked_counts_lock:
+            blocked_counts[id(counts)] = counts
+        try:
+            try:
+                kernels = _build_kernels(
+                    spec,
+                    config.ub_calculator,
+                    threads=kernel_threads,
+                    memory_budget_bytes=kernel_memory_budget,
+                )
+                while True:
+                    if retire.is_set() or cancellation.is_set():
+                        return
+                    try:
+                        assembly = ready_queue.get(
+                            timeout=_POLL_TIMEOUT_SECONDS
+                        )
+                    except Empty:
+                        counts[0] += 1
+                        counts[1] += 1
+                        if all_groups_retired() and ready_queue.empty():
+                            return
+                        continue
+                    counts[1] += 1
+                    try:
+                        if not assembly.failed:
+                            group = assembly.frames
+                            _map_frame_group(
+                                spec,
+                                kernels,
+                                ray_arrays,
+                                detector_tiles,
+                                correction_pipeline,
+                                None if corrects_whole_frame else assembly.payloads,
+                                group,
+                                np.ascontiguousarray(bounds[group, 0]),
+                                np.ascontiguousarray(bounds[group, 1]),
+                                router,
+                                corrected_frames=(
+                                    assembly.corrected
+                                    if corrects_whole_frame
+                                    else None
+                                ),
+                            )
+                    except BaseException as exc:  # noqa: BLE001
+                        record_exception(exc)
+                    finally:
+                        # Drop the group buffer before releasing its slot,
+                        # so the gate bounds what is actually resident
+                        # rather than what has merely been claimed.
+                        assembly.corrected = []
+                        assembly.payloads = []
+                        with counter_lock:
+                            completed[0] += 1
+                        gate.release()
+                    if not assembly.failed:
+                        progress_events.put(len(assembly.frames))
+            except BaseException as exc:  # noqa: BLE001
+                record_exception(exc)
+        finally:
+            with blocked_counts_lock:
+                blocked_counts.pop(id(counts), None)
+
+    dispatch_cancellation = threading.Event()
+    dispatcher = threading.Thread(
+        target=dispatch_loop,
+        args=(dispatch_cancellation,),
+        name="orgui-rsmap-dispatch",
+        daemon=True,
+    )
+    dispatcher.start()
+    prepare_pool = _AdjustablePool(
+        prepare_loop, initial_size=prepare_workers, name="orgui-rsmap-prepare"
+    )
+    compute_pool = _AdjustablePool(
+        compute_loop, initial_size=compute_workers, name="orgui-rsmap-compute"
+    )
+    previous_blocked = 0
+    previous_total = 0
+    try:
+        while True:
+            deadline = time.monotonic() + _COORDINATOR_TICK_SECONDS
+            while time.monotonic() < deadline:
+                try:
+                    mapped_in_group = progress_events.get(
+                        timeout=max(0.0, deadline - time.monotonic())
+                    )
+                except Empty:
+                    break
+                mapped_images += mapped_in_group
+                if progress is not None:
+                    progress(
+                        mapped_images,
+                        total_images + 1,
+                        (
+                            f"Mapping images {mapped_images}/{total_images} "
+                            f"({spec.frames_per_group} frames/call, "
+                            f"{compute_pool.size} concurrent calls x "
+                            f"{kernel_threads} native threads, "
+                            f"{prepare_pool.size} prepare workers)"
+                        ),
+                    )
+            with blocked_counts_lock:
+                blocked = sum(counts[0] for counts in blocked_counts.values())
+                total = sum(counts[1] for counts in blocked_counts.values())
+            window_blocked = max(0, blocked - previous_blocked)
+            window_total = max(0, total - previous_total)
+            previous_blocked, previous_total = blocked, total
+            if window_total > 0 and not all_groups_retired() and not first_exception:
+                blocked_fraction = window_blocked / window_total
+                if blocked_fraction > _BLOCKED_FRACTION_GROW:
+                    prepare_pool.retarget(
+                        min(maximum_prepare_workers, prepare_pool.size + 2)
+                    )
+                elif (
+                    blocked_fraction < _BLOCKED_FRACTION_SHRINK
+                    and prepare_pool.size > 1
+                ):
+                    prepare_pool.retarget(prepare_pool.size - 1)
+            prepare_pool.reap()
+            compute_pool.reap()
+            with blocked_counts_lock:
+                still_computing = bool(blocked_counts)
+            if not still_computing and (all_groups_retired() or first_exception):
+                # The second condition is not redundant. A compute worker
+                # that dies -- building its kernels, or on a mapping
+                # failure -- leaves dispatched groups that nothing will
+                # ever complete, so waiting for the counters to agree
+                # would hang here forever instead of raising.
+                break
+        if first_exception:
+            raise first_exception[0]
+    finally:
+        dispatch_cancellation.set()
+        dispatcher.join(timeout=_POLL_TIMEOUT_SECONDS * 10)
+        prepare_pool.shutdown(wait=True)
+        compute_pool.shutdown(wait=True)
+
+
 def _map_pending_ranges(
     spec,
     scan,
@@ -2310,14 +2742,21 @@ def _map_pending_ranges(
     when the scan's angles do not advance monotonically
     (:func:`_angles_advance_monotonically`).
 
-    ``threads_per_image=None`` (Sec7 Phase 4b, "automatic") starts from an
-    I/O-optimistic seed (``kernel_threads=1``, all budget as
-    ``image_workers``) and periodically rebalances ``kernel_threads``/
-    ``image_workers`` live against a measured frame-delivery rate (every
-    ``_REBALANCE_INITIAL_SECONDS``, backing off while nothing
-    changes), via a wall-clock
-    ``_kernel_threads_sweep`` and the design doc's joint-balancing rule.
-    A concrete ``threads_per_image`` int keeps the static,
+    Above one frame per group the whole scheduling question changes, and
+    :func:`_map_frame_groups_streamed` answers the new one instead: a
+    group call saturates the thread budget by itself, so there is no pair
+    left to balance, and what matters is keeping that one call fed. See
+    its docstring.
+
+    At one frame per group the joint balance still applies, and is
+    unchanged. ``threads_per_image=None`` (Sec7 Phase 4b, "automatic")
+    starts from an I/O-optimistic seed (``kernel_threads=1``, all budget
+    as ``image_workers``) and periodically rebalances
+    ``kernel_threads``/``image_workers`` live against a measured
+    frame-delivery rate (every ``_REBALANCE_INITIAL_SECONDS``, backing off
+    while nothing changes), via a wall-clock ``_kernel_threads_sweep`` and
+    the design doc's joint-balancing rule. A concrete
+    ``threads_per_image`` int keeps the static,
     ``_frame_parallelism``-derived pair fixed for the whole run (today's
     Phase 4a behavior, unchanged) -- the explicit override escape hatch.
 
@@ -2426,6 +2865,38 @@ def layout_for(threads_per_image_seed):
         )
     ]
 
+    max_readers = (
+        _PREFETCH_POOL_MAX
+        if getattr(scan, "supports_concurrent_read", True)
+        else 1
+    )
+    if frames_per_group > 1:
+        seed_prepare_workers = min(_PREFETCH_POOL_INITIAL, max_readers)
+        group_workers, group_threads, group_depth = _group_pipeline_layout(
+            spec, tiles, scheduler_memory, seed_prepare_workers
+        )
+        _map_frame_groups_streamed(
+            spec,
+            scan,
+            config,
+            bounds,
+            detector_tiles,
+            ray_arrays,
+            frame_groups,
+            router,
+            correction_pipeline=correction_pipeline,
+            kernel_memory_budget=effective_memory,
+            compute_workers=group_workers,
+            kernel_threads=group_threads,
+            pipeline_depth=group_depth,
+            prepare_workers=seed_prepare_workers,
+            maximum_prepare_workers=max_readers,
+            total_images=total_images,
+            completed_images=completed_images,
+            progress=progress,
+        )
+        return
+
     if automatic:
         # A single representative tile/frame is enough for the sweep: it
         # measures kernel_threads scaling behavior (thread-count
@@ -2450,11 +2921,6 @@ def layout_for(threads_per_image_seed):
             for row_start, row_stop, column_start, column_stop in detector_tiles
         )
 
-    max_readers = (
-        _PREFETCH_POOL_MAX
-        if getattr(scan, "supports_concurrent_read", True)
-        else 1
-    )
     reader_pool_size = min(_PREFETCH_POOL_INITIAL, max_readers)
 
     progress_events = SimpleQueue()

From 34289a98e42976452ba7ade90922907c7e704761 Mon Sep 17 00:00:00 2001
From: "T. Fuchs" <tfuchs@cornell.edu>
Date: Mon, 10 Aug 2026 18:52:55 -0400
Subject: [PATCH 13/33] test: sweep group size, band height and depth in the
 pipeline benchmark

The knobs the grouped scheduler turned out to need measuring against;
the final progress line is now reported alongside the timing.
---
 .../benchmark_reconstruction_pipeline.py      | 60 +++++++++++++++++--
 1 file changed, 56 insertions(+), 4 deletions(-)

diff --git a/benchmarks/benchmark_reconstruction_pipeline.py b/benchmarks/benchmark_reconstruction_pipeline.py
index eeda093..29ea9e3 100644
--- a/benchmarks/benchmark_reconstruction_pipeline.py
+++ b/benchmarks/benchmark_reconstruction_pipeline.py
@@ -46,11 +46,13 @@
     _native_module,
     _PARTIAL_COLUMNS,
 )
+import orgui.reconstruction_job as reconstruction_job
 from orgui.reconstruction_job import (
     _base_provenance,
     _correction_pipeline,
     _execution_layout,
     _frame_parallelism,
+    _group_pipeline_layout,
     _load_assets,
     _map_pending_ranges,
     read_job,
@@ -97,6 +99,21 @@ def _arguments():
             "at the same total budget rather than by raising it."
         ),
     )
+    parser.add_argument(
+        "--compute-workers",
+        type=int,
+        default=None,
+        help=(
+            "Override the grouped scheduler's concurrent group calls. "
+            "Native threads are split evenly between them."
+        ),
+    )
+    parser.add_argument(
+        "--depth",
+        type=int,
+        default=None,
+        help="Override spec.max_depth (the job's accuracy setting).",
+    )
     parser.add_argument("--scratch", type=Path)
     parser.add_argument("--output", type=Path)
     return parser.parse_args()
@@ -211,6 +228,8 @@ def main():
     config = job.config_data
     scan = job.scan
     spec = job.internal_spec()
+    if arguments.depth is not None:
+        spec = replace(spec, max_depth=arguments.depth)
     if arguments.group != 1:
         if not hasattr(spec, "frames_per_group"):
             raise SystemExit(
@@ -305,16 +324,46 @@ def counting_route(grid_name, frame_index, batch, **kwargs):
         "angle_fallback": job.angle_fallback,
         "effective_memory_bytes": int(effective_memory),
         "active_budget_bytes": int(active_budget_bytes),
-        # What the memory budget affords at the I/O-optimistic seed, which
-        # is where automatic mode starts and what a larger group shrinks.
+        # What the memory budget affords. Above one frame per group the
+        # scheduler uses _group_pipeline_layout instead, which the final
+        # progress line reports; these are the per-frame pipeline's.
         "seed_image_workers": int(seed_workers),
         "seed_kernel_threads": int(seed_threads),
         "worker_memory_mb": round(worker_bytes / 1e6, 1),
+        "group_layout": (
+            list(_group_pipeline_layout(spec, tiles, effective_memory, 4))
+            if getattr(spec, "frames_per_group", 1) > 1
+            else None
+        ),
         "native": _native_module().__file__,
         "scratch": str(scratch),
     }
     print(json.dumps(header, sort_keys=True), flush=True)
 
+    if arguments.compute_workers:
+        # A benchmark-only hook. The layout is a memory decision inside
+        # the scheduler, and the point of sweeping it is to find out what
+        # that decision should be aiming at.
+        original_layout = reconstruction_job._group_pipeline_layout
+
+        def fixed_layout(spec, tiles, memory_bytes, prepare_workers):
+            _workers, _threads, depth = original_layout(
+                spec, tiles, memory_bytes, prepare_workers
+            )
+            workers = max(1, arguments.compute_workers)
+            return (
+                workers,
+                max(1, spec.threads // workers),
+                workers + max(1, depth - _workers),
+            )
+
+        reconstruction_job._group_pipeline_layout = fixed_layout
+
+    messages = []
+
+    def capture(value, maximum, message):
+        messages.append(message)
+
     started = perf_counter()
     _map_pending_ranges(
         spec,
@@ -334,7 +383,7 @@ def counting_route(grid_name, frame_index, batch, **kwargs):
         accumulation_budget_bytes=job.accumulation_budget_bytes,
         total_images=total_images,
         completed_images=0,
-        progress=None,
+        progress=capture,
     )
     elapsed = perf_counter() - started
 
@@ -345,6 +394,7 @@ def counting_route(grid_name, frame_index, batch, **kwargs):
         "routed_records": routed["records"],
         "routed_records_per_frame": routed["records"] / total_images,
         "route_calls": routed["calls"],
+        "final_layout": messages[-1] if messages else None,
         "checkpoints_written": len(router.written),
         "grids": _summarize_checkpoints(checkpoint_dir, grid_names),
     }
@@ -354,9 +404,11 @@ def counting_route(grid_name, frame_index, batch, **kwargs):
     print(f"{total_images} frames in {elapsed:.1f} s "
           f"({total_images / elapsed:.3f} frames/s, "
           f"{elapsed / total_images * 1000:.1f} ms/frame)")
+    if messages:
+        print(f"  {messages[-1]}")
     print(f"  routed {routed['records']:,} records in {routed['calls']} calls "
           f"({routed['records'] / total_images:,.0f} per frame); "
-          f"seed layout {seed_workers} workers x {seed_threads} threads")
+          f"per-frame layout {seed_workers} workers x {seed_threads} threads")
     for grid_name, values in result["grids"].items():
         print(
             f"  {grid_name}: {values['rows']:,} rows, "

From 353f233c86b97ed2056cebbd5fe1c50834adc464 Mon Sep 17 00:00:00 2001
From: "T. Fuchs" <tfuchs@cornell.edu>
Date: Mon, 10 Aug 2026 18:52:56 -0400
Subject: [PATCH 14/33] docs: record the grouped scheduler result and correct
 two claims

Mapping is 0.866x the per-frame pipeline at four frames per group.
Corrects the thread-saturation claim, and records that this machine
varies 30% run to run, so arms must be interleaved.
---
 CHANGELOG.md                                  |   8 +
 ..._space_mapping_locality_and_subdivision.md | 145 ++++++++++++------
 2 files changed, 109 insertions(+), 44 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index b8c92a0..5b9d88c 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -79,6 +79,14 @@ Scientific correctness and performance fixes:
   detector samples reach each of them, are unchanged. Existing checkpoint
   files remain resumable, but a job resumed across this change will contain
   parts from both association orders.
+- Mapping several images per call uses a scheduler of its own, which runs a
+  few concurrent native calls with the thread budget split between them and
+  hoists whole-frame correction into the prefetch pool, so one call's
+  GIL-held Python work overlaps the next call's native work instead of
+  queueing behind it. Measured 0.87x the mapping wall time of the
+  per-image pipeline at four images per call on a 3651-frame rotation scan.
+  Group size is not yet chosen automatically, so this path is only taken
+  when a group size is set explicitly.
 
 A ***critical bug*** was fixed that affects bulk CTR calculations:
 
diff --git a/doc/design/reciprocal_space_mapping_locality_and_subdivision.md b/doc/design/reciprocal_space_mapping_locality_and_subdivision.md
index 5d71897..1efb160 100644
--- a/doc/design/reciprocal_space_mapping_locality_and_subdivision.md
+++ b/doc/design/reciprocal_space_mapping_locality_and_subdivision.md
@@ -1,10 +1,11 @@
 # Reciprocal-space mapping: locality, subdivision cost, and the scheduling that would carry them
 
-> **Status: partly implemented.** Phases 1 and 2 have landed, and so have
-> phase 3's kernel and pipeline wiring; phase 3's scheduler and phase 4 have
+> **Status: partly implemented.** Phases 1 and 2 have landed, and so has
+> phase 3 except for choosing the group size per job (step 6); phase 4 has
 > not. Each step below says which it is and what it measured. Everything
 > here was measured against real beamtime data, and where a prediction was
-> made before implementing, the step records whether it held.
+> made before implementing, the step records whether it held — including
+> where it did not.
 
 Third companion to `reciprocal_space_scratch_architecture.md` (what the pipeline
 *is*) and `reciprocal_space_performance_findings.md` (what has been measured
@@ -15,10 +16,10 @@ to do to carry them.
 - **A — cross-frame locality.** The native work block deduplicates voxels within
   one image, mostly by accident. Extending it across adjacent images cuts
   emitted records by ~42% and *shrinks* the cache working set. **The kernel and
-  the pipeline wiring are implemented; the scheduler is not.** Records through
-  the router fall to 0.598x at F = 8 on the real job, matching the geometric
-  prediction — but end-to-end throughput *regresses* until the scheduler stops
-  paying for group size out of the worker count. See phase 3.
+  the pipeline wiring and the scheduler are implemented.** Records through the
+  router fall to 0.598x at F = 8, matching the geometric prediction, and
+  end-to-end mapping is **0.866x at F = 4** on the real job. F is not yet
+  chosen per job, so grouping stays opt-in. See phase 3.
 - **B — subdivision cost.** Above depth 3 the shared-corner cache switched off
   and the kernel did several times more ray work than it needed to. A
   corner-passing recursion removes that, bit-for-bit. **Phase 1 is implemented**
@@ -674,17 +675,76 @@ Not required by either finding; do not fold it in.
    already established. Same voxels, same contributor counts, totals equal
    to rounding. That is a `phys`-scope change to every job, not only to
    jobs that opt into grouping.
-5. *Scheduler.* **Not done, and now the blocker** — see below. Pin
-   `kernel_threads`, size the correct pool from the measured
-   Python-to-native ratio, retarget `_BoundedGate`, repoint the live
-   rebalance at `F`.
-6. *Choosing F per job.* **Not done, and deliberately moved behind step 5.**
-   Extend the calibration probe: run `accumulate_group` on one sample brick
-   at F in {1, 2, 4, 8, 16}, take the smallest F past which the marginal
-   density gain falls below a few per cent, capped by memory. Five small
-   native calls, and it measures the quantity that actually matters rather
-   than deriving it from the angular step. There is no point choosing F
-   against a scheduler that punishes every value above 1.
+5. *Scheduler.* **Done**, as `_map_frame_groups_streamed`, and it makes
+   grouping pay: **0.866x at F = 4** on the real job, measured against the
+   per-frame pipeline. One correction moved, one premise of this document
+   fell over, and the second is the more useful result.
+
+   **What worked: hoisting correction out of the compute worker.** Loading
+   and correcting a frame is GIL-held Python; mapping is not. Correction
+   now runs in a prepare pool, so group N+1's Python work overlaps group
+   N's native call instead of queueing behind it. Prepare workers
+   cooperate on *one* group at a time rather than each taking a group of
+   their own: a group buffer is F full-detector corrected frames, the
+   largest resident term in the pipeline, so per-worker groups would
+   multiply it by the pool size. Frame granularity keeps live buffers down
+   to the pipeline depth while still spreading a group's F loads and
+   corrections across the whole pool.
+
+   **What was wrong: "one group call saturates the thread budget".** This
+   document argued that a group call has ~6000 bricks and so needs no
+   help, which makes `kernel_threads = spec.threads` with one call in
+   flight the natural layout. Implemented exactly that way it measured
+   *1.38x worse* than the old scheduler at F = 4 — 342.4 ms/frame against
+   248.1. Brick count is not the constraint; whatever the call does around
+   its parallel region is. Three concurrent calls of eight threads beat
+   one call of twenty-four, and `_group_pipeline_layout` now buys
+   concurrency from memory much as `_frame_parallelism` did, the
+   difference being that a group buffer is shared by the prepare pool
+   rather than owned by one frame worker.
+
+   Interleaved repeats, 234 frames, the job's own budget and tiling:
+
+   | arm | layout | ms/frame | vs F = 1 |
+   |---|---|---|---|
+   | F = 1 (per-frame pipeline) | 8 workers x 1 | 232.8 | 1.000 |
+   | **F = 4** | **3 calls x 8 threads** | **201.5** | **0.866** |
+   | F = 4, 256-row bands | 4 calls x 6 threads | 205.8 | 0.884 |
+   | F = 8, 256-row bands | 2 calls x 12 threads | 236.1 | 1.014 |
+   | F = 8, the job's own bands | 1 call x 24 threads | 288.5 | 1.24 |
+
+   Every F = 4 run was faster than every F = 1 run, so the separation
+   survives the noise even though each arm spreads about +-7%.
+
+   **F = 4 is the optimum here because the two effects cross.** Records
+   fall monotonically with F (0.776 / 0.657 / 0.598 at F = 2 / 4 / 8) but
+   concurrency falls with it too, since both the group buffer and the
+   call's native working set scale with F. At F = 8 the job's own tiling
+   affords one concurrent call, and that costs more than the extra 6% of
+   records is worth. Banding more finely buys the concurrency back — 256-row
+   bands restore two calls and take F = 8 from 1.24x to 1.014x — but not
+   enough to beat F = 4. **Band height and group size trade against each
+   other for the same memory**, which is a knob this document had not
+   identified, and picking the pair rather than picking F alone is what
+   step 6 should actually do.
+6. *Choosing F per job.* **Not done.** The probe should now choose a
+   (band height, F) pair rather than F alone, and it should optimise
+   throughput rather than record density: records keep falling past the
+   point where throughput turns around, so density is the wrong objective
+   on its own. `F = 4` is a reasonable default for a 0.1 deg/frame scan at
+   depth 0 on a 24-thread machine, and nothing weaker than a measurement
+   should be trusted to generalise that.
+
+### Measurement noise on this machine
+
+Worth recording, because it invalidated a first round of conclusions.
+Single runs of the pipeline benchmark vary by up to **30%** — the same F = 4
+configuration measured 183.4, 202.0 and 342.4 ms/frame on different
+occasions. Comparisons must be **interleaved** (A, B, A, B, ...) and
+repeated, never one run of each in sequence, and nothing else may run on
+the machine at the time. Two conclusions drawn from single sequential runs
+before this was noticed — a "1.47x regression" at F = 8 and a "0.85x win"
+from finer bands — both dissolved when measured properly.
 
 ### What the wired pipeline measured
 
@@ -709,35 +769,21 @@ benchmark measured on a single tile and the 0.5749 the exhaustive geometric
 model predicted before any of it existed. Finding A survives contact with
 the whole pipeline.
 
-**The throughput prediction did not.** The band stated before implementing
-was 0.7-1.0x; F = 8 landed at 1.47x, outside it and on the wrong side. The
+**The throughput prediction did not**, on the per-frame scheduler. The
+band stated before implementing was 0.7-1.0x; F = 8 landed at 1.47x. The
 cause is visible in the last column: `_frame_parallelism` charges a worker
 for F resident frames, so `image_workers` collapses from 8 to 1, and at one
 worker a group's GIL-held correction has nothing to overlap with. This is
 the synchronisation point this document flagged as the main scheduling
-risk, arriving exactly where it was predicted to.
-
-It is a scheduling problem and not a kernel one, which the same instrument
-can show directly. Cutting the detector bands from ~421 rows to 160 lowers
-a worker's native working set, so the same budget affords more workers —
-no extra memory, just less per worker:
-
-| F | bands | workers | ms/frame | records/frame |
-|---|---|---|---|---|
-| 1 | 160-row | 10 | 274.4 | 2,584,531 |
-| 8 | 160-row | 2 | **233.3** | 1,545,463 |
-
-One extra worker takes F = 8 from 347.4 to 233.3 ms/frame — **0.85x against
-like-for-like F = 1**, with byte-identical record volume. So the kernel's
-gain does reach the pipeline; what stands between them is `image_workers`
-being the wrong lever for a work unit that is no longer one frame.
+risk, arriving where it was predicted to — and it is what step 5 below
+fixes, after which F = 4 reaches 0.866x. Read the two together: on this
+scheduler the wiring alone was not worth turning on, and
+`frames_per_group` stayed at 1 until step 5 landed.
 
-**This retires the question of what order the remaining steps go in.**
-Step 5 must precede step 6: `kernel_threads` pinned at the budget, one
-group at a time, and a prefetch/correct pool sized from the measured
-Python-to-native ratio, so that F stops competing with worker count for
-the same memory. Until then `frames_per_group` defaults to 1 and grouping
-is opt-in.
+The individual figures in the table above are single runs, taken before
+the run-to-run spread on this machine was understood, and should be read
+as the shape of the effect rather than as values — see "Measurement noise"
+under step 5. The record column is unaffected: it is deterministic.
 
 Two smaller things the measurement settled. Records *written* to checkpoint
 parts barely move with F (0.97x at F = 8) because the checkpoint
@@ -777,8 +823,19 @@ intermediate roundings, so slightly better conditioned, but any test pinning an
 XXH3 digest of record values will move. That is a `phys`-scope change and needs
 saying so.
 
-**The group synchronisation point**, as described under Scheduling. Prefer F = 8
-until measured.
+**The group synchronisation point**, as described under Scheduling. *Real, and
+resolved by moving correction into the prepare pool (phase 3 step 5).* The
+preference for F = 8 was wrong on this job: F = 4 measured better, because
+concurrency is bought from the same memory the group buffer spends.
+
+**The Scheduling section above is superseded in one respect.** Its claim that a
+group call saturates the thread budget on its own, and that `kernel_threads =
+spec.threads` with one group in flight is therefore the natural layout, was
+implemented and measured 1.38x worse than keeping several concurrent calls. It
+is retained as the reasoning that led to the experiment; phase 3 step 5 records
+what replaced it. The prediction that the pool would need about five workers was
+sound in spirit — the prepare pool grows to its ceiling at F = 4 — but
+`image_workers` does not dissolve, it becomes concurrent group calls.
 
 **MSVC inlining** of the recursion is an assumption; check the measurement, not
 the intent. The 19-point octree bookkeeping is the fiddly part.

From 83d2622cfcee8c00600c71aa4c049430dd529bb9 Mon Sep 17 00:00:00 2001
From: "T. Fuchs" <tfuchs@cornell.edu>
Date: Mon, 10 Aug 2026 19:39:40 -0400
Subject: [PATCH 15/33] feat(phys): choose the frame group size by measuring
 the job

frames_per_group now defaults to measuring the job, picking four frames
per call at 0.88x mapping time. Chosen on throughput, since density
alone picks the slowest configuration.

Turns grouping on by default, so contributions associate differently.
---
 orgui/datautils/xrayutils/reconstruction.py   |  10 +-
 .../xrayutils/test/test_reconstruction_job.py | 135 +++++++++++
 orgui/reconstruction_job.py                   | 215 ++++++++++++++++--
 3 files changed, 341 insertions(+), 19 deletions(-)

diff --git a/orgui/datautils/xrayutils/reconstruction.py b/orgui/datautils/xrayutils/reconstruction.py
index 1fbeeb4..bd316cc 100644
--- a/orgui/datautils/xrayutils/reconstruction.py
+++ b/orgui/datautils/xrayutils/reconstruction.py
@@ -193,8 +193,10 @@ class _ReconstructionSpec:
     #: frame) collapses redundancy a per-image block cannot see. One frame
     #: per call is the degenerate case of the same machinery, not a
     #: separate path. Past roughly one voxel of travel per frame the gain
-    #: is gone, so this is a per-job quantity rather than a constant.
-    frames_per_group: int = 1
+    #: is gone, so this is a per-job quantity rather than a constant --
+    #: ``None`` measures the job and chooses (the normal case), an
+    #: explicit integer overrides that measurement.
+    frames_per_group: int | None = None
     memory_budget_bytes: int = 512 * 1024 * 1024
     checkpoint_count: int = 10
     compression: str = "bitshuffle-lz4"
@@ -218,8 +220,8 @@ def __post_init__(self):
             raise ValueError("threads must be positive")
         if self.work_block_pixels < 1:
             raise ValueError("work_block_pixels must be positive")
-        if self.frames_per_group < 1:
-            raise ValueError("frames_per_group must be positive")
+        if self.frames_per_group is not None and self.frames_per_group < 1:
+            raise ValueError("frames_per_group must be positive when set")
         if self.memory_budget_bytes < 1024 * 1024:
             raise ValueError("memory_budget_bytes must be at least 1 MiB")
         if self.checkpoint_count < 1:
diff --git a/orgui/datautils/xrayutils/test/test_reconstruction_job.py b/orgui/datautils/xrayutils/test/test_reconstruction_job.py
index c70f1fe..00bb037 100644
--- a/orgui/datautils/xrayutils/test/test_reconstruction_job.py
+++ b/orgui/datautils/xrayutils/test/test_reconstruction_job.py
@@ -25,11 +25,13 @@
     WORK_BLOCK_PRESETS,
     _FRAME_RECORD_BYTES_PER_PIXEL,
     _PYTHON_CORRECTION_BYTES_PER_PIXEL,
+    _GROUP_SIZE_CANDIDATES,
     _RESERVED_RECORDS_PER_PIXEL,
     ReconstructionGrid,
     ReconstructionJob,
     _angles_advance_monotonically,
     _frame_groups,
+    _choose_frames_per_group,
     _frame_parallelism,
     _group_pipeline_layout,
     _node_checkpoint_plan,
@@ -1117,3 +1119,136 @@ def test_group_pipeline_layout_never_returns_zero_workers_on_a_tiny_budget():
 
     assert (workers, threads) == (1, 8)
     assert depth > workers
+
+
+def _layout_spec(frames_per_group=None, threads=24):
+    grid = _GridSpec(
+        minimum=(0.0, 0.0, 0.0),
+        maximum=(1.0, 1.0, 1.0),
+        step=(0.1, 0.1, 0.1),
+        frame="lab",
+        chunk_shape=(8, 8, 8),
+    )
+    return _ReconstructionSpec(
+        grids=(grid,),
+        max_depth=0,
+        threads=threads,
+        frames_per_group=frames_per_group,
+    )
+
+
+def test_choose_frames_per_group_stops_where_frames_stop_overlapping():
+    """Grouping merges frames that reach the same voxels. Once a frame
+    advances a whole voxel they tile instead, and the group buffer costs
+    concurrency for nothing."""
+    tiles = [(0, 256, 0, 1024), (256, 512, 0, 1024)]
+    budget = 8 * 1024**3
+
+    assert (
+        _choose_frames_per_group(_layout_spec(), tiles, budget, 0.7, 4) > 1
+    )
+    assert _choose_frames_per_group(_layout_spec(), tiles, budget, 1.0, 4) == 1
+    assert _choose_frames_per_group(_layout_spec(), tiles, budget, 3.2, 4) == 1
+
+
+def test_choose_frames_per_group_keeps_concurrency_rather_than_density():
+    """Records fall monotonically with the group size, so density alone
+    would pick the largest group memory allows -- which measured *slowest*
+    on the reference job, because the group crowds out concurrent calls.
+    The choice must stop where concurrency does.
+
+    Sized so the floor actually binds inside the candidate list, the way
+    it does on a real detector: a tile small enough for every candidate to
+    fit would test nothing.
+    """
+    tiles = [(0, 1024, 0, 2048)]
+    spec = _layout_spec()
+    budget = 8 * 1024**3
+
+    chosen = _choose_frames_per_group(spec, tiles, budget, 0.7, 4)
+
+    assert 1 < chosen < max(_GROUP_SIZE_CANDIDATES)
+    workers, _threads, _depth = _group_pipeline_layout(
+        dataclasses.replace(spec, frames_per_group=chosen), tiles, budget, 4
+    )
+    assert workers >= 3
+    # The next size up must be the one that broke the floor, or the choice
+    # stopped early and left throughput on the table.
+    larger = _group_pipeline_layout(
+        dataclasses.replace(spec, frames_per_group=chosen * 2), tiles, budget, 4
+    )[0]
+    assert larger < 3
+
+
+def test_choose_frames_per_group_falls_back_to_one_on_a_tight_budget():
+    """A budget that cannot afford concurrent group calls must not group:
+    one call with every thread is the configuration that measured worst."""
+    tiles = [(0, 2048, 0, 2048)]
+
+    assert (
+        _choose_frames_per_group(_layout_spec(), tiles, 512 * 1024**2, 0.5, 4)
+        == 1
+    )
+
+
+def test_frame_advance_voxels_is_zero_when_the_grid_does_not_rotate():
+    """A lab-frame grid is the degenerate case worth pinning: Q in the lab
+    frame does not depend on the sample angles, so every frame reaches the
+    same voxels however far the sample turns. Zero travel is the correct
+    answer, and it is the case grouping helps most."""
+    grid = ReconstructionGrid(
+        minimum=(-20.0, -20.0, -20.0),
+        maximum=(20.0, 20.0, 20.0),
+        step=(0.05, 0.05, 0.05),
+        frame="lab",
+        chunk_shape=(8, 8, 8),
+    )
+    config = _config()
+    spec = _ReconstructionSpec(grids=(_GridSpec(**grid.__dict__),), max_depth=0)
+    kernel = reconstruction_job_module._kernel_for_grid(
+        spec, spec.grids[0], config.ub_calculator, threads=1
+    )
+    rays = reconstruction_job_module._detector_corner_rays(
+        config.detector, (0, 2, 0, 2)
+    )
+    bounds = np.zeros((3, 2, 4), dtype=np.float64)
+    bounds[:, 0, 1] = np.deg2rad(5.0) * np.arange(3)
+    bounds[:, 1, 1] = bounds[:, 0, 1]
+
+    assert reconstruction_job_module._frame_advance_voxels(
+        kernel, spec.grids[0], rays, bounds, [(0, 1)]
+    ) == 0.0
+
+
+def test_frame_advance_voxels_scales_with_the_angular_step():
+    """The probe has to measure the job, so a scan that turns twice as far
+    per frame must read as twice the travel."""
+    grid = ReconstructionGrid(
+        minimum=(-1.0, -1.0, -1.0),
+        maximum=(1.0, 1.0, 1.0),
+        step=(0.002, 0.002, 0.002),
+        frame="hkl",
+        chunk_shape=(8, 8, 8),
+    )
+    config = _config()
+    spec = _ReconstructionSpec(grids=(_GridSpec(**grid.__dict__),), max_depth=0)
+    kernel = reconstruction_job_module._kernel_for_grid(
+        spec, spec.grids[0], config.ub_calculator, threads=1
+    )
+    rays = reconstruction_job_module._detector_corner_rays(
+        config.detector, (0, 2, 0, 2)
+    )
+
+    def advance(step_degrees):
+        bounds = np.zeros((3, 2, 4), dtype=np.float64)
+        angles = np.deg2rad(step_degrees) * np.arange(3)
+        bounds[:, 0, 1] = angles
+        bounds[:, 1, 1] = angles
+        return reconstruction_job_module._frame_advance_voxels(
+            kernel, spec.grids[0], rays, bounds, [(0, 1)]
+        )
+
+    small = advance(0.1)
+    large = advance(0.2)
+    assert small > 0.0
+    assert large == pytest.approx(2.0 * small, rel=0.05)
diff --git a/orgui/reconstruction_job.py b/orgui/reconstruction_job.py
index 9681b8d..230e3ff 100644
--- a/orgui/reconstruction_job.py
+++ b/orgui/reconstruction_job.py
@@ -31,6 +31,7 @@
     _discover_checkpoint_state,
     _files_per_job,
     _finalize_reconstruction,
+    _kernel_for_grid,
     _kernel_threads_sweep,
     _map_frame_group,
     _tile_ray_arrays,
@@ -1523,7 +1524,7 @@ def _frame_parallelism(
     # the group size exactly as the tile size, and this side must scale
     # with it too or the two would disagree in the one direction that
     # matters (this side permissive, the kernel throwing).
-    frames_per_group = max(1, int(getattr(spec, "frames_per_group", 1)))
+    frames_per_group = _frames_per_group(spec)
     native_memory = (
         largest_tile_pixels * frames_per_group * native_bytes_per_pixel
     )
@@ -2202,6 +2203,36 @@ def _kernel_threads_candidates(total_threads, include=()):
     return sorted(candidates)
 
 
+_GROUP_SIZE_CANDIDATES = (1, 2, 4, 8, 16)
+"""Group sizes the automatic choice considers."""
+_GROUP_ADVANCE_VOXEL_LIMIT = 1.0
+"""Per-frame reciprocal-space travel, in voxels, past which grouping stops
+paying.
+
+Frame grouping merges frames that reach the same voxels. Measured over a
+striding sweep of the reference scan, the gain is already gone by the time
+a frame advances a whole voxel -- past that, consecutive frames tile
+rather than overlap, and a group buffer buys nothing while still costing
+concurrency. This is a statement about grid step against angular step: a
+finer grid moves a job into the regime, a coarser scan moves it out."""
+_GROUP_MINIMUM_CONCURRENT_CALLS = 3
+"""Concurrent native calls a group size has to leave affordable.
+
+Purely empirical, and the reason the automatic choice is not simply "the
+largest group that fits". On the reference job, four frames per group with
+three concurrent calls mapped at 0.866x the per-frame pipeline; eight with
+two was 1.014x, and eight with one was 1.24x. Records keep falling as the
+group grows, so density alone would pick the losing configuration -- what
+turns over is the concurrency the group buffer and its native working set
+crowd out."""
+
+
+def _frames_per_group(spec):
+    """Resolved group size for ``spec``: ``None`` means one frame."""
+    value = getattr(spec, "frames_per_group", None)
+    return 1 if value is None else max(1, int(value))
+
+
 _GROUP_PIPELINE_MINIMUM_READAHEAD = 1
 """Groups prepared *beyond* the ones being mapped.
 
@@ -2284,7 +2315,7 @@ def _group_pipeline_layout(spec, tiles, memory_bytes, prepare_workers):
             1 + _GROUP_PIPELINE_MINIMUM_READAHEAD,
         )
     detector_pixels = sum(tile_pixels)
-    frames_per_group = max(1, int(getattr(spec, "frames_per_group", 1)))
+    frames_per_group = _frames_per_group(spec)
     native_bytes_per_pixel = (
         128 + 2 * _RESERVED_RECORDS_PER_PIXEL * _CHECKPOINT_BYTES_PER_ROW
         if spec.max_depth > 0
@@ -2362,6 +2393,118 @@ def _angles_advance_monotonically(bounds, frame_indices):
     return True
 
 
+def _frame_advance_voxels(
+    kernel, grid, corner_rays, bounds, frame_pairs, *, samples_per_axis=8
+):
+    """Median distance a pixel travels between adjacent frames, in voxels.
+
+    The quantity finding A rests on, measured on the job rather than
+    inferred from its angular step: two frames merge usefully only while
+    they land within about a voxel of each other. Geometry and the grid
+    decide it, so this needs no image data and does no I/O -- it asks the
+    kernel where a pixel lands on one frame and on the next, and takes the
+    difference in units of the grid step.
+
+    A median rather than a mean: the distribution across a detector is
+    wide (measured p10 0.22, p90 1.48 voxels on the reference job), and
+    the tails are the corners rather than the bulk of the frame.
+
+    :param kernel:
+        Constructed ``ReconstructionKernel`` for ``grid``.
+    :param corner_rays:
+        Corner rays for one detector tile, ``(rows + 1, columns + 1, 3)``.
+        Pixels are sampled inside that tile.
+    :param bounds:
+        ``(len(scan), 2, 4)`` exposure angle bounds in radians.
+    :param frame_pairs:
+        ``(first, second)`` adjacent frame indices to sample across.
+    :returns:
+        Median displacement in voxels; ``0.0`` when nothing was sampled.
+    :rtype: float
+    """
+    rows = int(corner_rays.shape[0]) - 1
+    columns = int(corner_rays.shape[1]) - 1
+    if rows < 1 or columns < 1:
+        return 0.0
+    step = np.asarray(grid.step, dtype=np.float64)
+    row_samples = np.unique(
+        np.linspace(0, rows - 1, min(samples_per_axis, rows)).astype(int)
+    )
+    column_samples = np.unique(
+        np.linspace(0, columns - 1, min(samples_per_axis, columns)).astype(int)
+    )
+    distances = []
+    for first, second in frame_pairs:
+        angles = [
+            (
+                np.ascontiguousarray(bounds[index, 0]),
+                np.ascontiguousarray(bounds[index, 1]),
+            )
+            for index in (first, second)
+        ]
+        for row in row_samples:
+            for column in column_samples:
+                positions = [
+                    np.asarray(
+                        kernel.coordinate(
+                            corner_rays, start, end, int(row), int(column)
+                        )
+                    )
+                    for start, end in angles
+                ]
+                distances.append(
+                    float(
+                        np.linalg.norm((positions[1] - positions[0]) / step)
+                    )
+                )
+    return float(np.median(distances)) if distances else 0.0
+
+
+def _choose_frames_per_group(
+    spec,
+    tiles,
+    memory_bytes,
+    advance_voxels,
+    prepare_workers,
+    candidates=_GROUP_SIZE_CANDIDATES,
+):
+    """Largest group size that is both in the regime and affordable.
+
+    Two independent gates, and the second is the one that is easy to get
+    wrong. Records emitted fall monotonically as the group grows, so
+    choosing on record density alone picks the largest group that fits
+    memory -- which on the reference job is the configuration that maps
+    *slowest*, because the group buffer and the call's native working set
+    crowd out the concurrency that was paying for itself. So the rule is
+    the largest group that still leaves
+    ``_GROUP_MINIMUM_CONCURRENT_CALLS`` native calls affordable, not the
+    largest group that fits.
+
+    :param float advance_voxels:
+        Per-frame reciprocal-space travel from
+        :func:`_frame_advance_voxels`.
+    :returns:
+        Frames per group, at least 1.
+    :rtype: int
+    """
+    if advance_voxels >= _GROUP_ADVANCE_VOXEL_LIMIT:
+        return 1
+    chosen = 1
+    for candidate in sorted(int(value) for value in candidates):
+        if candidate <= 1:
+            continue
+        workers, _threads, _depth = _group_pipeline_layout(
+            replace(spec, frames_per_group=candidate),
+            tiles,
+            memory_bytes,
+            prepare_workers,
+        )
+        if workers < _GROUP_MINIMUM_CONCURRENT_CALLS:
+            break
+        chosen = candidate
+    return chosen
+
+
 def _frame_groups(frame_range, router, grid_names, frames_per_group):
     """Split one scheduling range into the groups mapped in one call each.
 
@@ -2784,13 +2927,11 @@ def _map_pending_ranges(
     ]
     # Grouping rests on frames adjacent in index being adjacent in angle.
     # Where they are not -- an interlaced scan -- the honest answer is one
-    # frame per call, decided here rather than assumed anywhere below.
-    frames_per_group = max(1, int(getattr(spec, "frames_per_group", 1)))
-    if frames_per_group > 1 and not _angles_advance_monotonically(
-        bounds, frame_indices
-    ):
-        frames_per_group = 1
-    spec = replace(spec, frames_per_group=frames_per_group)
+    # frame per call, whatever else the job asked for. Cheap, and it gates
+    # both the requested and the measured group size, so it is settled
+    # here; the rest of the choice needs the ray arrays and the memory
+    # split and waits for them.
+    grouping_is_sound = _angles_advance_monotonically(bounds, frame_indices)
 
     ray_cache_bytes = sum(
         (tile[1] - tile[0] + 1)
@@ -2812,6 +2953,56 @@ def _map_pending_ranges(
         1024**2,
         effective_memory - (ray_cache_bytes if cache_detector_rays else 0),
     )
+
+    max_readers = (
+        _PREFETCH_POOL_MAX
+        if getattr(scan, "supports_concurrent_read", True)
+        else 1
+    )
+    seed_prepare_workers = min(_PREFETCH_POOL_INITIAL, max_readers)
+    requested_frames_per_group = getattr(spec, "frames_per_group", None)
+    if not grouping_is_sound:
+        frames_per_group = 1
+    elif requested_frames_per_group is not None:
+        frames_per_group = max(1, int(requested_frames_per_group))
+    elif len(frame_indices) < 2:
+        frames_per_group = 1
+    else:
+        # Sample the travel per frame at a few places in the scan, not
+        # one: a scan with a non-uniform angular step is the mild version
+        # of the interlaced problem, and one pair at the start would miss
+        # it entirely.
+        last_pair = len(frame_indices) - 2
+        probe_pairs = [
+            (frame_indices[position], frame_indices[position + 1])
+            for position in sorted(
+                {0, min(last_pair, len(frame_indices) // 2), last_pair}
+            )
+            # Skip a pair that straddles an excluded frame: its travel is
+            # two frames' worth and would read as out of the regime.
+            if frame_indices[position] + 1 == frame_indices[position + 1]
+        ]
+        frames_per_group = (
+            _choose_frames_per_group(
+                spec,
+                tiles,
+                scheduler_memory,
+                _frame_advance_voxels(
+                    _kernel_for_grid(
+                        spec, spec.grids[0], config.ub_calculator, threads=1
+                    ),
+                    spec.grids[0],
+                    ray_arrays[detector_tiles[0]],
+                    bounds,
+                    probe_pairs,
+                ),
+                seed_prepare_workers,
+            )
+            if probe_pairs
+            else 1
+        )
+    spec = replace(spec, frames_per_group=frames_per_group)
+
     def layout_for(threads_per_image_seed):
         """Worker count and native threads this seed would run with."""
         workers = []
@@ -2865,13 +3056,7 @@ def layout_for(threads_per_image_seed):
         )
     ]
 
-    max_readers = (
-        _PREFETCH_POOL_MAX
-        if getattr(scan, "supports_concurrent_read", True)
-        else 1
-    )
     if frames_per_group > 1:
-        seed_prepare_workers = min(_PREFETCH_POOL_INITIAL, max_readers)
         group_workers, group_threads, group_depth = _group_pipeline_layout(
             spec, tiles, scheduler_memory, seed_prepare_workers
         )

From c819527adb17f1509861106a5f9161955f100830 Mon Sep 17 00:00:00 2001
From: "T. Fuchs" <tfuchs@cornell.edu>
Date: Mon, 10 Aug 2026 19:39:51 -0400
Subject: [PATCH 16/33] test: let the pipeline benchmark exercise automatic
 group sizing

--group 1 is now how the per-frame arm is requested explicitly.
---
 benchmarks/benchmark_reconstruction_pipeline.py | 13 ++++++++-----
 1 file changed, 8 insertions(+), 5 deletions(-)

diff --git a/benchmarks/benchmark_reconstruction_pipeline.py b/benchmarks/benchmark_reconstruction_pipeline.py
index 29ea9e3..e457294 100644
--- a/benchmarks/benchmark_reconstruction_pipeline.py
+++ b/benchmarks/benchmark_reconstruction_pipeline.py
@@ -74,8 +74,11 @@ def _arguments():
     parser.add_argument(
         "--group",
         type=int,
-        default=1,
-        help="Frames per kernel call (spec.frames_per_group).",
+        default=None,
+        help=(
+            "Frames per kernel call. Omitted leaves spec.frames_per_group "
+            "unset, so the scheduler measures the job and chooses."
+        ),
     )
     parser.add_argument(
         "--threads-per-image",
@@ -230,7 +233,7 @@ def main():
     spec = job.internal_spec()
     if arguments.depth is not None:
         spec = replace(spec, max_depth=arguments.depth)
-    if arguments.group != 1:
+    if arguments.group is not None:
         if not hasattr(spec, "frames_per_group"):
             raise SystemExit(
                 "This checkout has no spec.frames_per_group; --group only "
@@ -317,7 +320,7 @@ def counting_route(grid_name, frame_index, batch, **kwargs):
             (row_stop - row_start) * (column_stop - column_start)
             for row_start, row_stop, column_start, column_stop in tiles
         ),
-        "frames_per_group": getattr(spec, "frames_per_group", 1),
+        "frames_per_group": getattr(spec, "frames_per_group", None),
         "max_depth": spec.max_depth,
         "work_block_pixels": spec.work_block_pixels,
         "threads": spec.threads,
@@ -332,7 +335,7 @@ def counting_route(grid_name, frame_index, batch, **kwargs):
         "worker_memory_mb": round(worker_bytes / 1e6, 1),
         "group_layout": (
             list(_group_pipeline_layout(spec, tiles, effective_memory, 4))
-            if getattr(spec, "frames_per_group", 1) > 1
+            if (getattr(spec, "frames_per_group", None) or 1) > 1
             else None
         ),
         "native": _native_module().__file__,

From 15245947acb37a48df864338ca29e5d9da1fb1a7 Mon Sep 17 00:00:00 2001
From: "T. Fuchs" <tfuchs@cornell.edu>
Date: Mon, 10 Aug 2026 19:39:51 -0400
Subject: [PATCH 17/33] docs: record automatic group sizing, completing phase 3

The measured choice reproduces step 5's hand sweep: four frames per call
at 0.88x the per-frame pipeline. Band height is still taken as given.
---
 CHANGELOG.md                                  | 10 ++-
 ..._space_mapping_locality_and_subdivision.md | 72 ++++++++++++++-----
 2 files changed, 63 insertions(+), 19 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 5b9d88c..9003407 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -85,8 +85,14 @@ Scientific correctness and performance fixes:
   GIL-held Python work overlaps the next call's native work instead of
   queueing behind it. Measured 0.87x the mapping wall time of the
   per-image pipeline at four images per call on a 3651-frame rotation scan.
-  Group size is not yet chosen automatically, so this path is only taken
-  when a group size is set explicitly.
+- The number of images per call is now measured per job rather than
+  configured. Grouping only helps while consecutive images land within about
+  one voxel of each other, so the job's own geometry is probed at three
+  points in the scan to decide whether it does, and the group size is then
+  the largest the memory budget can afford without giving up concurrency.
+  Scans whose angles do not advance monotonically, such as interlaced ones,
+  map one image per call as before. On a 0.1 deg/frame rotation scan this
+  chooses four images per call and maps at 0.88x the previous wall time.
 
 A ***critical bug*** was fixed that affects bulk CTR calculations:
 
diff --git a/doc/design/reciprocal_space_mapping_locality_and_subdivision.md b/doc/design/reciprocal_space_mapping_locality_and_subdivision.md
index 1efb160..1dcb0dc 100644
--- a/doc/design/reciprocal_space_mapping_locality_and_subdivision.md
+++ b/doc/design/reciprocal_space_mapping_locality_and_subdivision.md
@@ -1,11 +1,10 @@
 # Reciprocal-space mapping: locality, subdivision cost, and the scheduling that would carry them
 
-> **Status: partly implemented.** Phases 1 and 2 have landed, and so has
-> phase 3 except for choosing the group size per job (step 6); phase 4 has
-> not. Each step below says which it is and what it measured. Everything
-> here was measured against real beamtime data, and where a prediction was
-> made before implementing, the step records whether it held — including
-> where it did not.
+> **Status: phases 1-3 implemented, phase 4 not.** Frame grouping is on by
+> default and chooses its own group size per job. Each step below says which
+> it is and what it measured. Everything here was measured against real
+> beamtime data, and where a prediction was made before implementing, the
+> step records whether it held — including where it did not.
 
 Third companion to `reciprocal_space_scratch_architecture.md` (what the pipeline
 *is*) and `reciprocal_space_performance_findings.md` (what has been measured
@@ -16,10 +15,10 @@ to do to carry them.
 - **A — cross-frame locality.** The native work block deduplicates voxels within
   one image, mostly by accident. Extending it across adjacent images cuts
   emitted records by ~42% and *shrinks* the cache working set. **The kernel and
-  the pipeline wiring and the scheduler are implemented.** Records through the
-  router fall to 0.598x at F = 8, matching the geometric prediction, and
-  end-to-end mapping is **0.866x at F = 4** on the real job. F is not yet
-  chosen per job, so grouping stays opt-in. See phase 3.
+  the pipeline wiring, the scheduler and the per-job group size are all
+  implemented.** Records through the router fall to 0.657x at the group size
+  chosen for the reference job, and end-to-end mapping to **0.88x**. See
+  phase 3.
 - **B — subdivision cost.** Above depth 3 the shared-corner cache switched off
   and the kernel did several times more ray work than it needed to. A
   corner-passing recursion removes that, bit-for-bit. **Phase 1 is implemented**
@@ -727,13 +726,52 @@ Not required by either finding; do not fold it in.
    other for the same memory**, which is a knob this document had not
    identified, and picking the pair rather than picking F alone is what
    step 6 should actually do.
-6. *Choosing F per job.* **Not done.** The probe should now choose a
-   (band height, F) pair rather than F alone, and it should optimise
-   throughput rather than record density: records keep falling past the
-   point where throughput turns around, so density is the wrong objective
-   on its own. `F = 4` is a reasonable default for a 0.1 deg/frame scan at
-   depth 0 on a 24-thread machine, and nothing weaker than a measurement
-   should be trusted to generalise that.
+6. *Choosing F per job.* **Done.** `spec.frames_per_group` now defaults to
+   `None`, meaning "measure this job and choose"; an explicit integer still
+   overrides. Two gates, and the second is the one that is easy to get
+   wrong.
+
+   **The regime gate.** `_frame_advance_voxels` asks the kernel where a
+   sample of pixels lands on one frame and on the next, and takes the
+   difference in units of the grid step. Geometry only — no image data and
+   no I/O — so it costs a few hundred `coordinate()` calls at job start.
+   Past `_GROUP_ADVANCE_VOXEL_LIMIT` (one voxel) consecutive frames tile
+   rather than overlap and the answer is `F = 1`. Sampled at three places
+   in the scan, not one, so a non-uniform angular step is caught. The
+   monotonicity check for interlaced scans gates both this and any
+   explicit request.
+
+   **The concurrency gate, which is what density alone would get wrong.**
+   Records fall monotonically with F, so choosing on density picks the
+   largest group that fits memory — which on this job is the configuration
+   that maps *slowest*. The rule is instead the largest F that still leaves
+   `_GROUP_MINIMUM_CONCURRENT_CALLS` (3) native calls affordable.
+
+   On the reference job this picks **F = 4 with 3 concurrent calls of 8
+   threads**, on every run, which is the configuration the sweep in step 5
+   found by hand. Interleaved against an explicit `F = 1`, three pairs:
+
+   | pair | automatic | F = 1 | ratio |
+   |---|---|---|---|
+   | 1 | 262.3 | 311.2 | 0.843 |
+   | 2 | 223.4 | 255.6 | 0.874 |
+   | 3 | 250.3 | 268.5 | 0.932 |
+
+   **0.88x**, with records at 0.657x. Note both arms are slower in absolute
+   terms than the step 5 table taken a day earlier (278 ms/frame for `F = 1`
+   against 233); the machine drifts between sessions, and only the paired
+   ratio is stable. That is the whole reason the arms are interleaved.
+
+   One case worth knowing about, pinned by a test: a **lab-frame grid**
+   measures zero travel, because Q in the lab frame does not depend on the
+   sample angles at all. Every frame reaches the same voxels, so grouping
+   helps most there and the gate correctly does not stand in the way.
+
+   What is *not* done: choosing the band height. Step 5 showed band height
+   and group size trade against each other for the same memory, and the
+   choice here takes the tiling as given. Re-tiling to afford a larger
+   group is the obvious next thing to try, and it is what would let `F = 8`
+   compete.
 
 ### Measurement noise on this machine
 

From 055a2500652dcc53017b75a10057fc4940c97128 Mon Sep 17 00:00:00 2001
From: "T. Fuchs" <tfuchs@cornell.edu>
Date: Mon, 10 Aug 2026 20:18:22 -0400
Subject: [PATCH 18/33] fix: bound the detector band height by the record
 ceiling, not 8**depth

Tile planning was the third site still sizing from worst-case leaves,
collapsing a band to one row at depth 5: 2527 native calls per image
instead of six. Throughput is a wash, so this is for correctness.
---
 .../xrayutils/test/test_reconstruction_job.py | 65 +++++++++++++++++++
 orgui/reconstruction_job.py                   | 19 +++++-
 2 files changed, 82 insertions(+), 2 deletions(-)

diff --git a/orgui/datautils/xrayutils/test/test_reconstruction_job.py b/orgui/datautils/xrayutils/test/test_reconstruction_job.py
index 00bb037..57a7134 100644
--- a/orgui/datautils/xrayutils/test/test_reconstruction_job.py
+++ b/orgui/datautils/xrayutils/test/test_reconstruction_job.py
@@ -1252,3 +1252,68 @@ def advance(step_degrees):
     large = advance(0.2)
     assert small > 0.0
     assert large == pytest.approx(2.0 * small, rel=0.05)
+
+
+def _large_detector_job(tmp_path, depth):
+    scan = SimulationScan((2, 2), 0.0, 1.0, 8)
+    assets = tmp_path / f"scratch{depth}" / "job-assets.nxs"
+    assets.parent.mkdir(parents=True, exist_ok=True)
+    with h5py.File(assets, "w") as h5file:
+        h5file.attrs["orgui_job_assets"] = 1
+    config = _config()
+    config.detector.detector.shape = (2527, 2463)
+    config.detector.detector.max_shape = (2527, 2463)
+    grid = ReconstructionGrid(
+        minimum=(-20.0, -20.0, -20.0),
+        maximum=(20.0, 20.0, 20.0),
+        step=(20.0, 20.0, 20.0),
+        frame="lab",
+        chunk_shape=(2, 2, 2),
+    )
+    scan_reference = ScanReference.from_scan(scan).to_dict()
+    job = ReconstructionJob(
+        config=config_data_to_json(config),
+        scan_reference=scan_reference,
+        grids=[grid.__dict__],
+        scratch_path=str(assets.parent),
+        output_path=str(tmp_path / f"result{depth}.h5"),
+        compression="Raw",
+        assets_path=str(assets),
+        assets_sha256=sha256(assets.read_bytes()).hexdigest(),
+        source_fingerprint_sha256="0" * 64,
+        threads_per_image=None,
+        accumulation_budget_bytes=None,
+        advanced_depth=depth,
+        runtime_threads=24,
+        runtime_memory_bytes=10 * 1000**3,
+        frame_batch=4,
+    )
+    return scan, job, config
+
+
+def test_detector_bands_do_not_collapse_at_high_adaptive_depth(tmp_path):
+    """Band height is bounded by the record ceiling, not by the worst-case
+    leaf count.
+
+    ``8**depth`` bytes per pixel is what this site was left with when the
+    two memory prechecks were corrected to the record ceiling. It is
+    enormously conservative: at depth 5 it asks for 2.6 MB per pixel,
+    which bands a Pilatus 6M into one row per band -- 2527 native calls
+    per frame instead of six. Thin bands are not free, and one row is not
+    a band at all.
+    """
+    band_counts = {}
+    for depth in (0, 2, 5):
+        scan, job, config = _large_detector_job(tmp_path, depth)
+        _ranges, tiles = reconstruction_job_module._execution_layout(
+            job, scan, config
+        )
+        band_counts[depth] = len(tiles)
+        # Full detector width, so a tile is a contiguous slice of the frame.
+        assert all(
+            (column_start, column_stop) == (0, 2463)
+            for _rs, _re, column_start, column_stop in tiles
+        )
+
+    assert band_counts[0] == band_counts[2] == band_counts[5]
+    assert band_counts[5] < 20
diff --git a/orgui/reconstruction_job.py b/orgui/reconstruction_job.py
index 230e3ff..cbb567f 100644
--- a/orgui/reconstruction_job.py
+++ b/orgui/reconstruction_job.py
@@ -1346,9 +1346,24 @@ def _execution_layout(job, scan, config, *, extra_excluded_frames=()):
         if job.advanced_depth is not None
         else ACCURACY_DEPTHS[job.accuracy]
     )
-    worst_leaves = 8**depth
+    # Bounded by the record ceiling the arena and both memory prechecks
+    # already use, not by the worst-case leaf count. 8**depth is what this
+    # site was left with when the prechecks were corrected, and it is far
+    # more conservative than anything a pixel actually leaves behind: at
+    # depth 2 it claims 5248 bytes per pixel against a real ~106, which
+    # bands this detector into 13 strips of 194 rows, and by depth 5 it
+    # asks for 2.6 MB per pixel and collapses a band to a single row.
+    #
+    # Thin bands are not free. Measured at depth 0 on the reference job,
+    # banding finer than the budget requires costs 10-15% -- more kernel
+    # calls per frame group, each with less work to spread over its
+    # threads, and a brick that is shorter in the row direction than the
+    # locality it is trying to exploit.
     estimated_native_bytes_per_pixel = (
-        128 + 2 * worst_leaves * _CHECKPOINT_BYTES_PER_ROW
+        128
+        + 2
+        * min(8**depth, _RESERVED_RECORDS_PER_PIXEL)
+        * _CHECKPOINT_BYTES_PER_ROW
     )
     tile_pixels = max(
         1,

From d1fa02b52a829edaa6dec78404a6b2956275c282 Mon Sep 17 00:00:00 2001
From: "T. Fuchs" <tfuchs@cornell.edu>
Date: Mon, 10 Aug 2026 20:18:23 -0400
Subject: [PATCH 19/33] docs: record that band height has nothing left to
 choose

Swept at four frames per call the surface is flat, so the pair needs no
joint search -- the default already sits in the region that matters.
---
 CHANGELOG.md                                  |  9 ++++
 ..._space_mapping_locality_and_subdivision.md | 52 +++++++++++++++++--
 2 files changed, 56 insertions(+), 5 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 9003407..7b779f0 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -93,6 +93,15 @@ Scientific correctness and performance fixes:
   Scans whose angles do not advance monotonically, such as interlaced ones,
   map one image per call as before. On a 0.1 deg/frame rotation scan this
   chooses four images per call and maps at 0.88x the previous wall time.
+- Corrected the detector band height used for reciprocal-space mapping,
+  which was still derived from the worst-case adaptive leaf count rather
+  than from the record ceiling the memory prechecks use. At the
+  ``very_high`` and ``maximum`` accuracy settings this over-estimated a
+  pixel's cost by four orders of magnitude and split the detector into
+  extremely thin bands — one row per band on a Pilatus 6M at ``maximum``,
+  giving 2527 native calls per image instead of six. Band height no longer
+  depends on the accuracy setting. Measured throughput at ``balanced``
+  accuracy is unchanged; the fix matters at the two highest settings.
 
 A ***critical bug*** was fixed that affects bulk CTR calculations:
 
diff --git a/doc/design/reciprocal_space_mapping_locality_and_subdivision.md b/doc/design/reciprocal_space_mapping_locality_and_subdivision.md
index 1dcb0dc..1155984 100644
--- a/doc/design/reciprocal_space_mapping_locality_and_subdivision.md
+++ b/doc/design/reciprocal_space_mapping_locality_and_subdivision.md
@@ -767,11 +767,53 @@ Not required by either finding; do not fold it in.
    sample angles at all. Every frame reaches the same voxels, so grouping
    helps most there and the gate correctly does not stand in the way.
 
-   What is *not* done: choosing the band height. Step 5 showed band height
-   and group size trade against each other for the same memory, and the
-   choice here takes the tiling as given. Re-tiling to afford a larger
-   group is the obvious next thing to try, and it is what would let `F = 8`
-   compete.
+7. *Choosing the band height too.* **Measured, and there is nothing to
+   choose.** Step 5 showed band height and group size trade against each
+   other for the same memory, so re-tiling to afford a larger group looked
+   like the obvious next lever. It is not: swept at `F = 4` on the
+   reference job, band height is flat across the region that matters and
+   the existing default sits in the middle of it.
+
+   | bands | tiles | layout | median ms/frame |
+   |---|---|---|---|
+   | 240 | 11 | 4 x 6 | 236.7 |
+   | **421 (the default)** | 7 | **3 x 8** | **205.4** |
+   | 506 | 5 | 3 x 8 | 207.1 |
+   | 600 | 4 | 2 x 12 | 237.1 |
+
+   Both directions lose, and for different reasons. Finer bands buy a
+   fourth concurrent call but take two threads off each, and 4 x 6 is
+   worse than 3 x 8 — more calls, each with less work to spread. Coarser
+   bands cost a call outright. **`F = 8` cannot be rescued this way at
+   all**: its group buffer does not shrink with the band height, so it
+   holds at two concurrent calls from 200 rows to 2527 and never reaches
+   the floor of three.
+
+   So the pair does not need choosing jointly; the band height that
+   maximises concurrency at the chosen `F` is already what the memory
+   budget produces.
+
+   **One real defect surfaced while sweeping it, and is fixed.** Tile
+   planning still sized bands by `8**depth` bytes per pixel — the
+   worst-case leaf count phase 2 replaced in both memory prechecks and
+   left behind at this third site. It is wildly conservative: at depth 2
+   it claims 5248 bytes for a pixel that costs about 106, banding this
+   detector into 13 strips of 194 rows, and at depth 5 it asks for 2.6 MB
+   per pixel and collapses a band to **one row — 2527 native calls per
+   frame instead of six**. Bounding it by the same record ceiling the
+   arena and both prechecks use makes the band height depth-independent.
+
+   Measured at depth 2, interleaved, the fix is a **wash** (619 against
+   629 ms/frame, 0.98x) — the 5-20% predicted from the depth-0 sweep did
+   not appear. The reason is visible in the run: wider bands make the
+   group-size gate drop `F` from 4 to 2, and the two effects cancel almost
+   exactly. That is worth stating as a result in its own right — **band
+   height and group size substitute for each other**, which is why neither
+   is worth optimising hard, and why the joint search this step set out to
+   build would have found a flat surface.
+
+   The fix is kept for the depth-5 pathology and for consistency with
+   phase 2, not for throughput. A test pins the band count against depth.
 
 ### Measurement noise on this machine
 

From 50fec6ae873584df9d7c296aa24f372c22136643 Mon Sep 17 00:00:00 2001
From: "T. Fuchs" <tfuchs@cornell.edu>
Date: Mon, 10 Aug 2026 23:32:25 -0400
Subject: [PATCH 20/33] docs: track the reciprocal-space design records and
 update them

Two design documents were referenced by the tracked locality document
while untracked themselves, so a fresh clone got dangling references.
All three now agree with what has landed.
---
 doc/AGENTS.md                                 |   13 +-
 ..._space_mapping_locality_and_subdivision.md |  202 ++-
 .../reciprocal_space_performance_findings.md  |  299 +++++
 .../reciprocal_space_scratch_architecture.md  | 1094 +++++++++++++++++
 4 files changed, 1576 insertions(+), 32 deletions(-)
 create mode 100644 doc/design/reciprocal_space_performance_findings.md
 create mode 100644 doc/design/reciprocal_space_scratch_architecture.md

diff --git a/doc/AGENTS.md b/doc/AGENTS.md
index f8d30b3..cf2cb55 100644
--- a/doc/AGENTS.md
+++ b/doc/AGENTS.md
@@ -17,6 +17,12 @@ keeps it in sync with releases:
 - `source/conf.py`: Sphinx configuration (`furo` theme, `nbsphinx`, autodoc).
 - `generate_release_notes.py`: extracts hand-written release notes from
   `../CHANGELOG.md` into `source/release_notes.rst`.
+- `design/*.md`: engineering design records, **not** part of the Sphinx
+  toctree and not user-facing. They record why a subsystem is built the way
+  it is, what was measured, and what remains open — including decisions that
+  were tried and reversed. Written for whoever picks the work up next, not
+  for users; a user-facing consequence belongs in `source/*.rst` and
+  `../CHANGELOG.md` as well.
 - `build/`, `_build/`: build output, not source.
 
 Use the repository root `AGENTS.md` together with this file.
@@ -31,8 +37,13 @@ Use the repository root `AGENTS.md` together with this file.
    record of user-facing changes, curated at release time per
    `source/release_procedure.rst`.
 
+3. **Design records** (`design/*.md`): why a subsystem is the way it is, what
+   was measured, what is still open. Not published, not in the toctree.
+
 A given code change may need updates on either track, both, or neither.
-Internal refactors typically need neither.
+Internal refactors typically need neither — but a refactor that invalidates a
+measurement or a decision recorded under `design/` should say so there, and a
+prediction that turned out wrong is worth recording rather than deleting.
 
 ## Updating Topic Pages
 
diff --git a/doc/design/reciprocal_space_mapping_locality_and_subdivision.md b/doc/design/reciprocal_space_mapping_locality_and_subdivision.md
index 1155984..6950d89 100644
--- a/doc/design/reciprocal_space_mapping_locality_and_subdivision.md
+++ b/doc/design/reciprocal_space_mapping_locality_and_subdivision.md
@@ -31,6 +31,35 @@ identical, and it is what makes A viable at depth > 0 at all — see "Where the
 two meet". The measurements in "Finding B" below are the *pre-change* state,
 kept as the record of why the work was done.
 
+### Where this stands
+
+Everything is on branch `perf/subdivision-corner-passing`, unpushed.
+
+| phase | state |
+|---|---|
+| 1 — corner-passing subdivision | done, bit-for-bit, 1.65–2.98x |
+| 2 — the `worst_leaves` memory bound | done |
+| 3 — frame-group bricks at depth 0 | done, steps 1–7; **0.88x end-to-end** |
+| 4 — frame-group bricks above depth 0 | **not started** |
+
+What a reader should know before touching it:
+
+- **Grouping is on by default** and sizes itself per job. On the reference
+  scan it picks four frames per call and maps at 0.88x. Jobs outside the
+  regime (coarse scans, interlaced order, tight memory) get one frame per
+  call and are unaffected.
+- **Output moved.** Contributions merge inside the kernel rather than in
+  the checkpoint accumulator, so sums associate differently — in the last
+  bits, and better conditioned. Voxels reached and contributor counts are
+  unchanged, and tests pin that. This is a `phys` change and it applies to
+  every job, not only grouped ones.
+- **Read "Measuring any of this" before running a benchmark.** Two rounds
+  of conclusions in this document had to be withdrawn for measurement
+  error, not for reasoning error.
+- **The next real question is phase 4's first measurement**, not phase 4's
+  design: `F = 1` against automatic `F` at depth 1 and 2, interleaved. The
+  incidental depth-2 numbers suggest grouping does less there.
+
 ## How this was measured
 
 All numbers are from the `39_1-rsmap` job used throughout the findings document
@@ -368,6 +397,93 @@ Consequences worth stating plainly:
   group rather than degrading smoothly. This is the main scheduling risk, and it
   argues for the smaller F (8) until measured.
 
+## Measuring any of this
+
+Read this before running a benchmark. Most of it was learned by getting a
+measurement wrong first, and two rounds of conclusions had to be withdrawn
+because of it.
+
+### The instruments
+
+| script | measures | typical cost |
+|---|---|---|
+| `benchmark_reconstruction_subdivision.py` | kernel cost and evaluations per pixel by depth and exposure, with XXH3 fingerprints and a `--compare` mode | seconds |
+| `benchmark_reconstruction_group.py` | one `accumulate_group` call against per-image `accumulate`, by group size | seconds |
+| `benchmark_reconstruction_pipeline.py` | the whole mapping phase — reader pool, correction, kernel, router, checkpoint writes | ~1 min per 234-frame arm at depth 0 |
+
+`benchmark_reconstruction_mapping.py` is dead: it imports `_map_frame_range`
+and the Parquet writers, both retired. Do not resurrect it; the pipeline
+benchmark replaces it against the pipeline that exists.
+
+### Two traps that silently invalidate a run
+
+**`PYTHONPATH` must point at the checkout.** `python benchmarks/<script>.py`
+puts `benchmarks/` on `sys.path`, *not* the repository root, so a
+pip-installed `orgui` shadows the working tree and the run measures a
+binary you did not build. Both reconstruction benchmarks print the resolved
+extension path — check it. This has cost a whole measurement round.
+
+**The reference job is `schema_version: 5` and `JOB_SCHEMA_VERSION` has
+moved on**, so `read_job` rejects it. Copy the JSON and bump the field;
+nothing else in the dict changed.
+
+### Run-to-run noise is the hard part
+
+Single runs of the pipeline benchmark vary by up to **30%**. The same
+`F = 4` configuration measured 183.4, 202.0 and 342.4 ms/frame on different
+occasions. Consequences, all of them learned the hard way:
+
+- **Interleave the arms** (A, B, A, B, ...) with two or three repeats.
+  Never one run of each in sequence — ordering effects and drift are
+  confounded with the arm.
+- **Nothing else may run on the machine.** A `pytest` run in another shell
+  is enough to poison a sweep.
+- **Only paired ratios are stable.** Absolute values drift between
+  sessions: `F = 1` measured 233 ms/frame one day and 278 the next, in both
+  arms alike, while the ratio held at 0.87. Never compare a number taken
+  today against one written down yesterday.
+- **Prefer a clean separation over a mean.** The `F = 4` result was
+  trustworthy because every `F = 4` run beat every `F = 1` run, not
+  because the means differed.
+
+Two conclusions drawn from single sequential runs — a "1.47x regression"
+at `F = 8` and a "0.85x win" from finer bands — both dissolved when
+measured properly. Anything in this document from before that was
+understood is flagged where it appears.
+
+Baselines must come from the same checkout, captured immediately before
+the change: `git stash push -- <changed files>`, run, `git stash pop`.
+Never compare across builds.
+
+### Measure the right quantity
+
+**Records entering the router, not rows written to checkpoints.** The
+checkpoint accumulator already merges across frames, so it collapses the
+same redundancy later and more expensively and the saving never appears on
+disk — rows moved 0.97x at `F = 8` where routed records moved 0.598x.
+
+**Do not concatenate checkpoint parts to summarise them.** A 234-frame
+window is ~350 M rows; reading it all at once swapped the machine on top
+of the mapping run's own on-budget ~9.4 GB peak. Stream in row chunks.
+
+**Where bit-for-bit is impossible, pin the contributor-weighted key sum.**
+Frame grouping changes floating-point association by design, so a digest
+moves for legitimate reasons. `sum over rows of h(key) * contributors`,
+accumulated in wrapping `uint64`, is a sum and therefore independent of row
+order and of how the accumulator split its parts. It equals the same sum
+over distinct voxels with their total contributor counts, so it pins both
+the voxel set and the contributor apportionment. Two baseline runs with
+visibly different flush partitions (352.3 M against 350.1 M rows) agreed on
+it exactly, which is what makes it usable as an acceptance criterion.
+
+### Sizing a run
+
+234 frames is six scheduling ranges of the reference job and about a minute
+per arm at depth 0 — enough for three checkpoints to complete and flush.
+Depth 2 is roughly seven times the kernel cost, so use 78 frames there
+(~2 min per arm). A three-repeat, two-arm interleaved comparison is
+therefore 6–15 minutes; budget for it rather than trying to shortcut it.
+
 ## Plan
 
 ### Phase 1 — Corner-passing subdivision (finding B)
@@ -817,14 +933,9 @@ Not required by either finding; do not fold it in.
 
 ### Measurement noise on this machine
 
-Worth recording, because it invalidated a first round of conclusions.
-Single runs of the pipeline benchmark vary by up to **30%** — the same F = 4
-configuration measured 183.4, 202.0 and 342.4 ms/frame on different
-occasions. Comparisons must be **interleaved** (A, B, A, B, ...) and
-repeated, never one run of each in sequence, and nothing else may run on
-the machine at the time. Two conclusions drawn from single sequential runs
-before this was noticed — a "1.47x regression" at F = 8 and a "0.85x win"
-from finer bands — both dissolved when measured properly.
+See "Measuring any of this" above: single runs vary by up to 30%, arms must
+be interleaved, and only paired ratios mean anything. The step 5 table was
+taken that way. Two earlier conclusions were not, and were withdrawn.
 
 ### What the wired pipeline measured
 
@@ -876,13 +987,53 @@ record): the 3.4% block-shape gain is real in block records before the
 cross-block merge, but one call's merged output is the set of voxels that
 frame reached, which does not depend on how the frame was partitioned.
 
-### Phase 4 — Frame-group bricks above depth 0
-
-Only after phases 1 and 3. Pixel-outer / frame-inner, no cross-frame ray cache,
-brick dimensions re-derived against the corrected leaf bound from phase 2. All
-of finding A is measured at depth 0 only; at higher depth a pixel resolves into
-a footprint rather than a point, so overlap should be at least as good, but that
-is an expectation, not a result.
+### Phase 4 — Frame-group bricks above depth 0. Not started.
+
+Everything in phase 3 was designed and measured at depth 0. Grouping is not
+*disabled* above it — the gates run at any depth and the streamed scheduler
+works — but nothing about it has been tuned there, and the little evidence
+that exists is not encouraging.
+
+**What is already known, from the depth-2 runs taken while fixing band
+height.** At depth 2 on the reference job the group-size gate picks
+`F = 2`, not the `F = 4` it picks at depth 0, and mapping runs at
+~620 ms/frame against ~205 at depth 0. The reason `F` falls is structural:
+a group call's native working set is `F x tile pixels x bytes-per-pixel`,
+and the per-pixel term roughly quadruples from depth 0 to depth 2, so the
+same memory affords half the group at the same concurrency. Grouping and
+adaptive depth compete for the same budget.
+
+That was measured incidentally, on two arms that differed in band height as
+well, so treat it as an observation rather than a result. **The first real
+phase 4 measurement is the obvious one nobody has taken: `F = 1` against
+automatic `F` at depth 1 and depth 2, interleaved, three repeats.** Until
+that exists there is no evidence that grouping helps or hurts above depth 0,
+and the honest reading of the numbers above is that it does less.
+
+Only after that is the design work worth doing:
+
+- **Pixel-outer / frame-inner iteration.** Above depth 0 the per-pixel
+  subdivision state cannot be shared across frames (see "Where the two
+  meet"), so the brick loop needs its second mode. Corner-passing is what
+  makes this affordable at all — 1.2–3.5 KiB of stack instead of a
+  6.9–101 KiB dense array per pixel.
+- **No cross-frame ray cache.** The payoff is capped near 28% and the
+  cache is per pixel, so it can only live for one pixel at a time.
+- **Brick dimensions re-derived** against the phase 2 record bound rather
+  than the leaf count.
+- **Whether the record saving even survives.** At higher depth a pixel
+  resolves into a footprint rather than a point, so cross-frame overlap
+  should be at least as good — but "should" is doing all the work in that
+  sentence, and the records-converge-at-depth-3 observation below suggests
+  the marginal voxel is increasingly a partial-volume weight rather than a
+  new voxel. Measure the routed-record ratio at depth 1–3 before assuming
+  finding A transfers.
+
+There is also a prior question worth settling first, because it may make
+the whole of phase 4 moot: **records converge at depth 3** (0.632 → 0.650
+across depths 3–6 while cost rises 3x / 6x / 13x). If `very_high` and
+`maximum` do not change a reconstructed intensity measurably, the right
+answer is to discourage them rather than to optimise them.
 
 ## Risks and open questions
 
@@ -939,19 +1090,8 @@ worth.
 
 ## Also
 
-`benchmarks/benchmark_reconstruction_mapping.py` no longer imports: it wants
-`_map_frame_range` and the Parquet-era writers, both of which predate the
-checkpoint router. `benchmarks/benchmark_reconstruction_pipeline.py` is the
-replacement and measures the pipeline that actually exists.
-
-Read back the checkpoint parts in row chunks, never all at once. A 234-frame
-window is ~350 M rows, and concatenating them costs more than the mapping
-run whose peak is already sized to the job's whole memory budget — the
-first version of the pipeline benchmark swapped the machine doing exactly
-that. The comparable quantity is `sum over rows of h(key) * contributors`
-accumulated in wrapping `uint64`: it is a sum, so it is independent of row
-order and of how the accumulator happened to split its parts, and it pins
-both the voxel set and the contributor apportionment. Two baseline runs
-with visibly different flush partitions (352.3 M against 350.1 M rows)
-agreed on it exactly, which is what makes it usable as an acceptance
-criterion where a digest is not.
+The benchmark mechanics that used to live here — which script to run, the
+`PYTHONPATH` and schema-version traps, the noise discipline, why routed
+records rather than rows on disk, and the contributor-weighted key sum used
+where bit-for-bit is impossible — are now collected under "Measuring any of
+this" above, before the plan rather than after it.
diff --git a/doc/design/reciprocal_space_performance_findings.md b/doc/design/reciprocal_space_performance_findings.md
new file mode 100644
index 0000000..5defb2d
--- /dev/null
+++ b/doc/design/reciprocal_space_performance_findings.md
@@ -0,0 +1,299 @@
+# Reciprocal-space mapping: performance findings and open items
+
+Companion to `reciprocal_space_scratch_architecture.md`, which describes what
+the pipeline *is*. This one records what was measured, what those measurements
+settled, and what is still open.
+
+> **Read `reciprocal_space_mapping_locality_and_subdivision.md` alongside
+> this.** Since these numbers were taken, the subdivision was rewritten
+> (corner-passing, 1.65-2.98x at depth >= 1), the memory prechecks were
+> re-bounded, and mapping now maps several adjacent frames in one native
+> call by default. The timings below are the **per-frame** pipeline and
+> still describe it; where a grouped run differs, it is noted inline. That
+> document also carries the measurement discipline this one's "Measurement
+> traps" section started, and it is the more complete of the two on that
+> subject.
+
+## How to read the numbers here
+
+Everything below was measured on one machine against one dataset:
+
+| | |
+|---|---|
+| CPU | AMD Ryzen AI 9 HX 370, 12 cores / 24 threads |
+| Cache | L1d 48 KiB per core, L2 1 MiB per core, L3 24 MiB total |
+| RAM | 31 GiB |
+| Storage | NVMe, 967 MB/s cold on this dataset (19.4 frames/s) |
+| Dataset | `39_1-rsmap`, La3Ni2O7, CHESS QM2, 2527 x 2463 = 6,224,001 px/frame, 49.8 MB/frame |
+| Job | `center` accuracy (depth 0), 2000^3 hkl grid, 64^3 chunks, 24 threads, 10 GiB budget |
+
+The *reasoning* generalises. The *constants* are calibrated to this machine —
+notably the 16384-pixel work block (an L2-sized working set), the 25%
+accumulator share, and one record per pixel as the per-frame record estimate.
+On a machine with a different L2, or a detector with very different record
+density, they should be re-measured rather than trusted.
+
+## Where the time goes
+
+Per frame, one kernel thread, before this round of work (native grid):
+
+| Stage | ms/frame | Share |
+|---|---|---|
+| load | 48.3 | 4.4% |
+| correct_frame | 42.3 | 3.9% |
+| slice/copy per tile | 24.0 | 2.2% |
+| **kernel** | **853.4** | **78.3%** |
+| reduce (merge tile batches) | 121.2 | 11.1% |
+| **total** | **1089.3** | 0.92 frames/s |
+
+The kernel dominates a single frame, but it releases the GIL; the other ~114 ms
+(10.5%) does not. That is the ceiling on image-level parallelism — see open
+item 1.
+
+**This table is the per-frame path.** A grouped run redistributes it rather
+than shrinking it: `load` and `correct_frame` move out of the compute worker
+into a prepare pool, where they overlap the next group's kernel instead of
+queueing behind it, and `reduce` shrinks because one merged batch replaces
+`F` per-frame batches. The kernel's own per-sample work is unchanged — which
+is why the 34% record saving converts to only ~12% wall time.
+
+## Subdivision hierarchy, with footprints
+
+One image passes through seven levels. Sizes are for the job above.
+
+| # | Scope | Structure | Footprint |
+|---|---|---|---|
+| 1 | per pixel | `std::vector<VoxelWeight>` + recursion stack | tiny; **absent at depth 0** (fast path); 1.2-3.5 KiB of stack at the deepest setting since corner-passing replaced the dense cache |
+| 2 | per block | `std::pmr::map<RecordKey, RecordAccum>` (arena-backed) | ~0.5 MiB |
+| 3 | per block | `std::vector<Record>`, sorted snapshot of (2) — **all blocks live at once** | records x 40 B |
+| 4 | per call | loser-tree merge of block runs -> six NumPy arrays | ~17 MiB per tile |
+| 5 | per frame | `tile_batches` + `_reduce_batches` | ~100 MB |
+| 6 | **per checkpoint** | `_CheckpointAccumulator._levels`, binary-counter tree, spans ~85 frames | **the dominant term** |
+| 7 | finalization | `_CheckpointRangeReader` + one output chunk at a time | ~1 GiB |
+
+Note there is **no thread-level structure**: a worker thread processes blocks in
+sequence, reusing one arena. The loser tree merges *per-block* runs at the
+*call* level.
+
+With frame grouping the *scope* of levels 2-5 changes but their structure
+does not. A "block" becomes a brick in (row, column, frame) holding the same
+sample count, so level 2's working set is unchanged and level 3 falls with
+the record density. Level 5 becomes per *group* — one merged batch for `F`
+frames, measured 0.575x the sum of the `F` it replaces — and level 6 gains
+a new resident term of its own: the group buffer, `F` full-detector
+corrected frames, which is the largest single allocation in the pipeline
+and what bounds how many group calls can run at once.
+
+Other fixed costs: detector ray cache 143 MiB for the job; per worker thread a
+1.13 MiB arena; grid chunks 32,768 of 7 MiB dense each.
+
+## Settled — please do not re-litigate without new evidence
+
+**The accumulator container stays `std::pmr::map`.** Measured alternatives at
+realistic redundancy (block 4096, ~7:1 in-block):
+
+| Container | ns/pixel | Bytes/key |
+|---|---|---|
+| `pmr::map` + last-key cache (current) | **30.1** | 72 |
+| open-addressing hash + sort | 27.6 | ~80 effective |
+| tlx B+ tree, key -> index + side array | 32.8 | 55 |
+| tlx B+ tree holding values | 41.4 | 80 |
+| sorted array, dedup on insert | 126.5 | 40 |
+| vector + sort | 40.4 | 40 |
+
+The B+ tree's memory win (~1.3x) costs speed and needs a side array, because a
+B-tree relocates values on split and the last-key cache holds a pointer. The
+sorted-run idea fails because the key stream is *not* monotone at any block
+size: `backscans/pixel` stays at 0.97-0.996 from 1024 to 65536-pixel blocks. A
+`(chunk, local)` key blocks reciprocal space in three dimensions, so one voxel
+of movement in x jumps the key by a whole chunk row.
+
+**Tiling does not reduce memory pressure.** 1 / 6 / 12 tiles measured 9.61 /
+10.12 / 9.81 GiB peak — flat. Tiling subdivides *one frame's* native call; the
+memory that matters accumulates *across* frames in the router, which no tiling
+arrangement touches. Fewer tiles is better for throughput — *within a
+limit*, see the next entry, which was measured later and bounds this one.
+
+**Band height is flat, and "fewer tiles is better" has a limit.** Swept in
+2026-08 at four frames per group over 240 / 421 / 506 / 600 rows,
+throughput is flat across the region that matters and the height the memory
+budget already produces sits in the middle of it:
+
+| bands | tiles | concurrent calls x threads | median ms/frame |
+|---|---|---|---|
+| 240 | 11 | 4 x 6 | 236.7 |
+| **421 (the default)** | 7 | **3 x 8** | **205.4** |
+| 506 | 5 | 3 x 8 | 207.1 |
+| 600 | 4 | 2 x 12 | 237.1 |
+
+Note the last row against the claim above it: **four tiles is worse than
+seven**. Fewer tiles is better only while the tile stays small enough for
+the memory budget to afford the same concurrency — past that, a larger tile
+costs a whole concurrent call and the tile-count saving does not come close
+to paying for it. Between 421 and 506 rows, where concurrency is equal, the
+two are indistinguishable, which is the form the original finding should
+have taken.
+
+Band height and group size also substitute for each other almost exactly —
+at depth 2, widening the bands makes the group-size gate halve the group
+and the two cancel — which is why neither repays hard optimisation. Detail
+in the locality document's phase 3 step 7.
+
+**The transition-rate correlation is container cost, not branch
+misprediction.** `voxel_id`'s cost is flat across transition rates (17.3 ->
+17.4 ns/px from 0% to 100%); what scales is the `pmr::map` insert (0.5 -> 90
+ns/px). An earlier session's 0.9959 correlation was measuring unique-key
+working set.
+
+**Reciprocal arithmetic was implemented, measured, and rejected.** Replacing the
+ray-normalisation and grid-step divisions with reciprocal multiplies is worth
+~35% in a cache-resident microbenchmark but only ~3% on real data, where those
+divisions hide under memory stalls. It shifts a coordinate by at most 1.28e-12
+voxel widths — zero assignment changes across 9.2M real records and 600M
+uniform samples, but 4.7% for coordinates sitting exactly on a voxel edge,
+which lattice-aligned synthetic data produces routinely.
+
+## Open items
+
+### 1. GIL-serialised per-frame Python work
+
+The largest remaining structural limit. Load + correct + per-tile copy is
+~114 ms of a 1089 ms frame and holds the GIL. Amdahl on a 10.5% serial fraction
+predicts a 6.1x ceiling at 16 workers; measured 4.2x (26% efficiency; 14% at 24
+workers on the coarser grid). Roughly two thirds of the loss is explained by
+that serial fraction, the rest most likely memory bandwidth.
+
+Candidate directions: move the per-tile slice/copy into the native call, or
+release the GIL across correction. Note the memory work reduced worker counts,
+so re-measure the balance before assuming the old figures still hold.
+
+**Partly addressed, structurally rather than by tuning.** The grouped
+scheduler hoists `load` and `correct_frame` into a prepare pool, so one
+group's GIL-held Python overlaps the previous group's native call instead of
+competing with it. That is the direction this item asked for, and it is what
+turned frame grouping from a 1.4x regression into a 0.88x win. It applies
+only above one frame per group; the per-frame path still has the serial
+fraction described above, and the per-tile slice/copy is still in Python on
+both paths.
+
+### 2. `bounded_block_size` still clamps memory with a depth-blind constant
+
+The Python side now caps the work block against the arena the kernel will
+reserve (`work_block_memory_cap`). The kernel's own `bounded_block_size()`
+still divides its budget by a magic `512` that ignores `worst_leaves`, so the
+two disagree about what a block costs.
+
+**The conclusion this item drew has been superseded.** It argued the
+authoritative cap belongs in the kernel, because the kernel knows whether an
+exposure is stationary while the Python side must assume the worst. Both
+memory prechecks are now bounded by the record ceiling instead of the leaf
+count, and that bound saturates above depth 0 — so neither side depends on
+the exposure model any more and the asymmetry disappeared rather than being
+relocated. See the locality document's phase 2.
+
+Still open, and narrower than it was: `bounded_block_size`'s magic `512` is
+untouched, and a third site — the detector band height in tile planning —
+was found in 2026-08 still using `8**depth` and has been corrected. Any
+remaining depth-blind constant in this area is worth checking against the
+same record ceiling.
+
+### 3. Depth 3+ block sizes are extrapolated
+
+The `16384 >> depth` rule was measured at depths 0, 1 and 2 (optima 16384, 8192,
+4096, consistent at 12 and 24 threads). Depths 3-8 follow by extrapolation only.
+
+### 4. `image_workers = min(len(pending_ranges), ...)`
+
+Concurrency is capped by the *scheduling* granularity. Production sizes
+`frame_batch` for roughly 4x as many ranges as workers, so it is fine mid-job,
+but the tail of every job runs with fewer workers by construction. It is also
+the single most effective way to invalidate a benchmark — see below.
+
+**Gone above one frame per group.** The grouped scheduler does not use
+`image_workers` at all: work is a queue of frame groups drained by a fixed
+number of concurrent native calls, so scheduling granularity no longer bounds
+concurrency. The per-frame path is unchanged, and the benchmark trap still
+applies to it.
+
+### 5. Finalization scaling at full job length
+
+Shrinking the accumulator share raises the checkpoint *part* count, and
+finalization k-way merges across all parts per output chunk. Measured flat over
+2 checkpoints (finalize 38.7-40.6 s across a 4x accumulator range), but a real
+job has 43 checkpoints, where the run count would grow roughly 86 -> 300, i.e.
+log2 6.4 -> 8.2. Still on a non-dominant term against HDF5 compression, but
+unverified at scale.
+
+### 6. The probe's reported error is conservative by construction
+
+`records_per_pixel_relative_error` treats the Sobol tiles as independent while
+the sequence is stratified, so it overstates true scatter by 2-4x (reported
+5.5-7.5% against observed 1.6-3.4%). Safe for a margin, not calibrated. A
+calibrated figure needs randomised QMC — several independent scrambles, variance
+across their means — which is too noisy at the 10-15 tiles a 0.1 s budget buys.
+
+### 7. Two fixes are unexercised on this hardware
+
+The rebalance unit fix (sample tile -> whole frame) and the adaptive re-check
+cadence only change a decision when the delivery rate is high enough that a
+candidate's requirement exceeds its worker ceiling. Here mapping is
+compute-bound — ~5 frames/s against 19 available — so the requirement stays at
+its floor either way. They need a machine where I/O outpaces compute.
+
+## Measurement traps
+
+Every one of these produced a confident, wrong conclusion during this work.
+
+**`image_workers = min(len(pending_ranges), ...)`.** Passing one pending range
+pins the compute pool to a single worker regardless of thread settings. This
+invalidated two separate rounds of conclusions, including a recommended
+accumulator constant that had to be retracted: with one worker, shrinking the
+accumulator looked like it *halved* peak memory; with a real pool it *raised*
+it, because faster mapping puts more frames in flight.
+
+**Harness costs inside the timed region.** An early tiling comparison timed its
+own `np.ascontiguousarray` copies and reported 2.4x where the real figure was
+1.13x.
+
+**Unequal work between arms.** Synthetic per-tile geometry that gave each
+"tile" the full angular range produced 8x the records in the split arm.
+
+**Overriding depth without overriding block size.** `max_depth` alone leaves
+`work_block_pixels` at its depth-0 value, inflating the kernel's arena from
+4.8 GB to 77 GB and making depth 8 look broken. Always derive both.
+
+**This machine is bimodal.** Identical binaries measure 42 or 71 ns/pixel
+depending on thermal/clock state. Single samples at high thread counts are
+worthless; alternate arms and take best-of-N.
+
+**End-to-end pipeline timings are worse than bimodal.** Single runs vary by
+up to 30% — the same configuration measured 183, 202 and 342 ms/frame on
+different occasions — and absolute values drift between *sessions*, so a
+number written down yesterday cannot be compared with one taken today. Only
+paired, interleaved ratios carry, nothing else may run on the machine while
+measuring (a `pytest` run in another shell is enough), and a clean
+separation between arms is worth more than a difference of means. Two rounds
+of conclusions in the locality document were withdrawn for this. Its
+"Measuring any of this" section is the full version.
+
+## Calibration constants and what they rest on
+
+| Constant | Value | Basis |
+|---|---|---|
+| `WORK_BLOCK_PRESETS["medium"]` | 16384 px | L2-sized working set; measured optimum at depths 0-2, both thread counts |
+| block scaling | `>> depth` | measured optima 16384 / 8192 / 4096 at depths 0 / 1 / 2 |
+| `CHECKPOINT_MEMORY_SHARE` | 0.25 | peak tracks 2x the accumulator budget; 4x shrink cost +6% scratch |
+| `_FRAME_RECORD_BYTES_PER_PIXEL` | 2 x 40 B | bounds measured density 0.46-0.87 rec/px, doubled for the merge |
+| `_RESERVED_RECORDS_PER_PIXEL` | 4 | 6-8x headroom over measured density; heap fallback covers the rest |
+| `_REBALANCE_INITIAL_SECONDS` | 30 | first check well inside a short job; backs off x2 to the old 600 |
+| probe `budget_seconds` | 0.1 | holds 0.04-0.10 s at every depth 0-8; GUI limit is 0.5 s |
+| `_GROUP_ADVANCE_VOXEL_LIMIT` | 1.0 voxel | striding sweep: the grouping gain is already gone by one voxel of travel per frame |
+| `_GROUP_MINIMUM_CONCURRENT_CALLS` | 3 | **purely empirical**: 4 frames/3 calls measured 0.87x, 8/2 was 1.01x, 8/1 was 1.24x |
+| `_GROUP_SIZE_CANDIDATES` | 1, 2, 4, 8, 16 | powers of two spanning the useful range; 4 is what the reference job picks |
+| `_GROUP_PIPELINE_*_READAHEAD` | 1 to 3 groups | one is the minimum for double-buffering; more costs a full group buffer each and buys nothing measured |
+
+The group constants are the least well-founded in this table. The voxel limit
+comes from a striding sweep on one scan; the concurrency floor of 3 is fitted
+to a single job's throughput curve and reproduces its optimum but has not
+been tested anywhere else. Treat both as calibrated to this machine and this
+scan in the same way the work-block size is.
diff --git a/doc/design/reciprocal_space_scratch_architecture.md b/doc/design/reciprocal_space_scratch_architecture.md
new file mode 100644
index 0000000..8bddbc2
--- /dev/null
+++ b/doc/design/reciprocal_space_scratch_architecture.md
@@ -0,0 +1,1094 @@
+# Reciprocal-space reconstruction: scratch-file and threading architecture
+
+Status: single-node and cluster implementation complete (Phases 1-3 of the
+implementation plan: calibration probe/file-count formula, HDF5 checkpoint
+format/tree-merge accumulator, run_job integration, and Sec13 cluster
+coordination via `run_cluster_map_task`/`run_cluster_finalize`/
+`generate_cluster_scripts`). Cluster array size is never read from a
+scheduler environment variable (most schedulers besides Slurm do not
+expose one to a running task); it is baked into the generated commands as
+an explicit `--total-tasks` argument instead.
+
+Sec7 (thread allocation policy) is fully implemented, in two parts. Phase
+4a: the previously fully-serial per-frame mapping loop is now a
+producer/consumer pipeline (`_AdjustablePool`, `_BoundedGate`,
+`_map_pending_ranges` in `orgui/reconstruction_job.py`) -- a small
+prefetch-reader pool that adapts continuously via a blocked-fraction
+signal (asymmetric hysteresis: grow eagerly, shrink cautiously) feeding a
+depth-bounded queue that a compute-worker pool drains. Phase 4b: when
+`ReconstructionJob.threads_per_image` is `None` ("automatic", the new
+default -- a concrete int remains a supported pinned advanced override),
+`_map_pending_ranges` seeds `kernel_threads` from how many frames the
+memory budget can hold in flight (`ceil(thread_budget /
+affordable_workers)`, from the same `_frame_parallelism` the pool is
+sized with) and then rebalances `kernel_threads`/`image_workers` live
+against the reader pool's own measured frame-delivery rate, via a
+wall-clock `_kernel_threads_sweep` (`orgui/datautils/xrayutils/reconstruction.py`)
+and the design doc's own joint-balancing rule (largest feasible
+`kernel_threads` from a thread-budget-derived candidate set, hysteresis
+against noise). A `kernel_threads` change swaps the whole compute-worker
+generation (each worker's native kernel is built once, at worker-start);
+an `image_workers`-only change just resizes the existing pool. This is a
+schema-affecting change: `JOB_SCHEMA_VERSION` 4 -> 5, since
+`threads_per_image`'s type is now `int | None` and its default changed
+from `4` to `None`.
+
+> **Revised after measurement.** The seed and the rebalance cadence
+> described above both changed once they were measured against real
+> beamtime data; see
+> `reciprocal_space_performance_findings.md`.
+>
+> The seed was an I/O-optimistic `kernel_threads=1`. That is only
+> sensible if memory affords enough concurrent frames to occupy the
+> thread budget, and it does not: the budget caps concurrent frames at
+> nine at `center` accuracy and five at `balanced`, so one native thread
+> each ran a 24-thread job on five threads — measured 1.85x slower than
+> using them all, and the rebalance could not correct it until its first
+> interval had elapsed.
+>
+> The cadence was a fixed `_REBALANCE_INTERVAL_SECONDS`, illustratively
+> 600s. One fixed value serves neither end of the range: a job shorter
+> than the interval never re-evaluated at all, and every longer job spent
+> its first ten minutes on the seed. It is now
+> `_REBALANCE_INITIAL_SECONDS` (30s), doubling by `_REBALANCE_BACKOFF` up
+> to `_REBALANCE_MAXIMUM_SECONDS` (the same 600s ceiling) while nothing
+> changes, and resetting whenever the pair does change.
+>
+> The joint-balancing rule itself also compared a frames-per-second
+> delivery rate against the time to map the sweep's *sample tile* rather
+> than a whole frame, understating the required worker count by the ratio
+> of the two (6x on a 6.2-megapixel detector). `_kernel_threads_sweep`
+> now takes `frame_pixels` and scales its result accordingly.
+
+**Sec7 now has a second scheduler beside the one described above.** Where a
+job's geometry makes consecutive images overlap in reciprocal space,
+mapping maps several of them in one native call
+(`_map_frame_groups_streamed`), and that is a different scheduling problem:
+one call covers `F` images, so there is no `image_workers`/`kernel_threads`
+pair left to balance. It is replaced by a fixed number of concurrent group
+calls, fed by a pool that loads *and corrects* — moving the GIL-held Python
+work off the compute worker so it overlaps the previous call instead of
+queueing behind it. The group size is measured per job. The pipeline
+described above is what runs when a job maps one image per call, which is
+what happens outside that regime. See
+[`reciprocal_space_mapping_locality_and_subdivision.md`](reciprocal_space_mapping_locality_and_subdivision.md),
+phase 3, which also records the two premises of Sec7 that this measured
+false.
+
+Sec14 (UI reporting) is implemented: `ReconstructionDialog`'s "Output
+grids" tab live-reports estimated checkpoint file and cluster-job counts
+(`ReconstructionDialog._refresh_file_count_summary`, reusing
+`estimate_checkpoint_plan` from Phase 1 unchanged). This completes every
+phase of the implementation plan; the redesign described by this document
+is done.
+
+**Reading this document after the fact.** It records the design as
+reasoned *before* the pipeline was profiled against real beamtime data.
+Several of its quantitative choices did not survive that measurement, and
+the passages affected now carry an inline "Revised after measurement"
+note pointing at
+[`reciprocal_space_performance_findings.md`](reciprocal_space_performance_findings.md),
+which holds the measurements, the settled questions, and what remains
+open. The architecture — checkpoint scratch format, tree-merge
+accumulator, prefetch/compute pipeline, cluster coordination — is
+unchanged and still described accurately here. What changed are constants
+and two rules: the `kernel_threads` seed, the rebalance cadence and its
+unit error, how the memory budget is divided, the native work-block size
+and detector tiling, and the record row size.
+
+One structural addition postdates that: the unit of native work is no
+longer necessarily one image, so Sec7's thread policy is one of two
+schedulers rather than the only one, and Sec9's accumulator receives one
+batch per *group* rather than per frame. Both are described in
+[`reciprocal_space_mapping_locality_and_subdivision.md`](reciprocal_space_mapping_locality_and_subdivision.md);
+neither changes the scratch format, the resume model, or the cluster
+coordination this document specifies.
+Relates to: issue #71, `orgui/datautils/xrayutils/reconstruction.py`,
+`orgui/reconstruction_job.py`, `orgui/reconstruction_cluster.py`,
+`orgui/app/ReconstructionDialog.py`
+
+**On the numbers in this document.** It contains many specific measured
+values — thread counts, RAM budgets, I/O throughput figures, per-call CPU
+timings, compression ratios — all measured on one development machine
+during this investigation. They are evidence for the architectural
+conclusions (which mechanism to build, which formula to use, which
+format/algorithm to adopt), **not defaults, thresholds, or configuration
+values to hardcode**. None of the following may be fixed in the
+implementation:
+
+- **Thread budget** (`total_threads`) and **RAM budget**
+  (`ram_budget_bytes`) — always the user's own resource request for that
+  job/node. `24` appears often below because it's this document's own test
+  environment, not a recommended value.
+- **I/O throughput** / achievable frame delivery rate — discovered live
+  (§7), since it depends on the real storage, network, and concurrent
+  cluster load at run time, none of which any benchmark run ahead of time
+  can represent.
+- **Per-call compute cost** (`per_call_time(kernel_threads)`,
+  mapping/reduction timings) — depends on the actual CPU, the actual
+  crystal orientation, and the actual grid/depth settings of each job;
+  measured by the §6 probe for every job, not read from this document.
+- **Tail-flush latency** (§10) — likewise measured locally per job, not
+  carried over from this investigation's numbers.
+
+What *is* meant to carry over as a fixed decision is the qualitative,
+structural conclusions — HDF5 over Parquet (§8), columnar over compound
+layout (§8), XXH3 over SHA-256 (§8), tree-merge over naive incremental
+merge (§9) — because those are comparisons of one approach against another
+expected to hold on ordinary hardware, not absolute throughput figures to
+target or replicate.
+
+## 1. Problem statement
+
+The current out-of-core reconstruction pipeline (introduced in `d2421e1`,
+"feat: add centralized reciprocal-space reconstruction") ties scratch-file
+count to **grid structure** (`partition_chunk_span` for map-phase Parquet
+partitions, and one reduce-phase Parquet file per `(chunk_id, shard)` pair)
+rather than to **data volume versus available memory**, which was the
+original intent. On a real production job
+(`39_1-rsmap`, CHESS QM2, La3Ni2O7, Pilatus 6M, 1000x1000x1500 `hkl` grid,
+3651 frames) this produced 4096 reduce-phase shard files from 58 map-phase
+tasks, and a live re-run at `max_depth=0` crashed with
+`OSError: [Errno 24] Too many open files` partway through the reduce phase
+(measured: mapping completed in 882.7 s, reduce crashed 17 s later at
+shard 63/4096).
+
+Root cause: `_ParquetRangeReader` objects are cached per worker per
+partition URI in `reduce_plan_batch()` (`reconstruction.py:1294`), and are
+only released when the `(grid_name, bucket)` group changes
+(`readers.clear()`). With `partition_chunk_span=4096` the entire job's grid
+fit in one bucket, so the group never changed and the cache never cleared:
+16 reduce workers each accumulated up to 58 open partition readers,
+~928 concurrently open files, past the platform's file-handle limit
+(most acute on Windows, whose default per-process limit is far below
+Linux's).
+
+Separately, `threads_per_image` (a per-job setting, inherited unchanged
+across accuracy presets) does not scale with `max_depth`. At `max_depth=0`
+the kernel does ~1.8 CPU-seconds of work per frame, but `threads_per_image=8`
+(tuned for the job's original `max_depth=3` run) capped concurrent frame
+processing to `spec.threads // 8` workers, wasting most of the available
+thread budget on 8-way parallelism for work too small to benefit from it.
+Measured: mapping phase took 882.7 s versus a ~70-140 s estimate from
+isolated per-frame kernel-cost measurements at proper parallelism.
+
+## 2. Goals
+
+- Scratch/checkpoint file count should be driven by
+  **data volume vs. memory budget**, not by output-grid chunking.
+- File count should be small: on the order of a user-configured
+  checkpoint count (default 10), not thousands.
+- Checkpointing should still support resuming an interrupted job.
+- Fix the file-handle exhaustion bug as a byproduct of the redesign, not as
+  a separate patch.
+- Address the `threads_per_image` mismatch: image-level and kernel-level
+  parallelism should be chosen from a measured, settings-aware cost
+  estimate, not a static inherited value.
+- No resource budget or environment-dependent rate — thread count, RAM
+  budget, I/O throughput, per-call compute cost — may be hardcoded from
+  this document's own measurements. Every one is either the user's actual
+  resource request or a quantity the implementation measures live, on
+  whatever machine and job are actually running (see the note at the top
+  of this document).
+
+## 3. Non-goals
+
+- Changing the scientific algorithm (adaptive footprint splitting itself,
+  voxel binning, variance propagation) is out of scope. This document is
+  about the I/O/scheduling layer around the existing kernel.
+- Redesigning the GUI beyond the reporting fields specified in §14.
+- A general-purpose file-format migration outside reconstruction scratch
+  data (Parquet remains available/optional elsewhere if needed).
+
+## 4. User-facing parameters
+
+Two parameters, both per job:
+
+- **`ram_budget_bytes`** — combined memory for the whole job: all threads,
+  one cluster node. Already exists conceptually as
+  `runtime_memory_bytes`/`memory_override_bytes`; this redesign makes it
+  actually drive file count, not just gate the native kernel's memory
+  precheck.
+- **`checkpoint_count`** — minimum number of resumable checkpoint files.
+  **Default 10.** A floor, not a target: actual file count can exceed it
+  if data volume requires more (see §5).
+
+## 5. File-count formula
+
+```
+files_per_job = max(checkpoint_count, ceil(job_data_bytes / ram_budget_bytes))
+```
+
+`job_data_bytes` is the **reduced** (post in-frame-dedup) record volume for
+the job's frame-range slice — not raw per-pixel/per-corner records, and
+explicitly decoupled from output-grid chunk count.
+
+Checkpoint frame ranges are **contiguous**, computed once and persisted
+(see §11) — not recomputed on every resume, because the estimator in §6 has
+sampling variance and recomputation could shift boundaries between runs
+with identical settings.
+
+### Cluster case
+
+Each cluster job (SGE/Slurm array task, one node) is assigned a disjoint
+contiguous frame-range slice up front, and applies the formula
+independently to its own slice's `job_data_bytes`. Total files across the
+cluster = `sum(files_per_job over all nodes)`. No cross-node coordination
+is needed to compute this (see §13).
+
+## 6. Estimating `job_data_bytes` and per-frame cost: the calibration probe
+
+A single mechanism serves both the file-count formula (§5) and live UI
+responsiveness (per-image time estimates), because a single native
+`accumulate(..., profile=True)` call already returns both
+`block_mapping_cpu_seconds` (time) and `reduced_block_records`
+(post-dedup record count -> bytes) from the same sampled pixels.
+
+### Why it doesn't need a real loaded image
+
+Coordinate-evaluation cost depends only on geometry (corner rays, angle
+bounds, grid, UB) and the mask, not on pixel intensity values. The probe
+uses the **real mask** and **real angle bounds** for a representative
+frame, but **dummy zero intensity/variance** — decoupling the probe from
+image I/O and disk-speed variability entirely.
+
+### Two-pass, budget-bounded (~100 ms) design
+
+1. **Bootstrap pass** (a few ms): run the kernel on one tiny tile
+   (e.g. 16x16 = 256 pixels) to get a preliminary per-pixel rate
+   `r0 = t0 / 256`.
+2. **Sized pass** (remaining budget): compute a target sample size
+   `N = remaining_budget / r0` (clamped to a sane range), split across
+   several small tiles **scattered across the detector** (stratified, not
+   one contiguous block — per-pixel split cost varies spatially, e.g. near
+   voxel boundaries or the specular rod). Run one `accumulate()` call per
+   tile, sum `block_mapping_cpu_seconds` and pixel/record counts across all
+   of them.
+
+This self-scales usefully with `max_depth`: at high depth (measured
+~64 us/valid-pixel at `max_depth=3` on the real job) the 100 ms budget only
+affords a small, noisier sample — acceptable, because at that cost regime
+the qualitative answer ("expensive") is unambiguous even with noise. At low
+depth (~0.3 us/valid-pixel at `max_depth=0`) the same budget affords a much
+larger, more precise sample, which is where fine distinctions (depth 0 vs
+1) are more likely to matter.
+
+### Known limitation
+
+A small tile under-counts cross-tile-boundary duplicate voxel hits relative
+to a full frame, so the record-count half of the estimate is slightly
+pessimistic (more records than reality). Should be surfaced in the UI as a
+conservative estimate, not treated as a correctness bug.
+
+### Multi-grid extension
+
+The probe naturally extends to jobs with multiple simultaneous output
+grids (`spec.grids` is already a tuple): each grid has its own kernel
+(`_kernel_for_grid`), and running each grid's kernel on the same sampled
+tile costs little extra since no new image data or geometry sampling is
+needed. Per-grid `job_data_bytes` and per-grid `files_per_job` fall out
+directly (§12).
+
+## 7. Thread allocation policy: `image_workers`, `kernel_threads`, and I/O prefetch
+
+Replaces the static, depth-unaware `threads_per_image` setting responsible
+for the mapping-phase slowdown in §1 (measured 882.7 s against an estimated
+~70-140 s, traced to `threads_per_image=8` capping frame-level concurrency
+to `spec.threads // 8 = 3` workers at `spec.threads=24`).
+
+> **Revised after measurement.** The `work_block_pixels=4096` used
+> throughout this section's sweeps is no longer a fixed value: the block
+> is now chosen by cache scale and adaptive depth
+> (`WORK_BLOCK_PRESETS`, default `medium` = 16384 pixels at `center`,
+> halving per depth — so 4096 at `balanced`, which is what this section
+> measured). Detector tiles are also full-width row bands of equal height
+> rather than a square grid. Both change the absolute timings tabulated
+> below, though not the shape of the thread-scaling argument they were
+> gathered to make.
+
+> **This section now describes one of two schedulers.** Mapping maps
+> several adjacent frames in one native call where the job's geometry
+> supports it, and a frame group is a different scheduling problem: one
+> call covers `F` frames, so the joint `image_workers`/`kernel_threads`
+> balance below is replaced by a fixed number of concurrent group calls
+> fed by a prepare pool that also does the correction. The rule that
+> survives is the *metric* — aggregate throughput per committed thread,
+> not per-call latency — and the blocked-fraction signal, which now sizes
+> the prepare pool. Everything below still describes the path taken when
+> a job maps one frame per call, which is what happens outside the regime
+> (coarse scans, interlaced frame order, tight memory).
+>
+> One claim in this section was tested directly and failed. A group call
+> has thousands of bricks, which suggested it would saturate the thread
+> budget on its own and make `kernel_threads = spec.threads` with one call
+> in flight the natural layout. Measured, that is **1.38x worse** than
+> three concurrent calls of eight threads. Available work is not what
+> limits thread scaling here. See
+> `reciprocal_space_mapping_locality_and_subdivision.md`, phase 3 step 5.
+
+### The metric: aggregate throughput per committed thread, not per-call latency
+
+A policy that picks `kernel_threads` to minimize a single call's wall time
+picks wrong, because it ignores what those threads aren't doing (mapping
+other frames). Measured directly at `max_depth=0` (corrected
+`work_block_pixels=4096` — an initial pass with `work_block_pixels=1<<30`,
+intended as "unlimited," accidentally disabled all internal blocking
+(`blocks = ceil(pixels/block_size)` collapsed to 1 for every tile tested)
+and gave misleadingly flat results; corrected below):
+
+| pixels     | t=1     | t=4     | t=8     | t=16    | t=24    |
+|-----------:|--------:|--------:|--------:|--------:|--------:|
+| 65,536     | 6.08ms  | 2.41ms  | 2.76ms  | 4.14ms  | 3.79ms  |
+| 262,144    | 18.45ms | 7.82ms  | 5.86ms  | 6.12ms  | 7.28ms  |
+| 1,048,576  | 48.05ms | 17.44ms | 14.99ms | 13.48ms | 13.41ms |
+
+Real speedup exists up to ~4-8 threads, then flattens hard. Converting the
+1,048,576-pixel row to aggregate throughput for a 24-thread budget — the
+scale this design will actually be implemented and tested against
+(`image_workers = 24 / kernel_threads`, throughput = `image_workers /
+call_time`):
+
+| `kernel_threads` | ms/call | `image_workers` @24 threads | aggregate frames/s-equiv |
+|------------------:|--------:|------------------------------:|---------------------------:|
+| 1                  | 48.05   | 24                             | ~500                       |
+| 4                  | 17.44   | 6                              | ~344                       |
+| 8                  | 14.99   | 3                              | ~200                       |
+| 24                 | 13.41   | 1                              | ~75                        |
+
+(These ratios — 2.5-7x between `kernel_threads=1` and the larger values —
+are independent of `total_threads`: `image_workers` scales linearly with
+the budget for any fixed `kernel_threads`, so the throughput *ratio*
+between two `kernel_threads` choices cancels the budget out. Only the
+absolute frames/s-equiv numbers change with thread count, not the shape
+of the comparison.)
+
+Naively, `kernel_threads=1` looks like the winner by 2.5-7x. **That
+conclusion is wrong** — it silently assumes all 24 threads can be kept
+fed as 24 independent single-threaded workers, which only holds if
+something can actually deliver that many concurrent frames. The next
+subsection shows that this assumption is genuinely uncertain at this
+thread scale, not clearly true or false. The threads that can't be used
+as additional `image_workers` aren't protected by setting
+`kernel_threads=1` — they just sit idle, when the measured data above
+shows they could instead be speeding up the calls already in flight for
+free. See "Adopted" below for the corrected, unified rule.
+
+### At low depth, I/O is the real ceiling — the reverse holds at higher depth
+
+The ~2000 frames/s-equivalent figure above is compute-only and, **at
+`max_depth=0` specifically**, does not survive contact with real reads.
+Measured against the real scan data (concurrent `fabio.open()` reads via
+`ThreadPoolExecutor`, same 64-file subset reused across all trials):
+
+| readers | frames/s | MB/s   |
+|--------:|---------:|-------:|
+| 1       | 26.6     | 662.5  |
+| 2       | 92.4     | 2301.3 |
+| 4       | 155.0    | 3859.0 |
+| **8**   | **176.6**| **4395.9** |
+| 16      | 170.3    | 4238.7 |
+| 32      | 168.8    | 4202.3 |
+
+Read throughput saturates at 8 concurrent readers, ~176 frames/s. At a
+24-thread budget — the scale this design will actually run on, not a
+large cluster node — compute at `kernel_threads=1` (all 24 threads spent
+as `image_workers`) only reaches ~83 frames/s (24 / ~290 ms-per-frame,
+extrapolated to full-frame scale), which doesn't even reach that I/O
+figure. At this thread scale compute and I/O sit much closer together
+than a large-budget illustration would suggest — plausibly on either
+side, depending on the real (unmeasurable-in-advance, and itself
+unreliable as measured here) I/O rate. This is weaker and more honest
+than claiming I/O is clearly the ceiling at `max_depth=0`: at 24 threads
+it may not be, which is exactly why the rule below discovers the regime
+live rather than assuming a direction.
+
+This uncertainty does not extend to `max_depth>0` in the same way. Per-call
+compute cost grows steeply with depth (the §17-confirmed sweep: ~4.9x per
+depth step at fixed tile size), while the achievable I/O rate does not
+move — and at only 24 threads to begin with, there is far less compute
+headroom to spare before the crossover. At `max_depth=3`, 24
+single-threaded workers manage only ~0.066 frames/s in aggregate (worked
+out below) — nowhere close to any plausible I/O rate, however low — so
+compute, not I/O, is unambiguously the ceiling from `max_depth>=1`
+onward at this thread scale. The rule adopted below is built to discover
+which regime applies live, precisely because the answer is genuinely
+close at `max_depth=0` and not close at all past it.
+
+**This number must not be hardcoded into the policy — the measurement
+itself is unreliable, for two independent reasons:**
+
+1. The benchmark reused the same 64 files across every concurrency trial.
+   After the `readers=1` pass, those files were very likely sitting in the
+   OS page cache for every subsequent trial, so everything from
+   `readers=2` onward may be measuring cached-read plus CBF-decode
+   throughput, not storage I/O at all.
+2. Even with clean methodology (distinct, never-touched files per trial),
+   this was run against a local disk copy (`D:\sarker-4910-f`), not the
+   real NFS mount (`/nfs/chess/...`) the production job reads from. NFS
+   has its own client/server caching layers, network latency, and
+   contention from other users' concurrent jobs on shared beamline
+   storage — none of which a single-user local-machine benchmark can
+   reproduce. A "corrected" version of this same benchmark would still
+   only describe local SSD behavior.
+
+### Adopted: match compute throughput to observed I/O rate, spend surplus threads on `kernel_threads`
+
+Given the I/O ceiling cannot be reliably pre-measured — not as an
+implementation gap, but because NFS caching and multi-user cluster load
+make advance measurement structurally unreliable — the policy discovers it
+live rather than pre-computing it, and uses it as the target that
+`kernel_threads`/`image_workers` are jointly balanced against. Three
+pieces:
+
+1. **A small fixed-default prefetch pool** (e.g. 4 workers), decoupled
+   entirely from `image_workers`/`kernel_threads` sizing, replacing
+   today's fully serial per-task frame loop (`_map_frame_range` does not
+   overlap image N+1's read with image N's processing at all — the only
+   overlap that exists today is incidental, from different concurrent
+   tasks happening to be at different stages). Mirrors EpiQ-Map's
+   one-frame-ahead prefetch pattern (`ThreadPoolExecutor(max_workers=1)`
+   in `auto_rsm.py`'s `iter_frames()`), generalized to a small pool. The
+   default size is chosen because diminishing returns past a handful of
+   concurrent reads is a common pattern across storage backends, not
+   because 4 is known to be optimal for this job's actual NFS mount — it
+   isn't claimed to be.
+2. **Live I/O-rate adaptation.** Track, during actual job execution, how
+   often compute workers sit blocked waiting on a prefetched frame (I/O
+   can't keep up -> grow the prefetch pool, up to a cap) versus how often
+   the prefetch queue is comfortably full ahead of compute (I/O keeps up
+   easily). This produces a running estimate of the achievable frame
+   delivery rate against the real production storage, not a synthetic
+   benchmark that cannot represent it.
+3. **Joint `kernel_threads`/`image_workers` balancing against that
+   estimate.** The §6 probe is extended to measure `per_call_time` at a
+   small candidate set of `kernel_threads` values (1, 2, 4, 8, ... up to
+   where the sweep in this section shows returns flatten), not just one.
+   Given the current I/O-rate estimate `r` (frames/s) from (2), choose the
+   *largest* `kernel_threads` from that candidate set for which
+   `ceil(r * per_call_time(kernel_threads)) * kernel_threads <=
+   total_threads` still holds — i.e. the most threads concentrated per
+   call that still leaves enough `image_workers` to keep up with `r`. Any
+   thread capacity beyond what's needed to match `r` goes toward
+   `kernel_threads`, not toward more `image_workers` sitting idle waiting
+   for frames that aren't arriving any faster.
+
+   > **Note on units.** `r` is frames per second, so the quantity it
+   > multiplies must be the time to map a *whole frame*. The
+   > implementation measured a single sample tile (`_KERNEL_SWEEP_TILE_PIXELS`,
+   > ~1 megapixel) and used that directly, understating the requirement
+   > by the ratio of a frame to that tile — 6x on a 6.2-megapixel
+   > detector, enough that every candidate looked feasible and the choice
+   > fell through to the "largest" tie-break rather than the measurement.
+   > `_kernel_threads_sweep` now scales its result to a frame.
+
+This is one continuous rule, not a per-depth special case, and it produces
+both ends of the range from the same computation:
+
+- **At `max_depth=0`**, with a 24-thread budget, take a purely
+  illustrative I/O-rate estimate of 20 frames/s (a made-up number for
+  exposition — not a measurement, unlike the unreliable-but-real ~176
+  figure above, chosen here specifically because it's low enough to be
+  achievable within 24 threads at every candidate `kernel_threads`,
+  unlike 176). Matching 20 frames/s needs `ceil(20 * 0.290) = 6`
+  `image_workers` at `kernel_threads=1` (6 threads used), `ceil(20 *
+  0.104) = 3` at `kernel_threads=4` (12 threads used), `ceil(20 * 0.089) =
+  2` at `kernel_threads=8` (16 threads used) — and `kernel_threads=16`
+  would need 2 workers x 16 threads = 32, exceeding the 24-thread budget,
+  so it's infeasible. The rule picks `kernel_threads=8` here: the largest
+  candidate that still fits, using 16 of 24 threads with headroom to
+  spare. The exact numbers depend entirely on the live-measured rate, not
+  this illustration — a higher live estimate would push the choice back
+  toward lower `kernel_threads`/more `image_workers`, a lower one further
+  toward concentrating threads per call.
+- **At `max_depth=3`**, per-call time balloons to ~366 CPU-seconds at
+  `kernel_threads=1`. 24 such single-threaded workers could only manage
+  ~0.066 frames/s in aggregate — far below any plausible I/O rate, however
+  low — so no candidate `image_workers` count at `kernel_threads=1` can
+  satisfy the balancing condition in (3). The rule is then forced toward
+  the largest `kernel_threads` the candidate set offers, i.e.
+  `threads_per_image` "wins decisively," exactly as expected, derived from
+  the mechanism rather than asserted as a separate case.
+
+### Prefetch queue depth: bounded by compute's consumption rate, not by I/O's delivery rate
+
+The prefetch pool from (1) has two distinct parameters that must not be
+conflated: how many *reader threads* it runs, and how many *decoded
+images* it is allowed to hold buffered ahead of compute. The
+`blocked_fraction` signal in (2) correctly governs the first. It says
+nothing about the second, and leaving the second unbounded is a real
+failure mode in the compute-bound regime: at `max_depth=3`, I/O can
+deliver a frame roughly every ~43-60 ms while a single-threaded compute
+call takes ~3 s (measured, 65,536 px tile) — over a 3 s window I/O could
+hand over ~50 decoded images. Prefetching that far ahead would be wrong:
+each decoded Pilatus 6M frame is tens of MB, and buffering dozens of them
+achieves nothing (compute can only consume one at a time per active
+worker) while consuming real memory that competes with §10's checkpoint
+accumulators for the same `ram_budget_bytes`.
+
+The queue only needs to stay far enough ahead that a compute worker never
+finds it empty when it asks — which is a function of how many workers are
+consuming from it, not of how fast it could theoretically be filled:
+
+```
+N ~= image_workers + small_constant   (e.g. +2 to +4)
+```
+
+Implemented as an ordinary bounded queue: reader threads block on `put()`
+once it holds `N` images, resuming only as compute workers dequeue. This
+is passive backpressure, not another instance of the live-adaptation loop
+in (2) — it needs no monitoring of its own, and it automatically produces
+the right behavior in both regimes without knowing which one it's in:
+
+- **Compute-bound** (`max_depth=3`, illustratively `image_workers=1` at a
+  24-thread budget once `kernel_threads` claims most or all of it per the
+  rule above — the confirmed sweep shows depth-3's plateau is still
+  climbing at 16 threads, so at a 24-thread total budget `kernel_threads`
+  could plausibly claim the entire allocation): `N` is small (~3-5), the
+  queue fills almost immediately, and the (nominally larger, e.g.
+  4-worker) prefetch pool sits mostly blocked on `put()`, consuming no CPU
+  regardless of how much faster the storage could go. No 50-image
+  buildup.
+- **I/O-bound** (`max_depth=0`, `image_workers` potentially up to the full
+  24-thread budget): the queue rarely reaches `N` at all, since compute
+  drains it as fast or faster than the (I/O-limited) pool can fill it —
+  the cap is simply inactive, and the `blocked_fraction` mechanism in (2)
+  is what's actually governing behavior here, as designed.
+
+This adds a third constraint to the balancing rule in (3), alongside the
+thread-budget and I/O-rate-matching conditions: `image_workers *
+bytes_per_decoded_image <= image_buffer_budget_bytes`, a slice of
+`ram_budget_bytes` reserved separately from §10's checkpoint-accumulator
+allocation. In practice this rarely binds at a 24-thread scale
+(`image_workers` is bounded by `total_threads=24` regardless, so total
+buffer need tops out around `24 * ~50 MB ~= 1.2 GB` even in the most
+I/O-bound case, against typical multi-GB-to-tens-of-GB budgets), but it is
+an explicit, checked bound rather than an assumption, since it is exactly
+what prevents the compute-bound case from buffering images nobody is
+ready to process.
+
+## 8. Checkpoint file format: HDF5, not Parquet
+
+Benchmarked against the real record schema (`chunk_id`, `local_voxel_id`,
+`weighted_intensity`, `weighted_variance`, `weight`, `contributors`,
+48 bytes/row, 800k rows / 38.4 MB raw, matching one frame's depth-0 reduced
+output):
+
+> **Now 40 bytes/row.** `chunk_id`, `local_voxel_id` and `contributors`
+> were narrowed from 64-bit to 32-bit (`_CHECKPOINT_BYTES_PER_ROW`), which
+> changed both the checkpoint format and the final output's
+> `contributors` dtype. The relative comparison below is unaffected; only
+> the absolute row size is. The same correction applies to §12's
+> "fixed 48-bytes-per-row schema".
+
+| backend                | write MB/s | read MB/s | file size |
+|-------------------------|-----------:|----------:|----------:|
+| parquet:zstd (current)  |        334 |       775 |   22.2 MB |
+| parquet:lz4             |        401 |      1598 |   24.1 MB |
+| parquet:snappy          |        428 |      1753 |   24.4 MB |
+| hdf5:gzip4              |         85 |       398 |   18.9 MB |
+| hdf5:lzf                |        475 |       494 |   21.0 MB |
+| **hdf5:bitshuffle-lz4** |    **905** |   **840** | **19.7 MB** |
+| hdf5:none               |       2251 |      1726 |   44.1 MB |
+
+`bitshuffle-lz4` — already used by `_finalize_reconstruction` for the final
+output, via the already-present `hdf5plugin` dependency — writes ~2.3x
+faster than the current `parquet:zstd`, reads comparably, and produces
+smaller files. Switching removes the `pyarrow` dependency entirely
+(currently optional-but-required: `_require_pyarrow()` raises if it's
+missing) since HDF5 is already a hard dependency everywhere else in orGUI
+(config, job assets, final output). h5py's resizable datasets
+(`maxshape=(None,)` + `.resize()`) also fit the append-as-you-go checkpoint
+model better than Parquet's row-group/file-per-flush model.
+
+### Layout: separate 1D datasets per column, chunk=65536 rows
+
+Benchmarked columnar (six separate datasets, one per record field) against
+a single compound/table dataset, at four chunk sizes (16384-262144 rows),
+same schema and row count as above:
+
+| layout                     | write MB/s | read MB/s | size     |
+|-----------------------------|-----------:|----------:|---------:|
+| columnar, chunk=16384       |      851.9 |     865.1 | 19.35 MB |
+| compound, chunk=16384       |      886.0 |     792.7 | 19.44 MB |
+| **columnar, chunk=65536**   |  **973.5** | **935.5** | 19.36 MB |
+| compound, chunk=65536       |      908.6 |     904.0 | 19.45 MB |
+| columnar, chunk=131072      |      915.1 |     778.7 | 19.38 MB |
+| compound, chunk=131072      |      867.4 |     876.5 | 19.47 MB |
+| columnar, chunk=262144      |      887.9 |     762.2 | 19.42 MB |
+| compound, chunk=262144      |      792.4 |     784.1 | 19.50 MB |
+
+File size is essentially identical across layouts (~19.4 MB either way) —
+mixed-type compound records were expected to compress worse than pure
+same-type columns under bitshuffle; the measured difference is under 1%,
+not a real factor. Performance differences between layouts are a few
+percent either direction depending on chunk size; columnar at
+`chunk=65536` is the best-measured configuration.
+
+Adopted: **columnar, chunk=65536 rows** — the measured winner, but decided
+on a second ground independent of the numbers: the in-memory
+representation used everywhere else in this design (the native kernel's
+output, `_merge_sorted_batches`, the §9 tree-merge `levels` structure) is
+already a dict of separate arrays. A compound dataset would require
+assembling a structured array before every checkpoint write and
+disassembling it after every resume/finalize read — real conversion cost
+on a layout that doesn't even win on raw throughput. Columnar writes each
+already-existing array straight through. `chunk=65536` also gives a
+sensible chunk count per checkpoint (~11-15 chunks for the ~740k-985k row
+checkpoints in the §15 worked example) — fine enough to avoid forcing
+full-file reads for partial access during resume verification, coarse
+enough not to fragment.
+
+### Checksums: SHA-256 -> XXH3-128, computed inline
+
+Scratch-file verification currently uses `hashlib.sha256()`
+(`_uri_checksum_and_size`), not the fast native XXH3 binding already used
+for scientific-input fingerprinting. Benchmarked:
+
+| buffer  | SHA-256          | XXH3-128 (native)   | speedup |
+|---------|------------------:|---------------------:|--------:|
+| 256 MB  | 0.108 s (2.4 GB/s) | 0.0066 s (39.0 GB/s) |   16.5x |
+| 1024 MB | 0.436 s (2.3 GB/s) | 0.0272 s (37.7 GB/s) |   16.0x |
+| 4096 MB | 1.812 s (2.3 GB/s) | 0.1202 s (34.1 GB/s) |   15.1x |
+
+At the real job's scale (~66.5 GB compressed scratch volume total), SHA-256
+costs ~29 s CPU-bound, XXH3 ~1.9 s — both trivial next to reconstruction
+wall time, so the real win isn't the hash speed, it's that going from 4096
+files to ~10-30 also removes ~400x the per-file overhead (opens,
+Python-level calls, thread submission).
+
+Checksums should be computed **inline during write**, not by re-reading
+the file afterward (`_uri_checksum_and_size`'s current pattern) — this
+removes an entire extra disk-I/O pass, independent of which hash algorithm
+is used, and matters more on network scratch storage (this job's actual
+NFS mount) than on local SSD.
+
+## 9. Checkpoint write path: per-checkpoint tree-merge accumulator
+
+### Rejected: naive incremental merge
+
+A single lock-protected accumulator, with each finishing frame merged in
+one at a time via `_merge_sorted_batches`, is **quadratic** in
+frames-per-checkpoint: merge cost scales with current accumulator size, so
+total cost over N frames is O(R*N^2/2). Measured directly (batch size
+739,791 rows, matching real depth-0 output):
+
+| frames | naive merge | tree merge (`_reduce_batches`) | speedup |
+|-------:|------------:|--------------------------------:|--------:|
+|      8 |     0.28 s  |                          0.20 s |    1.4x |
+|     16 |     1.08 s  |                          0.52 s |    2.1x |
+|     32 |     3.90 s  |                          1.29 s |    3.0x |
+|     64 |    17.00 s  |                          3.20 s |    5.3x |
+|    128 |    67.21 s  |                          8.81 s |    7.6x |
+
+Extrapolated to a real checkpoint (~365 frames at `files_per_job=10` on a
+3651-frame job): naive ~546 s/checkpoint (~1.9x the checkpoint's own
+mapping cost at depth 0 -- it would dominate the job), tree ~31 s/checkpoint
+(~10% addition on top of mapping) — a per-frame cost (~85 ms amortized)
+comparable in magnitude to a single frame's image load and correction time,
+which is why §7 sizes `image_workers` from total per-frame pipeline cost
+rather than kernel cost alone.
+
+### Adopted: tree/binary-counter accumulator, per (checkpoint, grid)
+
+State per `(checkpoint, grid)` pair:
+
+- one lock
+- `levels: list[batch | None]` — the same binary-counter structure
+  `_reduce_batches` already builds internally, but persisted across time
+  and fed incrementally instead of built once from a pre-collected list
+- a running byte-total, for the budget check below
+
+On a worker finishing a frame's (already kernel-side-reduced) batch:
+
+1. Acquire the checkpoint's lock.
+2. Run `_reduce_batches`'s existing insert loop: descend levels, merging
+   and cascading exactly as it does today, placing the result at the
+   first empty level (or appending a new one).
+3. Update the byte-total; if it crosses this checkpoint's budget share
+   (§10), trigger a flush before releasing.
+4. Release the lock.
+
+Insert cost is *amortized* O(log N) per call (a cascade to depth k happens
+once per 2^k insertions), so total lock-held time across N insertions is
+O(N log N), matching the measured tree numbers. Mapping — the expensive,
+fully parallel part — happens entirely outside the lock, so contention
+stays low relative to how often workers need it.
+
+On checkpoint completion (frame range exhausted) or a mid-range budget
+split: fold remaining levels together
+(`_reduce_batches`'s existing finalization step), write to the
+checkpoint's HDF5 file via write-`.tmp`-then-`.replace()` (§11), reset
+`levels`.
+
+### Validation strategy: equivalence against the current reduce, not an external ground truth
+
+Neither the current per-flush reduce logic nor this replacement has a
+real validation dataset — the current implementation isn't independently
+verified either, though it isn't known to be wrong. That rules out
+"regression testing against known-correct output" as an option for either
+side. What's achievable instead: generate a small synthetic dataset from
+deterministic random noise (fixed RNG seed, a handful of frames, a small
+grid — no need for real detector/crystal data, per §6's finding that
+mapping cost and correctness depend only on geometry and mask, not pixel
+values), run it through both the current `_reduce_batches`/per-flush path
+and the new per-checkpoint tree-merge accumulator, and assert the two
+produce identical (or, if floating-point summation order differs,
+equivalent within floating-point tolerance) `(chunk_id, local_voxel_id) ->
+(weighted_intensity, weighted_variance, weight, contributors)` output.
+This establishes that the new path preserves the old path's behavior,
+which is the honest and available claim — not that either is correct
+against some external reference, since no such reference exists to test
+against.
+
+## 10. Concurrency and memory division
+
+Two independent axes, deliberately not conflated:
+
+- **`image_workers`**: how many frames are processed concurrently. Sized
+  per §7 (calibration probe + I/O-prefetch adaptation) — a
+  scheduling/throughput decision.
+- **`max_concurrent_active_checkpoints`**: how many checkpoints have
+  non-empty in-memory accumulator state at once. **Small, fixed, default
+  2** (current + next), not user-facing. Checkpoints are processed in
+  frame-range order; workers are allowed to start claiming the next
+  checkpoint's frames slightly before the current one fully drains, to
+  avoid idle-worker gaps at boundaries, without letting the active set
+  grow unbounded.
+
+```
+active_budget_per_accumulator =
+    ram_budget_bytes / (max_concurrent_active_checkpoints * number_of_grids)
+```
+
+Dividing by 2 (not `image_workers`, and not `files_per_job`) is not just
+simpler, it's more memory-efficient: checkpoints that haven't started yet
+hold zero memory, so there's no reason to reserve a slice of the budget for
+all of them upfront. A smaller divisor gives each active accumulator more
+headroom, meaning fewer safety-valve splits in practice.
+
+> **Revised after measurement.** This formula hands the accumulators the
+> *whole* budget: two active checkpoints, each allowed half of it. The
+> frame pipeline was sized against the whole budget independently, so
+> nothing constrained their sum, and measured peak resident memory
+> reached 12.4-14.5 GiB against a 10 GiB budget.
+>
+> `split_memory_budget()` now divides the budget once —
+> `CHECKPOINT_MEMORY_SHARE` (0.25) to the accumulators, the rest to the
+> pipeline — so the numerator above is a quarter of `ram_budget_bytes`,
+> not all of it. The reasoning for the divisor of 2 is unchanged; only
+> what is being divided.
+>
+> The "more headroom means fewer splits" argument holds but is worth
+> less than it sounds: quartering the accumulator share grew a checkpoint
+> set by 6%, because voxel overlap is concentrated among *neighbouring*
+> frames — the crystal rotates, so frames far apart touch largely
+> disjoint reciprocal space, and a smaller window already captures nearly
+> all the deduplication. Mapping was also slightly *faster* with the
+> smaller share. See `reciprocal_space_performance_findings.md`.
+
+Worked example (real job, one arbitrary choice of `ram_budget_bytes=24 GB`
+picked to illustrate the formula, not a recommended value — see the note
+at the top of this document — with `files_per_job=10` from the checkpoint
+floor dominating): each active checkpoint gets `24/2 = 12 GB`. The
+calibrated per-checkpoint estimate was ~12.97 GB raw — right at the edge,
+meaning this specific combination would occasionally trip the safety valve
+near the end of a checkpoint, producing 11 files for that run instead of
+10. That's the safety valve working as intended at the margin, not a bug:
+the calibration is close but not exact, and the system self-corrects
+instead of overrunning the real memory budget.
+
+### When `max_concurrent_active_checkpoints=2` isn't enough
+
+The "current + next" window only avoids idle-worker gaps if checkpoint N
+finishes committing (its final tree-merge fold and write, §9) *before*
+checkpoint N+1's frames are fully claimed and drained. If N+1 drains
+faster than N's tail-flush completes, workers run out of claimable work
+and stall — the window can't advance to N+2 until N is actually done, not
+just fully assigned. This is a real risk exactly in the regime this
+document keeps returning to: at `max_depth=0`, §7's rule pushes
+`image_workers` high and per-frame cost is tens of milliseconds, so a
+365-frame checkpoint can drain in a couple hundred milliseconds — very
+plausibly faster than N's tail-flush, which is small relative to the
+whole checkpoint's merge cost (most of the tree-merge already happened
+incrementally during accumulation) but not necessarily small in absolute
+terms.
+
+Unlike the I/O rate in §7, this isn't something that needs live
+adaptation. I/O rate depends on external, unpredictable factors (network
+conditions, other users' concurrent cluster load) that can only be
+discovered at runtime. This doesn't: `image_workers` (§7),
+frames-per-checkpoint (from `files_per_job` and total frame count), and
+per-frame cost (§6 probe) are all already known before the job runs, so
+whether checkpoint drain time (`frame_batch_size * per_frame_time /
+image_workers`) will outrun the estimated tail-flush latency is a
+comparison that can be made once, at prepare time, from numbers already
+in hand — no second runtime control loop needed, *provided* tail-flush
+latency itself has an estimate. It didn't have one; here's one that
+doesn't hardcode this session's specific measured numbers.
+
+**Tail-flush latency formula.** The tail fold combines whatever tree
+levels remain when the checkpoint's last frame lands — in the worst case
+(a single large final fold), that touches close to the checkpoint's full
+row count. A conservative (safe-to-overestimate) bound:
+
+```
+tail_flush_latency ~= checkpoint_final_rows / merge_rows_per_second
+```
+
+where:
+
+- `checkpoint_final_rows = (job_data_bytes / files_per_job) / bytes_per_row`
+  — already derivable from quantities §5/§6 establish (the per-checkpoint
+  share of `job_data_bytes`) and §8's fixed 48-bytes-per-row schema. No
+  new measurement needed for this half.
+- `merge_rows_per_second` — the throughput of `_merge_sorted_batches`
+  itself, which is a property of the machine running the job, not of this
+  session's benchmark runs. It should be measured with a small local
+  calibration call (merge two ~100k-row synthetic batches, time it, divide)
+  at prepare time — the same "measure it on this machine" pattern already
+  used by the §6 probe and the §7 `per_call_time(kernel_threads)` sweep,
+  applied once more to a third operation. This is a cheap, sub-second
+  calibration (the §9 benchmark's own numbers, e.g. ~739,791-row batches
+  merging in a fraction of a second, show `_merge_sorted_batches`
+  throughput is high relative to typical checkpoint row counts — but that
+  specific rate is this test machine's, illustrative of the *order of
+  magnitude* only, not a value to hardcode into the formula).
+
+This keeps both halves of the comparison — checkpoint drain time and
+tail-flush latency — built from quantities either already known or cheap
+to measure locally, portable to whatever machine actually runs the job.
+
+**Resolution**: a prepare-time check, not a runtime one, and not a
+primary UI setting:
+
+- If the computed checkpoint drain time comes out below the estimated
+  tail-flush latency, auto-bump `max_concurrent_active_checkpoints` (e.g.
+  2 -> 3 or 4, capped low) as part of preparing the job — automatic, not
+  something the user needs to notice or request.
+- The consequence (smaller `active_budget_per_accumulator`, more frequent
+  safety-valve splits, likely a few more files than the calibrated
+  `files_per_job` estimate) should be visible in §14's UI reporting, not
+  silent.
+- A manual advanced override stays available for cases the heuristic gets
+  wrong, following the same precedent as `advanced_depth` overriding
+  `accuracy` elsewhere in the job config — but it is not a primary
+  setting, since the default path is expected to get this right without
+  the user needing to know the parameter exists.
+
+## 11. Resume and staleness
+
+### Invalidation
+
+`checkpoint_count` becomes a new field on `_ReconstructionSpec`, alongside
+`ram_budget_bytes` (already present as `memory_budget_bytes`). Both then
+participate in the existing `spec.digest` mechanism unchanged — the same
+mechanism that already invalidates manifests on any scientific-setting
+change today. No new invalidation philosophy, just two new fields feeding
+an existing one.
+
+Checkpoint frame-range boundaries are computed once at prepare time (from
+the calibration estimate) and **persisted**, not recomputed on resume —
+recomputing from a fresh (sampling-variant) calibration run could shift
+boundaries between two runs with identical user settings, spuriously
+invalidating good checkpoints.
+
+**No backward compatibility with existing prepared jobs.** This code
+hasn't shipped to users yet, so job files prepared under the current
+schema (`JOB_SCHEMA_VERSION=3`, Parquet-based, `partition_chunk_span`,
+static `threads_per_image`) are not a migration concern. A schema version
+bump simply makes them unreadable by the new code — re-preparing from
+scratch is an acceptable answer, not something this design needs to
+smooth over.
+
+### Atomicity
+
+Checkpoint HDF5 files use the same write-`.tmp`-then-`.replace()` pattern
+already used twice in the codebase (`_write_manifest`,
+`_finalize_reconstruction`) — a crash mid-write leaves an orphaned `.tmp`,
+never a corrupt file at the real name.
+
+### Simplification: no separate manifest registry
+
+A checkpoint file's presence at its final name already implies
+completeness (atomic rename guarantees this). Each checkpoint stores its
+own spec digest as an HDF5 attribute (same pattern as
+`process.attrs["spec_sha256"]` on the final output today). Resume becomes:
+for each persisted checkpoint boundary, does the file exist at its final
+name with a matching digest attribute? Given file counts are now small
+(10-30s, not thousands) and checksums are now XXH3-fast, resume can also
+cheaply re-verify each existing checkpoint's data checksum every time as a
+belt-and-suspenders check against silent corruption — something too slow
+to do routinely under the old design. This removes an entire layer of
+JSON-manifest-list bookkeeping (`job.map_manifests`) that only existed
+because the old design had thousands of files to track.
+
+## 12. Multi-grid jobs
+
+`files_per_job` is computed **per grid**, not shared — grids can have very
+different data volumes (a coarse overview grid vs. a fine ROI grid), and
+forcing shared boundaries would either over-fragment the smaller grid or
+under-protect the larger one's memory footprint. Total file count for the
+job is the sum across grids (e.g. 3 grids at the checkpoint floor = 30
+files — still comfortably "tens, not thousands").
+
+Different grids' checkpoint boundaries may differ freely. A worker
+processing frame F routes that frame's per-grid results to whichever of
+*that grid's own* checkpoint boundaries currently contains F (a per-grid
+lookup) — this doesn't complicate frame-level scheduling. The §9
+tree-merge accumulator generalizes from "per checkpoint" to "per
+(checkpoint, grid)" with no new mechanism. The §10 memory-division formula
+gets a `number_of_grids` factor since all requested grids' accumulators
+for the active checkpoint(s) are alive in memory simultaneously (one
+corrected image feeds a kernel call per grid before moving to the next
+frame).
+
+## 13. Cluster (multi-node) coordination
+
+No new synchronization primitive needed. Frame ranges are disjoint across
+nodes (assigned up front), and each frame's contribution to the final grid
+is independent of every other frame's — so nodes need zero communication
+during mapping. Each node runs the full single-node design (§5-11)
+independently against its own slice and its own `ram_budget_bytes`.
+
+`reconstruction_cluster.py`'s `generate_cluster_scripts` already submits a
+finalize job with a scheduler-level dependency on the whole map array
+completing (`qsub -hold_jid` / `sbatch --dependency=afterok:...`) — this is
+the only synchronization point, and it already exists. The redesign
+changes what finalize processes (a small number of HDF5 checkpoints
+instead of thousands of Parquet shards, merged via the same §9 tree
+structure) but not the barrier mechanism itself.
+
+Remaining concerns are existing patterns, not new risks:
+
+- **Filename collisions on shared scratch**: avoided by construction —
+  each node's checkpoint files are named from its own frame-range/job
+  identity, so disjoint slices can't collide.
+- **Partial writes visible to other processes**: covered by the same
+  atomic rename from §11, which the current code already trusts for
+  manifests on the same (NFS) shared storage.
+- **A node crashing mid-slice**: handled by the node's own resume logic
+  (§11) plus the scheduler simply not firing the finalize dependency until
+  every array task reports success; a failed node just gets resubmitted.
+
+## 14. UI reporting
+
+Implemented (`ReconstructionDialog._refresh_file_count_summary`,
+`orgui/app/ReconstructionDialog.py`). At prepare time, before the job is
+committed, the calibration probe (§6, via `estimate_checkpoint_plan`)
+runs and the UI reports, live-updating as the user adjusts the grid
+table, `ram_budget_bytes` / `checkpoint_count`, accuracy, angle fallback,
+or cluster node count:
+
+- `number_of_files` (total, summed across grids -- and, when the Cluster
+  tab's node count is more than 1, a second reported total for that
+  many nodes, since the same prepared job can still be run either
+  locally or as a cluster job)
+- `number_of_jobs` (cluster array size, shown alongside the cluster
+  total, not unconditionally -- a job's cluster settings default to a
+  node count regardless of whether the user ends up running it locally)
+- `files_per_job` (per node; per-grid breakdown shown only when more
+  than one grid is requested)
+
+This goes in `ReconstructionDialog`'s **"Output grids"** tab
+(`_grid_tab()`), alongside the existing per-axis grid bounds — the natural
+place for it today, since file/job counts are a direct consequence of the
+grid the user is defining right there, not because it's the ideal final
+home. `ReconstructionDialog`'s tab layout more broadly (Experiment /
+Output grids / Performance / Cluster / Job and output / Preview and
+status) is expected to need a rework once this whole redesign lands, given
+how much of what "Performance" and "Cluster" mean is changing — that
+rework is out of scope here and left for later.
+
+## 15. Worked example (real job)
+
+The `ram_budget_bytes` values in the table below (4/8/13/24/128 GB) are
+arbitrary points chosen to show the formula's shape across a range, not
+recommended budgets — see the note at the top of this document.
+`39_1-rsmap`, Pilatus 6M, La3Ni2O7, 3651 frames, `max_depth=0`. Measured
+`final_records = 739,791` for a representative frame -> 35.5 MB/frame
+(post-dedup) -> ~129.7 GB total job data (pre-final-merge; the conservative
+planning number, since cross-frame overlap reduction can't be assumed
+ahead of time). Notably this volume is roughly depth-independent
+(`final_records` at `max_depth=3` was 985,539 for the same frame, same
+order of magnitude) — confirming the formula correctly decouples file
+count from both grid-chunk count and accuracy preset, the two things it
+was conflated with before.
+
+| `ram_budget_bytes` | `files_for_memory` | `files_per_job` (floor=10) |
+|--------------------:|--------------------:|-----------------------------:|
+|                4 GB |                  33 |                           33 |
+|                8 GB |                  17 |                           17 |
+|               13 GB |                  10 |                           10 |
+|               24 GB |                   6 |                       **10** |
+|              128 GB |                   2 |                       **10** |
+
+Split 4 ways across a cluster (~913 frames/node): ~32.4 GB/node; at 8
+GB/node budget, `ceil(32.4/8)=5` -> checkpoint floor wins -> 10 files/node
+x 4 nodes = **40 files total**, versus today's 4096.
+
+## 16. Migration summary
+
+| | current | proposed |
+|---|---|---|
+| map-phase file boundary | grid-chunk buckets (`partition_chunk_span`) | contiguous frame-range checkpoints |
+| reduce-phase files | one per `(chunk_id, shard)`, up to thousands | none — folded into the checkpoint write itself |
+| format | Parquet (optional dependency) | HDF5 (`bitshuffle-lz4`, already a hard dependency) |
+| checksum | SHA-256, re-read after write | XXH3-128, computed inline |
+| cross-frame merge | ad hoc, per flush cycle | tree/binary-counter, per (checkpoint, grid) |
+| resume tracking | JSON manifest list per partition/shard | checkpoint file presence + embedded digest attribute |
+| `threads_per_image` | static, inherited across accuracy presets | `kernel_threads`/`image_workers` jointly balanced live against an adaptively-measured I/O rate, plus a small adaptive prefetch pool (§7) |
+| images per native call | one | measured per job; several adjacent images share one call where the scan's geometry makes them overlap in reciprocal space |
+
+## 17. Open items
+
+- ~~Run the §7 `per_call_time(kernel_threads)` sweep at `max_depth=2/3`~~ —
+  done. Confirmed: at a 256x256 (65,536 px) tile, depth=2 scales cleanly to
+  16 threads (0.599s->0.105s) and depth=3 likewise (2.917s->0.459s), both
+  flattening between 16 and 24 threads; the same tile size at `max_depth=0`
+  showed no real scaling at all (noise-level, ~6ms flat). Confirms the
+  plateau genuinely shifts higher with depth rather than being asserted by
+  extrapolation — the balancing rule's candidate `kernel_threads` set needs
+  to be depth-specific (e.g. roughly {1,2,4,8} at depth 0 vs {1,4,8,16,24+}
+  at depth 3), which §7's design already accommodates by treating
+  `per_call_time(kernel_threads)` as a per-depth input. Not yet measured at
+  full-frame (6.2M px) scale, where the depth-2/3 plateau likely sits
+  higher still than 16.
+- ~~The exact form of the §7 live I/O-rate and `kernel_threads`/
+  `image_workers` adaptation~~ — resolved: `blocked_fraction` signal,
+  time-based re-evaluation cadence, asymmetric hysteresis bands (grow the
+  prefetch pool eagerly, shrink it cautiously), and a separate
+  compute-bounded prefetch *queue depth* (`N ~= image_workers +
+  small_constant`) distinct from prefetch *pool size*. The specific
+  threshold values (e.g. 20%/2% blocked-fraction bands, exact cadence in
+  seconds) are illustrative, not tuned against real production load — that
+  residual calibration is implementation work, not an open design
+  question.
+- ~~Whether `max_concurrent_active_checkpoints` should ever be exposed as
+  an advanced override~~ — resolved: a prepare-time check (checkpoint
+  drain time vs. tail-flush latency) auto-adjusts it when needed, surfaced
+  in §14's UI reporting; a manual advanced override remains available
+  (matching the `advanced_depth`/`accuracy` precedent) for cases the
+  heuristic gets wrong, but it is not a primary setting. Tail-flush
+  latency now has an explicit formula (§10) built from §5/§6/§8 quantities
+  plus one additional cheap local calibration (`_merge_sorted_batches`
+  throughput on this machine) — not hardcoded to this session's measured
+  numbers.
+- **Frame grouping changes what §7 balances, and §7 has not been rewritten
+  around it.** Above one image per call the joint
+  `image_workers`/`kernel_threads` problem is replaced by concurrent group
+  calls plus a prepare pool; §7 carries a note saying so but still reads
+  as though the joint balance is the only policy. Rewriting it properly
+  needs the phase 4 measurements first, since the balance above depth 0 is
+  unmeasured. Meanwhile the two paths coexist and the per-frame one is
+  unchanged.
+- **The group-size and concurrency constants rest on one job.** The
+  one-voxel travel limit comes from a striding sweep on a single scan; the
+  floor of three concurrent calls is fitted to one throughput curve. Both
+  reproduce that job's optimum and neither has been tested on a second
+  dataset or a second machine.
+- ~~Exact HDF5 layout for checkpoint files~~ — resolved: columnar (separate
+  1D datasets per record field), `chunk=65536` rows. Measured against a
+  compound/table dataset across four chunk sizes; columnar at 65536 was
+  both the best-performing configuration and the one requiring no
+  array-assembly/disassembly conversion against the in-memory
+  representation already used everywhere else in the design (§8).

From 372c6c8fb8953d184e68a95fe266426b0c7e12f0 Mon Sep 17 00:00:00 2001
From: "T. Fuchs" <tfuchs@cornell.edu>
Date: Tue, 11 Aug 2026 18:17:33 -0400
Subject: [PATCH 21/33] test: re-derive the work block when the benchmark
 overrides depth

--depth replaced max_depth alone, leaving work_block_pixels at whatever
the job's own accuracy implied. The preset halves with depth to hold the
native working set fixed, so a depth-0 arm ran an eight-times-too-small
block -- and frame-group brick dimensions are cut from that same count,
which biased the one comparison this benchmark exists to make.
---
 .../benchmark_reconstruction_pipeline.py       | 18 +++++++++++++++++-
 1 file changed, 17 insertions(+), 1 deletion(-)

diff --git a/benchmarks/benchmark_reconstruction_pipeline.py b/benchmarks/benchmark_reconstruction_pipeline.py
index e457294..eb44ce9 100644
--- a/benchmarks/benchmark_reconstruction_pipeline.py
+++ b/benchmarks/benchmark_reconstruction_pipeline.py
@@ -56,6 +56,7 @@
     _load_assets,
     _map_pending_ranges,
     read_job,
+    resolve_work_block_pixels,
     split_memory_budget,
 )
 
@@ -232,7 +233,22 @@ def main():
     scan = job.scan
     spec = job.internal_spec()
     if arguments.depth is not None:
-        spec = replace(spec, max_depth=arguments.depth)
+        # Re-derive the work block, exactly as ``internal_spec`` does.
+        # Overriding ``max_depth`` alone leaves the block at the job's own
+        # depth -- a documented way to measure something no real job runs:
+        # the preset halves with depth to hold the working set fixed, so a
+        # depth-0 arm carrying a depth-3 block is eight times too small,
+        # and frame-group brick dimensions are cut from that same count.
+        spec = replace(
+            spec,
+            max_depth=arguments.depth,
+            work_block_pixels=resolve_work_block_pixels(
+                job.work_block_pixels,
+                arguments.depth,
+                spec.memory_budget_bytes,
+                spec.threads,
+            ),
+        )
     if arguments.group is not None:
         if not hasattr(spec, "frames_per_group"):
             raise SystemExit(

From 32f5058489cfecff270f5bee9f2d1e13687607ff Mon Sep 17 00:00:00 2001
From: "T. Fuchs" <tfuchs@cornell.edu>
Date: Tue, 11 Aug 2026 18:17:38 -0400
Subject: [PATCH 22/33] test: add a benchmark for how depth changes
 reconstructed intensities

Record counts converging with depth says nothing about whether an
intensity moves. This compares weighted_intensity / weight across
depths against each voxel's own propagated error bar, and reports the
exposure at which a difference would become visible -- which is the
answer independent of how long the probe could afford to run.

Unlike the other reconstruction benchmarks it compares numbers rather
than times, so it needs no interleaving, repeats or a quiet machine.
---
 ...chmark_reconstruction_depth_convergence.py | 291 ++++++++++++++++++
 1 file changed, 291 insertions(+)
 create mode 100644 benchmarks/benchmark_reconstruction_depth_convergence.py

diff --git a/benchmarks/benchmark_reconstruction_depth_convergence.py b/benchmarks/benchmark_reconstruction_depth_convergence.py
new file mode 100644
index 0000000..d0d9c63
--- /dev/null
+++ b/benchmarks/benchmark_reconstruction_depth_convergence.py
@@ -0,0 +1,291 @@
+"""Does raising the subdivision depth change a reconstructed intensity?
+
+``doc/design/reciprocal_space_mapping_locality_and_subdivision.md``
+observes that emitted records converge at depth 3 (0.632 -> 0.650 across
+depths 3-6) while cost rises 3x/6x/13x, and asks whether ``very_high`` and
+``maximum`` are worth optimising at all. That is a question about
+intensities, not about record counts: a voxel can gain a finer
+partial-volume weight without its intensity moving anywhere a measurement
+could see.
+
+So this compares the quantity a user actually reads --
+``weighted_intensity / weight``, exactly as ``_reduce_chunk`` forms it --
+across depths, against that same voxel's own propagated error bar
+``sqrt(weighted_variance) / weight``. The statistic is the pull
+
+    (I_d - I_ref) / sigma_ref
+
+which is dimensionless and already scaled by what the measurement can
+resolve. Because sigma falls as 1/sqrt(contributors) while a systematic
+from the subdivision does not fall at all, the pull measured here also
+says at what exposure the difference *would* become visible:
+
+    contributors_critical = contributors_now / pull**2
+
+That last number is the honest answer to "measurably", independent of how
+many frames this particular probe could afford to run. Calibrate it
+against the exposure a real job reaches: this probe's 64 frames give
+20-45 contributors per voxel, and a 3651-frame job gives roughly
+1,000-2,500.
+
+Unlike the other reconstruction benchmarks this one compares numbers
+rather than times, so it is immune to the machine noise documented in
+``reciprocal_space_performance_findings.md`` and needs no interleaving or
+repeats.
+
+**Run this with ``PYTHONPATH`` pointing at the checkout.** ``python
+benchmarks/<script>.py`` puts ``benchmarks/`` on ``sys.path``, not the
+repository root, so a pip-installed ``orgui`` shadows the working tree.
+The resolved extension path is reported for that reason.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+from pathlib import Path
+from time import perf_counter
+
+import numpy as np
+
+from orgui.datautils.xrayutils.reconstruction import (
+    _detector_corner_rays,
+    _native_module,
+)
+
+
+def _arguments():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("job", type=Path)
+    parser.add_argument("--frame", type=int, default=1800)
+    parser.add_argument("--frames", type=int, default=64)
+    parser.add_argument("--rows", type=int, default=192)
+    parser.add_argument("--columns", type=int, default=192)
+    parser.add_argument("--row-origin", type=int, default=1100)
+    parser.add_argument("--column-origin", type=int, default=1100)
+    parser.add_argument("--depths", type=int, nargs="+", default=[0, 1, 2, 3, 4, 5])
+    parser.add_argument("--threads", type=int, default=12)
+    parser.add_argument("--output", type=Path)
+    return parser.parse_args()
+
+
+def _case(job_path, frames, tile):
+    """Corrected tile arrays and exposure bounds for a run of frames."""
+    from orgui.reconstruction_job import (
+        _correction_pipeline,
+        _load_assets,
+        read_job,
+    )
+
+    job = read_job(job_path)
+    config = job.config_data
+    scan = job.scan
+    spec = job.internal_spec()
+    grid = spec.grids[0]
+    bounds = np.asarray(
+        scan.exposure_angle_bounds(config, fallback=job.angle_fallback),
+        dtype=np.float64,
+    )
+    correct = _correction_pipeline(config, scan, _load_assets(job), {})
+    selection = np.s_[tile[0] : tile[1], tile[2] : tile[3]]
+    intensity = []
+    variance = []
+    mask = []
+    for frame in frames:
+        payload = scan.get_raw_img(frame)
+        corrected = correct.correct_frame(payload, np.asarray(payload.img), frame)
+        intensity.append(np.ascontiguousarray(corrected[0][selection], np.float64))
+        variance.append(np.ascontiguousarray(corrected[1][selection], np.float64))
+        mask.append(np.ascontiguousarray(corrected[2][selection], bool))
+    ub = np.asarray(config.ub_calculator.getUB(), dtype=np.float64)
+    u = np.asarray(config.ub_calculator.getU(), dtype=np.float64)
+    return {
+        "rays": _detector_corner_rays(config.detector, tile),
+        "bounds": bounds,
+        "frames": list(frames),
+        "intensity": intensity,
+        "variance": variance,
+        "mask": mask,
+        "minimum": np.asarray(grid.minimum),
+        "step": np.asarray(grid.step),
+        "shape": np.asarray(grid.shape, dtype=np.int64),
+        "chunk_shape": np.asarray(grid.chunk_shape, dtype=np.int64),
+        "frame_name": grid.frame,
+        "wavevector": float(config.ub_calculator.getK()),
+        "ub_inverse": np.ascontiguousarray(np.linalg.inv(ub)),
+        "u_inverse": np.ascontiguousarray(np.linalg.inv(u)),
+        "angle_fallback": job.angle_fallback,
+    }
+
+
+def _accumulate(case, max_depth, threads):
+    """Merge every frame's records at one depth into per-voxel sums.
+
+    Frames are accumulated one at a time and summed in Python rather than
+    through ``accumulate_group``: the question here is what depth does,
+    and mixing in the grouped kernel's different summation order would
+    put a second variable in the comparison.
+    """
+    native = _native_module()
+    kernel = native.ReconstructionKernel(
+        case["minimum"],
+        case["step"],
+        case["shape"],
+        case["chunk_shape"],
+        case["frame_name"],
+        case["wavevector"],
+        case["ub_inverse"],
+        case["u_inverse"],
+        max_depth,
+        threads,
+        max(1, 16384 >> max_depth),
+        1 << 50,
+    )
+    keys = []
+    columns = {name: [] for name in
+               ("weighted_intensity", "weighted_variance", "weight", "contributors")}
+    started = perf_counter()
+    for position, frame in enumerate(case["frames"]):
+        result = kernel.accumulate(
+            case["intensity"][position],
+            case["variance"][position],
+            case["mask"][position],
+            case["rays"],
+            np.ascontiguousarray(case["bounds"][frame, 0]),
+            np.ascontiguousarray(case["bounds"][frame, 1]),
+            False,
+        )
+        keys.append(
+            (np.asarray(result["chunk_id"]).astype(np.int64) << np.int64(32))
+            | np.asarray(result["local_voxel_id"]).astype(np.int64)
+        )
+        for name in columns:
+            columns[name].append(np.asarray(result[name]).astype(np.float64))
+    elapsed = perf_counter() - started
+    key = np.concatenate(keys)
+    unique, inverse = np.unique(key, return_inverse=True)
+    totals = {
+        name: np.bincount(
+            inverse, weights=np.concatenate(values), minlength=unique.size
+        )
+        for name, values in columns.items()
+    }
+    return {
+        "key": unique,
+        "seconds": elapsed,
+        **totals,
+    }
+
+
+def _compare(current, reference):
+    """Pull of one depth's intensities against a deeper reference."""
+    shared, here, there = np.intersect1d(
+        current["key"], reference["key"], return_indices=True
+    )
+    weight_here = current["weight"][here]
+    weight_there = reference["weight"][there]
+    live = (weight_here > 0) & (weight_there > 0)
+    intensity_here = current["weighted_intensity"][here][live] / weight_here[live]
+    intensity_there = (
+        reference["weighted_intensity"][there][live] / weight_there[live]
+    )
+    sigma = (
+        np.sqrt(reference["weighted_variance"][there][live]) / weight_there[live]
+    )
+    usable = sigma > 0
+    pull = np.abs(intensity_here[usable] - intensity_there[usable]) / sigma[usable]
+    weights = weight_there[live][usable]
+    contributors = reference["contributors"][there][live][usable]
+    order = np.argsort(pull)
+    cumulative = np.cumsum(weights[order])
+    total = cumulative[-1] if cumulative.size else 1.0
+
+    def weighted_quantile(fraction):
+        if not cumulative.size:
+            return float("nan")
+        return float(pull[order][np.searchsorted(cumulative, fraction * total)])
+
+    median_pull = weighted_quantile(0.5)
+    mean_contributors = float(np.average(contributors, weights=weights))
+    return {
+        "shared_voxels": int(shared.size),
+        "voxels_here_only": int(current["key"].size - shared.size),
+        "voxels_reference_only": int(reference["key"].size - shared.size),
+        "weight_shared_fraction": float(
+            weight_here[live].sum() / max(1e-30, current["weight"].sum())
+        ),
+        "mean_contributors": mean_contributors,
+        "pull_median": median_pull,
+        "pull_p90": weighted_quantile(0.9),
+        "pull_p99": weighted_quantile(0.99),
+        "pull_rms": float(
+            np.sqrt(np.average(pull**2, weights=weights))
+        ),
+        # Contributors at which this depth's systematic would equal the
+        # statistical error bar. Beyond the probe's own exposure, so it is
+        # an extrapolation -- but the scaling it rests on (sigma ~
+        # 1/sqrt(N), systematic flat) is not in doubt.
+        "contributors_for_unit_pull": (
+            float(mean_contributors / median_pull**2)
+            if median_pull > 0
+            else float("inf")
+        ),
+        "total_weight_ratio": float(
+            current["weight"].sum() / max(1e-30, reference["weight"].sum())
+        ),
+        "total_intensity_ratio": float(
+            current["weighted_intensity"].sum()
+            / max(1e-30, reference["weighted_intensity"].sum())
+        ),
+    }
+
+
+def main():
+    """Compare reconstructed intensities across subdivision depths."""
+    arguments = _arguments()
+    tile = (
+        arguments.row_origin,
+        arguments.row_origin + arguments.rows,
+        arguments.column_origin,
+        arguments.column_origin + arguments.columns,
+    )
+    frames = list(range(arguments.frame, arguments.frame + arguments.frames))
+    case = _case(arguments.job, frames, tile)
+    print(
+        json.dumps(
+            {
+                "job": str(arguments.job),
+                "tile": tile,
+                "frames": [frames[0], frames[-1] + 1],
+                "angle_fallback": case["angle_fallback"],
+                "native": _native_module().__file__,
+                "depths": arguments.depths,
+            },
+            sort_keys=True,
+        ),
+        flush=True,
+    )
+    depths = sorted(arguments.depths)
+    accumulated = {}
+    for depth in depths:
+        accumulated[depth] = _accumulate(case, depth, arguments.threads)
+        print(
+            f"depth {depth}: {accumulated[depth]['key'].size:,} voxels, "
+            f"{accumulated[depth]['seconds']:.1f} s",
+            flush=True,
+        )
+    reference = depths[-1]
+    report = []
+    for depth in depths[:-1]:
+        entry = {"depth": depth, "reference": reference}
+        entry.update(_compare(accumulated[depth], accumulated[reference]))
+        report.append(entry)
+        print(json.dumps(entry, sort_keys=True), flush=True)
+    if arguments.output:
+        arguments.output.write_text(
+            json.dumps(report, indent=2, sort_keys=True), encoding="utf-8"
+        )
+
+
+if __name__ == "__main__":
+    main()

From fea8aabba9b90d375d4f083eddbf2e9c7fa6b97c Mon Sep 17 00:00:00 2001
From: "T. Fuchs" <tfuchs@cornell.edu>
Date: Tue, 11 Aug 2026 18:17:44 -0400
Subject: [PATCH 23/33] docs: close phase 4 -- grouping does not pay above
 depth 0

Frame grouping's record saving transfers above depth 0 and even
improves there, but it stops converting into time: depth 1 and depth 2
are nulls on both grids, while the depth-0 control reproduces phase 3
at 0.872x with every pair favouring grouping. Above depth 0 each pixel
and frame still walks its own subdivision tree, so record density no
longer governs kernel time and no brick shape can recover it.
Pixel-outer iteration and re-derived brick dimensions are retired
unbuilt.

Records the prior question with it, which turned out to be the larger
result: depth 3 differs from depth 5 by 0.013-0.024 of a voxel's own
error bar, so balanced accuracy is enough for a full job and the three
settings above it are not distinguishable from it by the data.

Also records that the reference job on disk no longer has the grid
every phase 1-3 number was taken against, which invalidated a whole
sweep before it was noticed.
---
 ..._space_mapping_locality_and_subdivision.md | 296 ++++++++++++++----
 1 file changed, 240 insertions(+), 56 deletions(-)

diff --git a/doc/design/reciprocal_space_mapping_locality_and_subdivision.md b/doc/design/reciprocal_space_mapping_locality_and_subdivision.md
index 6950d89..2310926 100644
--- a/doc/design/reciprocal_space_mapping_locality_and_subdivision.md
+++ b/doc/design/reciprocal_space_mapping_locality_and_subdivision.md
@@ -18,7 +18,10 @@ to do to carry them.
   the pipeline wiring, the scheduler and the per-job group size are all
   implemented.** Records through the router fall to 0.657x at the group size
   chosen for the reference job, and end-to-end mapping to **0.88x**. See
-  phase 3.
+  phase 3. *Its reach is now bounded: the record saving survives at every
+  depth and grid measured, but it converts into time only where the kernel
+  dominates and record density dominates the kernel — depth 0 on a fine
+  grid. See phase 4.*
 - **B — subdivision cost.** Above depth 3 the shared-corner cache switched off
   and the kernel did several times more ray work than it needed to. A
   corner-passing recursion removes that, bit-for-bit. **Phase 1 is implemented**
@@ -39,8 +42,8 @@ Everything is on branch `perf/subdivision-corner-passing`, unpushed.
 |---|---|
 | 1 — corner-passing subdivision | done, bit-for-bit, 1.65–2.98x |
 | 2 — the `worst_leaves` memory bound | done |
-| 3 — frame-group bricks at depth 0 | done, steps 1–7; **0.88x end-to-end** |
-| 4 — frame-group bricks above depth 0 | **not started** |
+| 3 — frame-group bricks at depth 0 | done, steps 1–7; **0.88x end-to-end**, re-confirmed at 0.872x |
+| 4 — frame-group bricks above depth 0 | **measured, and closed without building it** |
 
 What a reader should know before touching it:
 
@@ -55,10 +58,17 @@ What a reader should know before touching it:
   every job, not only grouped ones.
 - **Read "Measuring any of this" before running a benchmark.** Two rounds
   of conclusions in this document had to be withdrawn for measurement
-  error, not for reasoning error.
-- **The next real question is phase 4's first measurement**, not phase 4's
-  design: `F = 1` against automatic `F` at depth 1 and 2, interleaved. The
-  incidental depth-2 numbers suggest grouping does less there.
+  error, not for reasoning error, and a third round in 2026-08 was
+  reinterpreted after the reference job turned out to have changed
+  underneath it.
+- **Phase 4 is closed, unbuilt.** Grouping's record saving transfers above
+  depth 0 and even improves (0.61–0.71x), but it stops converting into
+  time: depth 1 and depth 2 are nulls on both grids. Above depth 0 the
+  kernel's cost is the per-(pixel, frame) subdivision walk, which no brick
+  shape can share. See phase 4.
+- **Depth above 2 buys nothing observable**, which is a separate result
+  from the same round and bears on how much phases 1–2 are worth. See
+  "The prior question, answered".
 
 ## How this was measured
 
@@ -67,6 +77,19 @@ All numbers are from the `39_1-rsmap` job used throughout the findings document
 0.1 deg/frame over 365 deg, `center` accuracy = depth 0, 2000^3 hkl grid with
 steps [0.00457, 0.00208, 0.00265]).
 
+> **The job file on disk is no longer that job, and this cost a measurement
+> round in 2026-08.** `39_1-rsmap.json` now carries `high` accuracy and a
+> **1000^3** grid — every step exactly 2x the above. That is a different
+> regime, not a rescaling: voxels are 8x larger, a frame reaches ~3.5x
+> fewer of them, per-frame travel falls from 0.71 to ~0.35 voxels, and at
+> depth 0 the pipeline stops being kernel-bound altogether (~110 ms/frame,
+> using 7 of 24 cores, against a load-and-correct floor). Phase 3's whole
+> result is invisible on it. **Check the grid before comparing anything
+> here against a fresh run**, and if you need the documented
+> configuration, copy the job and halve every `step` — the grid's
+> `minimum`/`maximum` are unchanged, so the shape doubles on its own. Every
+> phase 4 number below was taken that way.
+
 Which voxel a pixel reaches depends only on geometry and the mask, never on the
 recorded intensity, so the depth-0 pixel-to-voxel map was reproduced in NumPy
 with the affine ray-to-coordinate transform extracted from `kernel.coordinate()`
@@ -268,6 +291,14 @@ inside a brick at all. **At depth 0, cache centre rays for the brick (24 KiB for
 32x32) and iterate frames-outer. Above depth 0, iterate pixels-outer and do not
 build a cross-frame ray cache.**
 
+> **Superseded above depth 0 by phase 4's measurement.** The second mode
+> this section calls for was never built, and should not be: grouping's
+> record saving does transfer above depth 0, but it no longer converts
+> into time, so there is nothing for a better brick loop to recover. The
+> reasoning here is sound and is kept as the analysis that made phase 4
+> worth measuring — it is the premise that the saving would still be
+> *worth* something above depth 0 that turned out to be false.
+
 ## Memory
 
 ### What finding B costs and frees
@@ -410,12 +441,48 @@ because of it.
 | `benchmark_reconstruction_subdivision.py` | kernel cost and evaluations per pixel by depth and exposure, with XXH3 fingerprints and a `--compare` mode | seconds |
 | `benchmark_reconstruction_group.py` | one `accumulate_group` call against per-image `accumulate`, by group size | seconds |
 | `benchmark_reconstruction_pipeline.py` | the whole mapping phase — reader pool, correction, kernel, router, checkpoint writes | ~1 min per 234-frame arm at depth 0 |
+| `benchmark_reconstruction_depth_convergence.py` | reconstructed intensities across depths, in units of each voxel's own error bar | ~7 min for depths 0–5 on one tile |
 
 `benchmark_reconstruction_mapping.py` is dead: it imports `_map_frame_range`
 and the Parquet writers, both retired. Do not resurrect it; the pipeline
 benchmark replaces it against the pipeline that exists.
 
-### Two traps that silently invalidate a run
+### Traps that silently invalidate a run
+
+**The reference job's own configuration drifts.** See the box under "How
+this was measured": the grid on disk is now 2x coarser than the one every
+phase 1-3 number was taken against, which moves the pipeline out of the
+kernel-bound regime entirely. A whole phase 4 sweep was run and had to be
+reinterpreted before this was noticed. Print the grid shape and step at
+the start of a session and compare them against the document you are
+reading.
+
+**Overriding `--depth` must re-derive `work_block_pixels`.** The findings
+document lists this as a trap and the pipeline benchmark had it anyway,
+until 2026-08: `--depth` replaced `max_depth` alone, leaving the block at
+whatever the job's *own* accuracy implied. The preset halves with depth
+(`16384 >> depth`, memory-capped) precisely to hold the working set fixed,
+so a depth-0 arm was running a 2048-pixel block — eight times too small —
+and frame-group brick dimensions are cut from that same count, so it
+biased the one comparison the benchmark exists to make. Fixed; `--depth`
+now calls `resolve_work_block_pixels` exactly as `internal_spec` does.
+
+**At depth 0 the arm is I/O-bound, and the page cache is the noise.** A
+234-frame window is 11.7 GB against 31 GB of RAM, and the mapping run
+itself holds several more, so how much of the window survives in the page
+cache between runs varies from one repeat to the next. Measured spread on
+a single fine-grid depth-0 arm: **176.5 to 340.6 ms/frame, a factor of
+1.9**, with the process using only 7 of 24 cores. Depth >= 1 is
+compute-bound and far better behaved (±7%). Three pairs is not enough at
+depth 0; use six, and expect the tails.
+
+**"Nothing else may run" needs checking, not assuming.** This machine
+carries a persistent ~1.8 cores of background load — a stray long-running
+process and the agent runtime itself — and a foreign test run landed in
+the middle of a sweep in 2026-08. Sampling total machine CPU against the
+benchmark child's own CPU during each run turns that from an invisible
+confounder into a per-run number you can gate on; a run showing a spike
+should be discarded and repeated, never corrected.
 
 **`PYTHONPATH` must point at the checkout.** `python benchmarks/<script>.py`
 puts `benchmarks/` on `sys.path`, *not* the repository root, so a
@@ -987,47 +1054,88 @@ record): the 3.4% block-shape gain is real in block records before the
 cross-block merge, but one call's merged output is the set of voxels that
 frame reached, which does not depend on how the frame was partitioned.
 
-### Phase 4 — Frame-group bricks above depth 0. Not started.
-
-Everything in phase 3 was designed and measured at depth 0. Grouping is not
-*disabled* above it — the gates run at any depth and the streamed scheduler
-works — but nothing about it has been tuned there, and the little evidence
-that exists is not encouraging.
-
-**What is already known, from the depth-2 runs taken while fixing band
-height.** At depth 2 on the reference job the group-size gate picks
-`F = 2`, not the `F = 4` it picks at depth 0, and mapping runs at
-~620 ms/frame against ~205 at depth 0. The reason `F` falls is structural:
-a group call's native working set is `F x tile pixels x bytes-per-pixel`,
-and the per-pixel term roughly quadruples from depth 0 to depth 2, so the
-same memory affords half the group at the same concurrency. Grouping and
-adaptive depth compete for the same budget.
-
-That was measured incidentally, on two arms that differed in band height as
-well, so treat it as an observation rather than a result. **The first real
-phase 4 measurement is the obvious one nobody has taken: `F = 1` against
-automatic `F` at depth 1 and depth 2, interleaved, three repeats.** Until
-that exists there is no evidence that grouping helps or hurts above depth 0,
-and the honest reading of the numbers above is that it does less.
-
-Only after that is the design work worth doing:
-
-- **Pixel-outer / frame-inner iteration.** Above depth 0 the per-pixel
-  subdivision state cannot be shared across frames (see "Where the two
-  meet"), so the brick loop needs its second mode. Corner-passing is what
-  makes this affordable at all — 1.2–3.5 KiB of stack instead of a
-  6.9–101 KiB dense array per pixel.
-- **No cross-frame ray cache.** The payoff is capped near 28% and the
-  cache is per pixel, so it can only live for one pixel at a time.
-- **Brick dimensions re-derived** against the phase 2 record bound rather
-  than the leaf count.
-- **Whether the record saving even survives.** At higher depth a pixel
-  resolves into a footprint rather than a point, so cross-frame overlap
-  should be at least as good — but "should" is doing all the work in that
-  sentence, and the records-converge-at-depth-3 observation below suggests
-  the marginal voxel is increasingly a partial-volume weight rather than a
-  new voxel. Measure the routed-record ratio at depth 1–3 before assuming
-  finding A transfers.
+### Phase 4 — Frame-group bricks above depth 0. Measured, and not worth doing.
+
+The measurement this section used to ask for has been taken. **Finding A
+transfers above depth 0 in full — and stops paying for itself entirely.**
+The design work below it is therefore not worth starting.
+
+**The record saving is real and gets better with depth.** Routed records,
+automatic `F` against `F = 1`, both grids:
+
+| grid | depth | `F` | routed records vs `F = 1` |
+|---|---|---|---|
+| 2000^3 | 0 | 4 | 0.665 |
+| 2000^3 | 1 | 2 | 0.714 |
+| 1000^3 | 0 | 4 | **0.450** |
+| 1000^3 | 1 | 2 | 0.620 |
+| 1000^3 | 2 | 2 | 0.613 |
+
+Note the coarse grid does *better*, which finding A predicts: per-frame
+travel is halved there, so consecutive frames overlap more. Records are
+deterministic, so these are exact, and they are unaffected by block size
+(the same depth-1 figure came out of a 2048- and an 8192-pixel block, as
+it must — one call's merged output is the set of voxels the frames
+reached, not a function of how the detector was partitioned).
+
+**The time saving does not survive depth 0.** Interleaved pairs, 234
+frames, six scheduling ranges so the per-frame arm reaches all six image
+workers the budget affords, every run's checkpoint fingerprint checked
+against its partner:
+
+| grid | depth | pair ratios | median | every pair favours grouping? |
+|---|---|---|---|---|
+| 2000^3 | **0** | 0.928, 0.671, 0.788, 0.966, 0.872 | **0.872** | **yes** |
+| 2000^3 | 1 | 0.993, 1.179, 1.047 | 1.047 | no — none do |
+| 1000^3 | 0 | 1.025, 0.878, 0.968 | 0.968 | no |
+| 1000^3 | 1 | 0.968, 0.971, 1.001 | 0.971 | no |
+| 1000^3 | 2 | 1.053, 0.909, 0.978 | 0.978 | no |
+
+The depth-0 row on the fine grid is the control, and it reproduces phase
+3's result cleanly: 0.872 against the 0.866-0.88 recorded there, at the
+same `F = 4` / 3 calls x 8 threads layout, with all five valid pairs
+below 1.0. Everything else is a null.
+
+**Why, and it is not the reason this section originally guessed.** The
+old text argued `F` falls at depth because a group call's working set
+grows — true (the gate picks 2 rather than 4 above depth 0) but not the
+cause. The cause is that **grouping removes emitted records, and above
+depth 0 emitted records are no longer where the time goes.** Each
+(pixel, frame) still walks its own subdivision tree; that walk is
+per-frame by construction and no brick shape can share it. It costs ~31
+coordinate evaluations per pixel at continuous depth 1 and ~149 at depth
+2, against ~1 at depth 0. So the fraction of kernel time that record
+density controls collapses exactly as the record saving improves, and
+the two cancel. A better brick cannot fix this; only making the
+subdivision itself cheaper can, which is what phase 1 already did.
+
+**A second null, independent of depth, is worth recording with it.** On
+the coarse 1000^3 grid, grouping buys nothing *even at depth 0* (0.968)
+despite the largest record saving measured anywhere (0.450). That job
+maps at ~110 ms/frame using 7 of 24 cores — it is bound by loading and
+correcting frames, not by the kernel at all. So grouping pays only where
+the kernel dominates *and* the kernel's cost is dominated by records:
+depth 0 on a grid fine enough to matter. That is a narrow window, and it
+is the one phase 3 already occupies.
+
+What this retires, and what it does not:
+
+- **Do not build pixel-outer / frame-inner iteration.** It exists to
+  share per-pixel subdivision state across frames above depth 0. There is
+  no time there to recover.
+- **Do not re-derive brick dimensions** against the phase 2 record bound.
+  Same reason.
+- **The cross-frame ray cache was already ruled out** on its own
+  ~28% ceiling; nothing here changes that.
+- **Grouping should stay on and stay automatic.** It is not a regression
+  above depth 0 — the nulls are nulls, not losses, and it remains a clear
+  win in the regime it was tuned for. The gate already declines to group
+  outside the geometric regime; it does not, and need not, decline on the
+  basis of depth.
+- **The concurrency floor of 3 is doing real work** and should not be
+  relaxed to let `F` grow above depth 0: `F` would rise, records would
+  fall further, and by the above that buys nothing while costing
+  concurrency.
 
 There is also a prior question worth settling first, because it may make
 the whole of phase 4 moot: **records converge at depth 3** (0.632 → 0.650
@@ -1035,6 +1143,80 @@ across depths 3–6 while cost rises 3x / 6x / 13x). If `very_high` and
 `maximum` do not change a reconstructed intensity measurably, the right
 answer is to discourage them rather than to optimise them.
 
+**Settled — see the next section. They do not.**
+
+## The prior question, answered: depth stops changing intensities at 2
+
+Record counts converging is suggestive but not the question. What a user
+reads is `weighted_intensity / weight`, and what decides whether a change
+to it is real is that voxel's own propagated error bar,
+`sqrt(weighted_variance) / weight`. So the statistic is the **pull**,
+
+    (I_depth - I_reference) / sigma_reference
+
+which is dimensionless and already scaled by what the measurement can
+resolve. Because sigma falls as `1/sqrt(contributors)` while a systematic
+from the subdivision does not fall at all, a pull measured at one exposure
+also says at what exposure the difference *would* become visible:
+`contributors_for_unit_pull = contributors / pull^2`. That number is the
+answer, independent of how long the probe could afford to run.
+
+Measured on real frames through the kernel itself, two independent tiles
+at different scan positions, every frame accumulated separately and summed
+in Python so that grouping's summation order is not a second variable:
+
+| depth | median pull | p90 | contributors for 1 sigma | voxels it never reaches |
+|---|---|---|---|---|
+| 0 | 0.525 / 0.412 | 1.43 / 1.15 | **72 / 261** | 4.3% |
+| 1 | 0.135 / 0.132 | 0.38 / 0.37 | 1,090 / 2,542 | 2.0% |
+| 2 | 0.051 / 0.057 | 0.13 / 0.15 | 7,688 / 13,816 | 0.9% |
+| 3 | 0.013 / 0.024 | 0.04 / 0.06 | **111,919 / 76,287** | 0.38% |
+| 4 | 0.003 | 0.009 | 2,084,863 | 0.13% |
+
+Two values per cell are the two tiles (192x192 over 64 frames, 19.7 and
+44.3 contributors per voxel); the reference is depth 5 for the first and
+depth 4 for the second. The last column agreed to within 0.1 points
+between the tiles, so it is given once. Total weight and total weighted
+intensity are conserved to 1.0 at every depth, which is what says this
+measures redistribution rather than a bug.
+
+The pull is taken over the voxels both depths reach, weighted by voxel
+weight so that well-measured voxels dominate. Voxels a shallower depth
+misses entirely are *not* in it — they are the last column, and they are a
+second, independent way in which depth 0 differs.
+
+**Calibrate it against a real job.** These probes used 64 frames; the job
+has 3651, so a well-covered voxel in a finished reconstruction gets
+roughly 1,000-2,500 contributors. Against that:
+
+- **`center` (0) → `low` (1) is a real difference.** It shows at 72-261
+  contributors, i.e. almost immediately, and depth 0 additionally fails to
+  reach 4.3% of the voxels deeper settings do.
+- **`low` (1) → `balanced` (2) sits exactly at the boundary**, 1,090-2,542.
+  Visible on a full-length job, marginal on a short one.
+- **`balanced` (2) → `high` (3) needs 7,700-13,800** — three to ten times
+  what a full job delivers.
+- **`high` (3) → `very_high` (4) → `maximum` (5) needs 76,000 to 2,000,000.**
+  Unreachable by any measurement the instrument can make, at 4.6x and 24x
+  the cost (3.2 s / 12-14 s / 37-64 s / 327 s for the same tile).
+
+So the honest recommendation is that **`balanced` is enough for a full job
+and the three settings above it are not distinguishable from it by the
+data itself.** That bears directly on how much phases 1 and 2 are worth:
+they made depths 4-5 affordable, and depths 4-5 turn out to buy nothing
+observable. Phase 1 keeps its value at depths 1-3, which is where jobs
+should actually run; it is the *justification* for optimising the top of
+the range that has gone.
+
+This does not argue for removing the settings — a user checking
+convergence for themselves is a legitimate thing to do, and the deeper
+runs are what make the table above possible. It argues for saying plainly
+in the user-facing documentation what they cost and what they buy.
+
+Instrument: `benchmarks/benchmark_reconstruction_depth_convergence.py`.
+Unlike everything else in this document it is immune to machine noise —
+it compares numbers, not times.
+
 ## Risks and open questions
 
 **Interlaced scans.** `orgui/backend/interlacedScanLoader.py` exists, so frames
@@ -1071,9 +1253,11 @@ sound in spirit — the prepare pool grows to its ceiling at F = 4 — but
 **MSVC inlining** of the recursion is an assumption; check the measurement, not
 the intent. The 19-point octree bookkeeping is the fiddly part.
 
-**Depth > 0 for finding A** is unmeasured, as is the interaction between swept
-exposures and cross-frame overlap above depth 0 — each frame then covers a
-0.71-voxel slab rather than a plane, and adjacent slabs tile rather than overlap.
+**Depth > 0 for finding A** is measured, and closed — see phase 4. The
+record saving transfers (0.61-0.71x) and buys no time. The interaction
+between swept exposures and cross-frame overlap was measured with it: all
+of these runs used `midpoint` bounds, so the slab-versus-plane concern is
+inside the measured numbers rather than beside them.
 
 **`excluded_frames`** must be skipped when forming groups, not merely masked
 inside them. *Resolved, and more cheaply than expected:* `_included_ranges`
@@ -1083,10 +1267,10 @@ to cut at checkpoint boundaries.
 
 **Records converge at depth 3.** 0.632 -> 0.642 -> 0.647 -> 0.650 across depths
 3-6, while cost rises 3x / 6x / 13x. Everything above depth 3 buys finer
-partial-volume weights, not new voxels. Whether `very_high` and `maximum` change
-a reconstructed intensity measurably is a separate question, but it is worth
-asking before optimising them — it bears directly on how much phases 1 and 2 are
-worth.
+partial-volume weights, not new voxels. *Resolved, and more sharply than
+this item expected: the convergence is in intensities too, and it starts
+lower. Depth 2 onward is indistinguishable from depth 5 by any measurement
+a full job can make. See "The prior question, answered".*
 
 ## Also
 

From a9a9fb4152e96bce83f551e4da50b520f3d67de6 Mon Sep 17 00:00:00 2001
From: "T. Fuchs" <tfuchs@cornell.edu>
Date: Tue, 11 Aug 2026 18:17:50 -0400
Subject: [PATCH 24/33] docs: record the phase 4 result and four measurement
 traps

Frame grouping pays only where the kernel dominates and record density
dominates the kernel, and nothing above balanced accuracy changes an
intensity measurably.

The traps are the ones that cost time this round. The reference job's
own grid has changed since these numbers were taken. The page cache,
not thermal state, dominates depth-0 noise, because the window is
11.7 GB against 31 GB of RAM. Foreign load must be summed per process:
taking it as machine-busy minus the benchmark charges the benchmark's
own I/O kernel time to something else, reporting 7.2 cores of
interference where per-process accounting showed 2.9. And a run that
maps nothing looks like a fast run, so a checkpoint fingerprint has to
be compared against its partner's.
---
 .../reciprocal_space_performance_findings.md  | 52 +++++++++++++++++++
 1 file changed, 52 insertions(+)

diff --git a/doc/design/reciprocal_space_performance_findings.md b/doc/design/reciprocal_space_performance_findings.md
index 5defb2d..73c5bd8 100644
--- a/doc/design/reciprocal_space_performance_findings.md
+++ b/doc/design/reciprocal_space_performance_findings.md
@@ -134,6 +134,24 @@ to paying for it. Between 421 and 506 rows, where concurrency is equal, the
 two are indistinguishable, which is the form the original finding should
 have taken.
 
+**Frame grouping pays only where the kernel dominates and records
+dominate the kernel.** Measured in 2026-08 across two grids and three
+depths: the record saving transfers everywhere and even improves with a
+coarser grid (0.450x at 1000^3 depth 0), but the *time* saving appears
+only at depth 0 on the fine 2000^3 grid (0.872x, every pair). Depth 1 and
+2 are nulls because each (pixel, frame) still walks its own subdivision
+tree — ~31 evaluations per pixel at continuous depth 1, ~149 at depth 2,
+against ~1 at depth 0 — so record density stops controlling kernel time.
+Coarse-grid depth 0 is a null for the opposite reason: it is not
+kernel-bound at all. Detail, and what it retires, in the locality
+document's phase 4.
+
+**Nothing above `balanced` changes a reconstructed intensity.** Depth 3
+differs from depth 5 by 0.013-0.024 of a voxel's own error bar, needing
+~10^5 contributors per voxel to reach 1 sigma against the ~1,000-2,500 a
+full job delivers. Depth 0 -> 1 is the only clearly real step. See the
+locality document, "The prior question, answered".
+
 Band height and group size also substitute for each other almost exactly —
 at depth 2, widening the bands makes the group-size gate halve the group
 and the two cancel — which is why neither repays hard optimisation. Detail
@@ -276,6 +294,40 @@ separation between arms is worth more than a difference of means. Two rounds
 of conclusions in the locality document were withdrawn for this. Its
 "Measuring any of this" section is the full version.
 
+**The reference job's own configuration drifts.** `39_1-rsmap.json` on
+disk is no longer the job the table at the top of this document
+describes: it now carries `high` accuracy and a 1000^3 grid, every step
+2x the documented one. That is a different regime — at depth 0 it maps at
+~110 ms/frame using 7 of 24 cores, bound by loading and correcting rather
+than by the kernel, so phase 3's whole result is invisible on it. A 2026-08
+sweep was run and had to be reinterpreted. Print the grid before trusting
+any comparison against numbers written here.
+
+**At depth 0 the page cache is the noise.** A 234-frame window is 11.7 GB
+against 31 GB of RAM and the run itself holds several more, so how much
+of the window survives between repeats varies. One fine-grid depth-0 arm
+spread 176.5 to 340.6 ms/frame — a factor of 1.9 — where depth >= 1 arms
+hold ±7%. Six pairs, not three, at depth 0.
+
+**"Nothing else may run" is worth verifying, and worth measuring
+correctly.** This machine carries ~1.8-2.9 cores of persistent background
+load. Sampling it per run turns that from an invisible confounder into a
+number you can gate on — but define it as *the sum of other user
+processes*, not as machine-busy minus the benchmark's own. Kernel,
+interrupt and storage-driver time is attributed to System rather than to
+the process that caused it, and mapping is I/O-heavy, so the
+total-minus-child definition charges the benchmark's own I/O to
+"foreign": it reported 7.2 cores of interference where per-process
+accounting showed 2.9.
+
+**A run that maps nothing looks like a fast run.** Roughly 2 of ~40
+pipeline runs in 2026-08 routed zero records, wrote a checkpoint with 0
+rows and `frames_covered` set to the full count, and exited 0 — at 50.5
+ms/frame against ~250, i.e. a 5x "speed-up". Always compare a run's
+checkpoint fingerprint against its partner's and discard mismatches; a
+timing-only harness will silently take these as wins. The underlying
+defect is a correctness issue and is not yet diagnosed.
+
 ## Calibration constants and what they rest on
 
 | Constant | Value | Basis |

From dfdf59c8e89203b8a764bf4cffdb77221398f826 Mon Sep 17 00:00:00 2001
From: "T. Fuchs" <tfuchs@cornell.edu>
Date: Tue, 11 Aug 2026 18:17:56 -0400
Subject: [PATCH 25/33] docs: say which reconstruction accuracy setting to
 choose

The footprint accuracy settings were described by what they do, not by
what they are worth. Each one's effect on a reconstructed intensity is
now given against that voxel's own counting error, together with the
exposure needed to see it.

Balanced is sufficient for a full rotation scan, which is why it is the
default. High, Very high and Maximum shift an intensity by 0.02 sigma
or less and are not distinguishable from Balanced by the data, at
roughly 4x, 15x and 100x the runtime. Center only is genuinely lossy
and also misses about 4% of the voxels the other settings reach.

No behaviour or default changes.
---
 CHANGELOG.md                                  |  9 ++++
 .../reciprocal_space_reconstruction.rst       | 51 +++++++++++++++++++
 doc/source/release_notes.rst                  |  5 ++
 3 files changed, 65 insertions(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 7b779f0..93c9fbd 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -102,6 +102,15 @@ Scientific correctness and performance fixes:
   giving 2527 native calls per image instead of six. Band height no longer
   depends on the accuracy setting. Measured throughput at ``balanced``
   accuracy is unchanged; the fix matters at the two highest settings.
+- The reciprocal-space reconstruction documentation now says which footprint
+  accuracy setting to choose, and why. Each setting's effect on a
+  reconstructed intensity was measured against that voxel's own counting
+  error: ``Balanced`` is sufficient for a full rotation scan, while
+  ``High``, ``Very high`` and ``Maximum`` shift intensities by 0.02 sigma or
+  less and are not distinguishable from ``Balanced`` by the data, at roughly
+  4x, 15x and 100x the runtime. ``Center only`` is genuinely lossy and also
+  misses about 4% of the voxels the other settings reach. No behaviour or
+  default has changed.
 
 A ***critical bug*** was fixed that affects bulk CTR calculations:
 
diff --git a/doc/source/reciprocal_space_reconstruction.rst b/doc/source/reciprocal_space_reconstruction.rst
index b67a176..a89a634 100644
--- a/doc/source/reciprocal_space_reconstruction.rst
+++ b/doc/source/reciprocal_space_reconstruction.rst
@@ -291,6 +291,57 @@ Stationary cells split into four spatial children per level. Swept cells split
 in detector row, detector column, and exposure position and can create eight
 children per level. Consequently, worst-case work grows rapidly with depth.
 
+Choosing a depth
+~~~~~~~~~~~~~~~~
+
+Depth changes an intensity by less than counting statistics well before it
+stops costing runtime, so the highest settings are rarely worth their price.
+Measured on real rotation-scan data by comparing each voxel's reconstructed
+intensity against the same voxel at depth 5, in units of that voxel's own
+propagated error bar:
+
+.. list-table::
+   :header-rows: 1
+
+   * - Setting
+     - Typical shift from the deepest result
+     - Contributors per voxel needed to see it
+   * - ``Center only (depth 0)``
+     - 0.4-0.5 sigma
+     - 70-260
+   * - ``Low (depth 1)``
+     - 0.13 sigma
+     - 1,100-2,500
+   * - ``Balanced (depth 2)``
+     - 0.05 sigma
+     - 7,700-13,800
+   * - ``High (depth 3)``
+     - 0.01-0.02 sigma
+     - 76,000-112,000
+   * - ``Very high (depth 4)``
+     - 0.003 sigma
+     - ~2,000,000
+
+A full rotation scan of several thousand frames delivers of order 1,000 to
+2,500 contributors to a well-covered voxel. Reading the table against that:
+
+* ``Center only`` is genuinely lossy. It also fails to reach about 4% of the
+  voxels that deeper settings populate, because a pixel contributes at one
+  point rather than over its footprint.
+* ``Low`` sits at the edge of what a long scan can resolve.
+* ``Balanced`` is sufficient for production work on such a scan, which is why
+  it is the default.
+* ``High``, ``Very high`` and ``Maximum`` are not distinguishable from
+  ``Balanced`` by the data itself, while costing roughly 4x, 15x and 100x its
+  runtime respectively. Use them to check convergence on a small region, not
+  for whole production jobs.
+
+These figures come from one dataset on one machine and depend on the ratio of
+voxel size to detector pixel footprint: a grid much finer than the pixel
+footprint moves every threshold upward. Treat the ordering as general and the
+constants as indicative. ``benchmarks/benchmark_reconstruction_depth_convergence.py``
+reproduces the table for a prepared job of your own.
+
 Performance Parameters
 ----------------------
 
diff --git a/doc/source/release_notes.rst b/doc/source/release_notes.rst
index d18c57f..13fdcc8 100644
--- a/doc/source/release_notes.rst
+++ b/doc/source/release_notes.rst
@@ -26,6 +26,11 @@ Scientific correctness and performance fixes:
 - Corrected film and epitaxy-interface anchoring, stacking, and support ownership across strained interfaces.
 - Fixed missing support in one-dimensional electron-density calculations and corrected atomic-coordinate stacking when splitting unit cells into layers.
 - Added a C++ electron-density backend and bounded caches for atomic form factors, anomalous scattering factors, and accelerated form-factor lookup.
+- Reciprocal-space mapping now maps detector images through a work block shaped as a brick in (row, column, frame) rather than a run of one flattened image, and can map several consecutive images in a single native call. On a rotation scan two adjacent images land as close together in reciprocal space as two adjacent pixels do, so a brick merges contributions the per-image block could not see: on a 3651-frame Pilatus 6M scan, mapping eight images together emits 0.60x the records for the same samples. **This changes reconstructed values in the last bits, for every job, including jobs that map one image at a time.** Contributions now merge inside the kernel rather than in the checkpoint accumulator, so the same sums associate differently — with fewer intermediate roundings, so slightly better conditioned. Which voxels are reached, and how many detector samples reach each of them, are unchanged. Existing checkpoint files remain resumable, but a job resumed across this change will contain parts from both association orders.
+- Mapping several images per call uses a scheduler of its own, which runs a few concurrent native calls with the thread budget split between them and hoists whole-frame correction into the prefetch pool, so one call's GIL-held Python work overlaps the next call's native work instead of queueing behind it. Measured 0.87x the mapping wall time of the per-image pipeline at four images per call on a 3651-frame rotation scan.
+- The number of images per call is now measured per job rather than configured. Grouping only helps while consecutive images land within about one voxel of each other, so the job's own geometry is probed at three points in the scan to decide whether it does, and the group size is then the largest the memory budget can afford without giving up concurrency. Scans whose angles do not advance monotonically, such as interlaced ones, map one image per call as before. On a 0.1 deg/frame rotation scan this chooses four images per call and maps at 0.88x the previous wall time.
+- Corrected the detector band height used for reciprocal-space mapping, which was still derived from the worst-case adaptive leaf count rather than from the record ceiling the memory prechecks use. At the ``very_high`` and ``maximum`` accuracy settings this over-estimated a pixel's cost by four orders of magnitude and split the detector into extremely thin bands — one row per band on a Pilatus 6M at ``maximum``, giving 2527 native calls per image instead of six. Band height no longer depends on the accuracy setting. Measured throughput at ``balanced`` accuracy is unchanged; the fix matters at the two highest settings.
+- The reciprocal-space reconstruction documentation now says which footprint accuracy setting to choose, and why. Each setting's effect on a reconstructed intensity was measured against that voxel's own counting error: ``Balanced`` is sufficient for a full rotation scan, while ``High``, ``Very high`` and ``Maximum`` shift intensities by 0.02 sigma or less and are not distinguishable from ``Balanced`` by the data, at roughly 4x, 15x and 100x the runtime. ``Center only`` is genuinely lossy and also misses about 4% of the voxels the other settings reach. No behaviour or default has changed.
 
 A ***critical bug*** was fixed that affects bulk CTR calculations:
 

From 18e3e44fc84a9d874438ae374644fe10f2ce5516 Mon Sep 17 00:00:00 2001
From: "T. Fuchs" <tfuchs@cornell.edu>
Date: Tue, 11 Aug 2026 19:35:24 -0400
Subject: [PATCH 26/33] fix(phys): warn when a mapping run produces no records
 at all

Roughly one run in twenty mapped every frame, routed nothing, wrote a
checkpoint claiming its full frame count with zero rows, and exited
successfully in a fifth of the usual time. Nothing downstream could tell
that from a fast but legitimately empty run, and on resume the empty
part counts as done.

This warns rather than raises, because a grid can genuinely cover a
region a given slice of the scan never reaches. It reports through the
progress channel as well as warnings, since a bare warning is invisible
in the GUI, where an empty reconstruction is most expensive to discover
late. Both schedulers are covered: the grouped one returns early, so a
check placed only at the end of the function would miss every grouped
job, which is the default whenever a scan's geometry qualifies.

The root cause is not yet known, and this does not address it. The
static correction factor, the exposure angle bounds and the grid are all
bit-identical across fresh job loads; both schedulers re-raise recorded
exceptions; and the native memory precheck throws rather than returning
empty.
---
 orgui/datautils/xrayutils/reconstruction.py   |   9 ++
 .../test/test_reconstruction_pipeline.py      | 121 ++++++++++++++++++
 orgui/reconstruction_job.py                   |  39 ++++++
 3 files changed, 169 insertions(+)

diff --git a/orgui/datautils/xrayutils/reconstruction.py b/orgui/datautils/xrayutils/reconstruction.py
index bd316cc..866fe41 100644
--- a/orgui/datautils/xrayutils/reconstruction.py
+++ b/orgui/datautils/xrayutils/reconstruction.py
@@ -1182,6 +1182,9 @@ def __init__(
             for grid_name, ranges in self._boundaries.items()
         }
         self._resumed = set(resumed)
+        #: Records handed to :meth:`route` over this router's lifetime,
+        #: summed across grids. Diagnostic only -- nothing schedules on it.
+        self.routed_records = 0
         self._accumulators: dict[tuple[str, int], _CheckpointAccumulator] = {}
         self._parts: dict[tuple[str, int], int] = {}
         self._frames_in_part: dict[tuple[str, int], int] = {}
@@ -1268,6 +1271,12 @@ def route(self, grid_name, frame_index, batch, *, frames=1) -> None:
                 )
         key = (grid_name, index)
         with self._lock:
+            # Counted before the resumed-checkpoint early return, and
+            # across every grid: this exists so a caller can tell "mapped
+            # and produced nothing" from "mapped and produced records",
+            # which is otherwise invisible until a reconstruction comes
+            # out empty. See _warn_if_nothing_was_routed.
+            self.routed_records += int(np.asarray(batch["chunk_id"]).size)
             if key in self._resumed:
                 return
             accumulator = self._accumulators.get(key)
diff --git a/orgui/datautils/xrayutils/test/test_reconstruction_pipeline.py b/orgui/datautils/xrayutils/test/test_reconstruction_pipeline.py
index 973295c..e3a35a7 100644
--- a/orgui/datautils/xrayutils/test/test_reconstruction_pipeline.py
+++ b/orgui/datautils/xrayutils/test/test_reconstruction_pipeline.py
@@ -5,8 +5,10 @@
 
 from __future__ import annotations
 
+import dataclasses
 import threading
 import time
+import warnings
 
 import numpy as np
 import pytest
@@ -266,6 +268,125 @@ def test_map_pending_ranges_matches_reduce_on_synthetic_frames(tmp_path):
     assert int(written["contributors"].sum()) == frame_count
 
 
+def _mask_everything(payload, raw, frame, tile):
+    """A correction that masks every pixel, as a fully-masked frame does."""
+    return (
+        raw.astype(np.float64),
+        np.maximum(raw, 0).astype(np.float64),
+        np.ones(raw.shape, dtype=bool),
+    )
+
+
+def test_map_pending_ranges_warns_when_nothing_is_routed(tmp_path):
+    """Mapping frames and producing nothing must not be silent.
+
+    Observed in the field: a run mapped every frame, routed no records,
+    wrote a checkpoint claiming its full frame count with zero rows, and
+    exited successfully in a fifth of the usual time. Nothing downstream
+    could distinguish that from a fast, legitimately empty run, and on
+    resume the empty part counts as complete.
+    """
+    frame_count = 4
+    scan = _SlowScan(frame_count, delay=0.0)
+    config = _FakeConfig()
+    spec = _spec()
+    bounds = np.zeros((frame_count, 2, 4), dtype=np.float64)
+    grid_name = spec.grids[0].grid_name
+    router = _router({grid_name: [(0, frame_count)]}, tmp_path=tmp_path)
+    reported = []
+
+    with pytest.warns(RuntimeWarning, match="produced no records"):
+        _map_pending_ranges(
+            spec,
+            scan,
+            config,
+            bounds,
+            [(0, 1, 0, 1)],
+            [(0, frame_count)],
+            router,
+            correction_pipeline=_mask_everything,
+            effective_memory=256 * 1024**2,
+            threads_per_image=1,
+            accumulation_budget_bytes=None,
+            total_images=frame_count,
+            completed_images=0,
+            progress=lambda done, total, message: reported.append(message),
+        )
+
+    assert router.routed_records == 0
+    # The GUI never sees a warnings.warn, so the same statement has to
+    # reach the progress channel it does watch.
+    assert any("produced no records" in message for message in reported)
+
+
+def test_map_frame_groups_streamed_warns_when_nothing_is_routed(tmp_path):
+    """The grouped scheduler returns early, so it needs its own check.
+
+    Its ``return`` sits before the per-frame path's end, and a guard
+    placed only at the end of the function would miss every grouped job
+    -- which is the default whenever a scan's geometry qualifies.
+    """
+    frame_count = 8
+    scan = _SlowScan(frame_count, delay=0.0)
+    config = _FakeConfig()
+    spec = dataclasses.replace(_spec(), frames_per_group=2)
+    bounds = np.zeros((frame_count, 2, 4), dtype=np.float64)
+    grid_name = spec.grids[0].grid_name
+    router = _router({grid_name: [(0, frame_count)]}, tmp_path=tmp_path)
+
+    with pytest.warns(RuntimeWarning, match="produced no records"):
+        _map_pending_ranges(
+            spec,
+            scan,
+            config,
+            bounds,
+            [(0, 1, 0, 1)],
+            [(0, frame_count)],
+            router,
+            correction_pipeline=_mask_everything,
+            effective_memory=256 * 1024**2,
+            threads_per_image=1,
+            accumulation_budget_bytes=None,
+            total_images=frame_count,
+            completed_images=0,
+            progress=None,
+        )
+
+    assert router.routed_records == 0
+
+
+def test_map_pending_ranges_is_quiet_when_records_are_routed(tmp_path):
+    """The counterpart: a normal run must not warn."""
+    frame_count = 4
+    scan = _SlowScan(frame_count, delay=0.0)
+    config = _FakeConfig()
+    spec = _spec()
+    bounds = np.zeros((frame_count, 2, 4), dtype=np.float64)
+    grid_name = spec.grids[0].grid_name
+    router = _router({grid_name: [(0, frame_count)]}, tmp_path=tmp_path)
+
+    with warnings.catch_warnings():
+        warnings.simplefilter("error", RuntimeWarning)
+        _map_pending_ranges(
+            spec,
+            scan,
+            config,
+            bounds,
+            [(0, 1, 0, 1)],
+            [(0, frame_count)],
+            router,
+            correction_pipeline=_correction,
+            effective_memory=256 * 1024**2,
+            threads_per_image=1,
+            accumulation_budget_bytes=None,
+            total_images=frame_count,
+            completed_images=0,
+            progress=None,
+        )
+
+    assert router.routed_records > 0
+
+
 def test_map_pending_ranges_grows_reader_pool_under_sustained_blocking(tmp_path):
     """A slow, high-latency get_raw_img should push the reader pool above
     its initial size within a few coordinator ticks."""
diff --git a/orgui/reconstruction_job.py b/orgui/reconstruction_job.py
index cbb567f..f64e42d 100644
--- a/orgui/reconstruction_job.py
+++ b/orgui/reconstruction_job.py
@@ -12,6 +12,7 @@
 import threading
 import time
 from typing import Any
+import warnings
 
 import h5py
 import numpy as np
@@ -2371,6 +2372,38 @@ def _group_pipeline_layout(spec, tiles, memory_bytes, prepare_workers):
     return compute_workers, kernel_threads, pipeline_depth
 
 
+def _warn_if_nothing_was_routed(router, routed_before, total_images, progress):
+    """Say so when a whole mapping run produced no records at all.
+
+    Mapping every frame of a range and emitting nothing is possible
+    legitimately -- a grid can cover a region this slice of the scan never
+    reaches -- so this warns rather than raising. But it is also what an
+    intermittent failure observed in 2026-08 looks like from the outside:
+    roughly one run in twenty mapped nothing, wrote a checkpoint claiming
+    its full frame count with zero rows, and exited successfully, in about
+    a fifth of the usual time. Nothing downstream could tell that from a
+    fast, empty-but-correct run, and on resume the empty part counts as
+    done.
+
+    Reported through ``progress`` as well as :mod:`warnings`, because a
+    warning alone is invisible in the GUI, which is where an empty
+    reconstruction is most expensive to discover late.
+    """
+    routed = getattr(router, "routed_records", None)
+    if routed is None or total_images <= 0 or routed > routed_before:
+        return
+    message = (
+        f"Mapped {total_images} frames and produced no records at all. "
+        "This is expected only if the grids genuinely cover no part of "
+        "the reciprocal space these frames reach; otherwise the frames "
+        "were masked away or their geometry placed every sample outside "
+        "every grid, and the resulting checkpoints will be empty."
+    )
+    warnings.warn(message, RuntimeWarning, stacklevel=2)
+    if progress is not None:
+        progress(total_images, total_images, message)
+
+
 def _angles_advance_monotonically(bounds, frame_indices):
     """Whether frames adjacent in index are also adjacent in angle.
 
@@ -3071,6 +3104,8 @@ def layout_for(threads_per_image_seed):
         )
     ]
 
+    routed_before = getattr(router, "routed_records", 0)
+
     if frames_per_group > 1:
         group_workers, group_threads, group_depth = _group_pipeline_layout(
             spec, tiles, scheduler_memory, seed_prepare_workers
@@ -3095,6 +3130,9 @@ def layout_for(threads_per_image_seed):
             completed_images=completed_images,
             progress=progress,
         )
+        _warn_if_nothing_was_routed(
+            router, routed_before, total_images, progress
+        )
         return
 
     if automatic:
@@ -3478,6 +3516,7 @@ def compute_loop(retire, pool_cancellation):
     finally:
         reader_pool.shutdown(wait=True)
         compute_pool.shutdown(wait=True)
+    _warn_if_nothing_was_routed(router, routed_before, total_images, progress)
 
 
 def run_cluster_map_task(

From 20a824248f0e1567e8f2464063dfeb9776751cfa Mon Sep 17 00:00:00 2001
From: "T. Fuchs" <tfuchs@cornell.edu>
Date: Tue, 11 Aug 2026 19:35:29 -0400
Subject: [PATCH 27/33] docs(phys): record what the zero-record investigation
 eliminated

Four candidates measured and ruled out rather than reasoned about, so
that the next person does not spend the time again: the static
correction factor, the exposure angle bounds and the grid, a swallowed
exception, and the native memory precheck. Thirty consecutive runs of
one window returned the identical record count, so it is not a plain
per-run race either.

What remains is a raw frame reading back non-finite for a whole run,
which the finite-value check then masks away entirely. Unproven.
---
 .../reciprocal_space_performance_findings.md  | 32 +++++++++++++++++--
 1 file changed, 30 insertions(+), 2 deletions(-)

diff --git a/doc/design/reciprocal_space_performance_findings.md b/doc/design/reciprocal_space_performance_findings.md
index 73c5bd8..d9057e5 100644
--- a/doc/design/reciprocal_space_performance_findings.md
+++ b/doc/design/reciprocal_space_performance_findings.md
@@ -325,8 +325,36 @@ pipeline runs in 2026-08 routed zero records, wrote a checkpoint with 0
 rows and `frames_covered` set to the full count, and exited 0 — at 50.5
 ms/frame against ~250, i.e. a 5x "speed-up". Always compare a run's
 checkpoint fingerprint against its partner's and discard mismatches; a
-timing-only harness will silently take these as wins. The underlying
-defect is a correctness issue and is not yet diagnosed.
+timing-only harness will silently take these as wins.
+
+`_map_pending_ranges` now warns, on both schedulers and through the
+progress channel as well as `warnings`, when a whole run routes nothing.
+That makes the next occurrence visible; it does not explain it. What the
+investigation ruled out, each by direct measurement rather than
+inspection:
+
+- **Not the static correction factor.** `1 / solidAngleArray` and the
+  polarization array are bit-identical across 12 fresh job loads, with no
+  zeros and no non-finite entries. A bad one would poison a whole run,
+  which fits the symptom, but it does not vary.
+- **Not the angle bounds or the grid.** Bit-identical across 20 fresh
+  loads. This was the best remaining candidate — garbage bounds would put
+  every sample outside the grid, which is silent, fast and record-free.
+- **Not a swallowed exception.** Both schedulers re-raise the first
+  recorded exception, and the failing runs exited 0 with no traceback, so
+  correction did not raise.
+- **Not the native memory precheck.** It throws; it has no path that
+  returns empty.
+- **Not reproducible on demand.** 30 consecutive runs of a 39-frame
+  window returned the identical record count every time.
+
+What remains is that the raw frames themselves, or the mask derived from
+them, occasionally come back degenerate for an entire run — consistent
+with the `RuntimeWarning` seen at `intensity *= static_factor` on one
+occurrence, which implies a non-finite *raw* image rather than a bad
+factor. Both observed failures came late in long, memory-heavy sequences.
+Unproven. A separate one-off hang of the same pipeline under `pytest`,
+not reproduced, may or may not be related.
 
 ## Calibration constants and what they rest on
 

From 085b3bc8fd58fd4f9ec46033882fd3106e9ddf3a Mon Sep 17 00:00:00 2001
From: "T. Fuchs" <tfuchs@cornell.edu>
Date: Tue, 11 Aug 2026 19:35:34 -0400
Subject: [PATCH 28/33] docs(phys): accept the last-bit change, which was never
 released

No tag contains the reciprocal-space reconstruction feature, so the
association change has no published behaviour to break and no stored
result a user could have produced under the old order. Recorded as
decided rather than left pending.

The changelog had described this as changing reconstructed values for
every job, which says nothing to a reader for whom the entire feature is
new. What survives is the part that is still true: a job resumed against
a scratch directory written by an earlier development build will hold
checkpoint parts from both association orders.
---
 CHANGELOG.md                                     | 16 ++++++++--------
 ...cal_space_mapping_locality_and_subdivision.md | 13 +++++++++++++
 doc/source/release_notes.rst                     |  2 +-
 3 files changed, 22 insertions(+), 9 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 93c9fbd..487939b 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -71,14 +71,14 @@ Scientific correctness and performance fixes:
   reciprocal space as two adjacent pixels do, so a brick merges
   contributions the per-image block could not see: on a 3651-frame Pilatus
   6M scan, mapping eight images together emits 0.60x the records for the
-  same samples. **This changes reconstructed values in the last bits, for
-  every job, including jobs that map one image at a time.** Contributions
-  now merge inside the kernel rather than in the checkpoint accumulator, so
-  the same sums associate differently — with fewer intermediate roundings,
-  so slightly better conditioned. Which voxels are reached, and how many
-  detector samples reach each of them, are unchanged. Existing checkpoint
-  files remain resumable, but a job resumed across this change will contain
-  parts from both association orders.
+  same samples. Contributions merge inside the kernel rather than in the
+  checkpoint accumulator, which associates the same sums differently — with
+  fewer intermediate roundings, so slightly better conditioned. Which voxels
+  are reached, and how many detector samples reach each of them, are
+  unchanged. Reciprocal-space reconstruction has not appeared in a release
+  before this one, so no previously published result is affected; but a job
+  resumed against a scratch directory written by an earlier development
+  build will contain checkpoint parts from both association orders.
 - Mapping several images per call uses a scheduler of its own, which runs a
   few concurrent native calls with the thread budget split between them and
   hoists whole-frame correction into the prefetch pool, so one call's
diff --git a/doc/design/reciprocal_space_mapping_locality_and_subdivision.md b/doc/design/reciprocal_space_mapping_locality_and_subdivision.md
index 2310926..723fd55 100644
--- a/doc/design/reciprocal_space_mapping_locality_and_subdivision.md
+++ b/doc/design/reciprocal_space_mapping_locality_and_subdivision.md
@@ -1236,6 +1236,19 @@ intermediate roundings, so slightly better conditioned, but any test pinning an
 XXH3 digest of record values will move. That is a `phys`-scope change and needs
 saying so.
 
+*Decided, 2026-08: accepted.* Reciprocal-space reconstruction has never
+been released — no tag contains the feature commit — so there is no
+published behaviour for the association change to break, and no stored
+result a user could have produced with the old order. The commits keep
+their `!` and `BREAKING CHANGE:` footers, which are accurate about the
+code and cost nothing; the release notes should not describe this as a
+change to previously produced numbers, because for anyone reading them
+the whole feature is new. The one group this does affect is people
+running the development branch against existing scratch directories: a
+job resumed across the change contains checkpoint parts from both
+association orders. That is still worth saying, and is the only part of
+the warning that survives.
+
 **The group synchronisation point**, as described under Scheduling. *Real, and
 resolved by moving correction into the prepare pool (phase 3 step 5).* The
 preference for F = 8 was wrong on this job: F = 4 measured better, because
diff --git a/doc/source/release_notes.rst b/doc/source/release_notes.rst
index 13fdcc8..7429e7a 100644
--- a/doc/source/release_notes.rst
+++ b/doc/source/release_notes.rst
@@ -26,7 +26,7 @@ Scientific correctness and performance fixes:
 - Corrected film and epitaxy-interface anchoring, stacking, and support ownership across strained interfaces.
 - Fixed missing support in one-dimensional electron-density calculations and corrected atomic-coordinate stacking when splitting unit cells into layers.
 - Added a C++ electron-density backend and bounded caches for atomic form factors, anomalous scattering factors, and accelerated form-factor lookup.
-- Reciprocal-space mapping now maps detector images through a work block shaped as a brick in (row, column, frame) rather than a run of one flattened image, and can map several consecutive images in a single native call. On a rotation scan two adjacent images land as close together in reciprocal space as two adjacent pixels do, so a brick merges contributions the per-image block could not see: on a 3651-frame Pilatus 6M scan, mapping eight images together emits 0.60x the records for the same samples. **This changes reconstructed values in the last bits, for every job, including jobs that map one image at a time.** Contributions now merge inside the kernel rather than in the checkpoint accumulator, so the same sums associate differently — with fewer intermediate roundings, so slightly better conditioned. Which voxels are reached, and how many detector samples reach each of them, are unchanged. Existing checkpoint files remain resumable, but a job resumed across this change will contain parts from both association orders.
+- Reciprocal-space mapping now maps detector images through a work block shaped as a brick in (row, column, frame) rather than a run of one flattened image, and can map several consecutive images in a single native call. On a rotation scan two adjacent images land as close together in reciprocal space as two adjacent pixels do, so a brick merges contributions the per-image block could not see: on a 3651-frame Pilatus 6M scan, mapping eight images together emits 0.60x the records for the same samples. Contributions merge inside the kernel rather than in the checkpoint accumulator, which associates the same sums differently — with fewer intermediate roundings, so slightly better conditioned. Which voxels are reached, and how many detector samples reach each of them, are unchanged. Reciprocal-space reconstruction has not appeared in a release before this one, so no previously published result is affected; but a job resumed against a scratch directory written by an earlier development build will contain checkpoint parts from both association orders.
 - Mapping several images per call uses a scheduler of its own, which runs a few concurrent native calls with the thread budget split between them and hoists whole-frame correction into the prefetch pool, so one call's GIL-held Python work overlaps the next call's native work instead of queueing behind it. Measured 0.87x the mapping wall time of the per-image pipeline at four images per call on a 3651-frame rotation scan.
 - The number of images per call is now measured per job rather than configured. Grouping only helps while consecutive images land within about one voxel of each other, so the job's own geometry is probed at three points in the scan to decide whether it does, and the group size is then the largest the memory budget can afford without giving up concurrency. Scans whose angles do not advance monotonically, such as interlaced ones, map one image per call as before. On a 0.1 deg/frame rotation scan this chooses four images per call and maps at 0.88x the previous wall time.
 - Corrected the detector band height used for reciprocal-space mapping, which was still derived from the worst-case adaptive leaf count rather than from the record ceiling the memory prechecks use. At the ``very_high`` and ``maximum`` accuracy settings this over-estimated a pixel's cost by four orders of magnitude and split the detector into extremely thin bands — one row per band on a Pilatus 6M at ``maximum``, giving 2527 native calls per image instead of six. Band height no longer depends on the accuracy setting. Measured throughput at ``balanced`` accuracy is unchanged; the fix matters at the two highest settings.

From a95f1e77e4fe60849e6970c47e28d7447a467a3a Mon Sep 17 00:00:00 2001
From: "T. Fuchs" <tfuchs@cornell.edu>
Date: Tue, 11 Aug 2026 20:40:52 -0400
Subject: [PATCH 29/33] docs: plan the per-frame serial fraction as the next
 mapping feature

Phase 4 measured something it was not looking for: at depth 0 the
pipeline runs at ~110 ms/frame using 7-8 of 24 cores, bound by GIL-held
loading and correction rather than by mapping. That is the largest
remaining win in the mapping phase, and it takes over findings open item
1 as a plan of its own.

Three mechanisms in order, each with a predicted band and each measured
on its own: drop the per-tile stack and copy, fuse and GIL-release the
correction, then extend the prepare pool to the per-frame path. The
first step is none of them but a re-measurement -- the existing stage
table predates grouping and was taken in a kernel-dominated regime, so
it cannot apportion 110 ms/frame today.

Corrects the headroom claim in passing. Cold reads cap the pipeline at
51.5 ms/frame, so the prize is 2.1x; an earlier draft reasoned from idle
cores alone and said 3x.

Also records what phase 3 step 2 actually shipped: the per-brick centre
ray precompute is implemented and was never marked done, while the
whole-detector centre-ray cache it suggested is not, and is estimated at
2-4% for 149 MB.
---
 ..._space_mapping_locality_and_subdivision.md |  60 ++++++-
 ...eciprocal_space_mapping_serial_fraction.md | 147 ++++++++++++++++++
 .../reciprocal_space_performance_findings.md  |  29 +++-
 3 files changed, 230 insertions(+), 6 deletions(-)
 create mode 100644 doc/design/reciprocal_space_mapping_serial_fraction.md

diff --git a/doc/design/reciprocal_space_mapping_locality_and_subdivision.md b/doc/design/reciprocal_space_mapping_locality_and_subdivision.md
index 723fd55..ee11442 100644
--- a/doc/design/reciprocal_space_mapping_locality_and_subdivision.md
+++ b/doc/design/reciprocal_space_mapping_locality_and_subdivision.md
@@ -812,12 +812,22 @@ Not required by either finding; do not fold it in.
    contributor counts, totals equal to rounding. The fallback is
    scientifically identical rather than byte-identical — and slightly better,
    at 3.4% fewer records.
-2. *Brick loop, depth 0.* Precompute the brick's unit centre rays once into a
-   24 KiB stack buffer, iterate frames-outer / pixels-inner. Note separately
-   that at depth 0 the centre ray is frame-invariant for the whole job — pure
-   detector geometry, currently recomputed 3651 times per pixel. A
+2. *Brick loop, depth 0.* **Done, and it landed with step 1** — the
+   `max_depth_ == 0` branch of `accumulate_brick` precomputes the brick's
+   unit centre rays into a 24 KiB buffer and iterates frames-outer /
+   pixels-inner, exactly as specified here. It is what makes a single-frame
+   group 1.1x faster from block shape alone.
+
+   *The follow-on idea is not done, and is not worth doing.* A
    whole-detector centre-ray cache (149 MB, alongside the existing 143 MiB
-   corner cache) removes it entirely, with or without grouping.
+   corner cache) would remove the recomputation entirely rather than once
+   per brick per call. But the brick buffer already amortises `ray_at` over
+   the `F` frames of a group, so what remains to save is roughly
+   `7 ns / F` against a per-sample cost of 40-80 ns — about 2-4% of
+   depth-0 kernel time at `F = 4`, for 149 MB and a new whole-job cache to
+   invalidate. Estimated, not measured; if anyone wants it, measure before
+   building, because phase 4 is a standing reminder that a saving of known
+   size need not convert.
 3. *Banding.* **Done, and it needed no new code.** The detector tiling that
    already exists is the banding: `_execution_layout` gives this job six
    full-width bands of ~421 x 2463 (1.04 Mpx), and `_map_frame_group` maps
@@ -1217,6 +1227,46 @@ Instrument: `benchmarks/benchmark_reconstruction_depth_convergence.py`.
 Unlike everything else in this document it is immune to machine noise —
 it compares numbers, not times.
 
+## What comes next
+
+Every phase in this document is resolved, so this is where the work
+stands rather than a plan to continue it. In priority order:
+
+1. **The zero-record defect, before anything else.** Roughly one run in
+   twenty maps nothing and says it succeeded. It now warns, but the cause
+   is unknown and four candidates have been eliminated (see the findings
+   document's measurement traps). This is correctness and it gates using
+   the branch on real data; nothing below matters beside it.
+2. **The branch is ready to merge otherwise.** Phases 1-4 are complete or
+   deliberately closed, the association-order decision is settled — the
+   feature has never been released, so nothing published moves — and the
+   tests pin voxels, contributor counts and totals.
+3. **The binding constraint is no longer the kernel.** This is the useful
+   thing phase 4 turned up by accident. At depth 0 the whole pipeline
+   runs at ~110 ms/frame on the coarse grid using **7-8 of 24 cores**,
+   bound by GIL-held loading and correction rather than by mapping. That
+   is findings open item 1, it is now measured rather than inferred, and
+   it is worth up to 2.1x — bounded by the 51.5 ms/frame cold-read floor,
+   not by the idle cores — which is more than anything left in this
+   document. **It is the next feature, and it has its own plan in
+   [`reciprocal_space_mapping_serial_fraction.md`](reciprocal_space_mapping_serial_fraction.md).**
+4. **The group constants have never been tested off this job.**
+   `_GROUP_ADVANCE_VOXEL_LIMIT` (one voxel) and
+   `_GROUP_MINIMUM_CONCURRENT_CALLS` (three) reproduce this scan's optimum
+   and nothing else is known about them. A second dataset would say
+   whether they generalise, and phase 4 narrows where they even apply:
+   only depth 0, and only where the kernel dominates.
+
+Deliberately *not* next, and why:
+
+- **Anything above depth 3.** Phases 1 and 2 made those depths
+  affordable, and they turn out to change no intensity a measurement can
+  resolve. Findings open item 3 — that depth 3+ block sizes are
+  extrapolated rather than measured — loses most of its value with them.
+- **Better bricks above depth 0** — phase 4, measured and closed.
+- **Band height and group size** — both measured flat, and they
+  substitute for each other.
+
 ## Risks and open questions
 
 **Interlaced scans.** `orgui/backend/interlacedScanLoader.py` exists, so frames
diff --git a/doc/design/reciprocal_space_mapping_serial_fraction.md b/doc/design/reciprocal_space_mapping_serial_fraction.md
new file mode 100644
index 0000000..9d0c33c
--- /dev/null
+++ b/doc/design/reciprocal_space_mapping_serial_fraction.md
@@ -0,0 +1,147 @@
+# Reciprocal-space mapping: the per-frame serial fraction
+
+> **Status: planned, nothing implemented.** This is the next feature in
+> the mapping phase. It exists because phase 4 of
+> `reciprocal_space_mapping_locality_and_subdivision.md` measured
+> something it was not looking for: at low adaptive depth the pipeline
+> leaves two thirds of the machine idle, and the kernel is not what is
+> holding it back.
+
+Fourth document in the set, after `reciprocal_space_scratch_architecture.md`
+(what the pipeline is), `reciprocal_space_performance_findings.md` (what has
+been measured), and `reciprocal_space_mapping_locality_and_subdivision.md`
+(the locality and subdivision work, complete). This one takes over findings
+open item 1, which has been the largest structural limit in that document
+since it was written and is now the largest *measured* one.
+
+## The finding
+
+Sampled per process during live runs on the reference job at depth 0:
+
+| | |
+|---|---|
+| wall time | ~110 ms/frame |
+| cores in use | **7-8 of 24** |
+| cold I/O floor | 51.5 ms/frame (49.8 MB at 967 MB/s) |
+
+Mapping is not the constraint in this regime. Loading and correcting are,
+and they hold the GIL, so adding workers does not help. The old Amdahl
+estimate — 10.5% serial fraction, 6.1x ceiling at 16 workers, 4.2x
+measured — understated how lopsided this becomes once the kernel is cheap.
+
+**The headroom is about 2.1x, not more.** An earlier draft of this claim
+said 3x, reasoning from idle cores alone; that is wrong, because cold
+reads cap the pipeline at 51.5 ms/frame however free the CPU work
+becomes. 110 to 51.5 is 2.14x and that is the whole prize. It shrinks
+further whenever the page cache holds the window, which on this machine
+it partly does — see the depth-0 noise trap in the findings document.
+
+## Where this pays, and where it does not
+
+Grouping (phase 3 of the locality document) pays where the kernel
+dominates: depth 0 on a grid fine enough to make record density the cost.
+This work pays where the kernel is cheap: depth 0 on a coarse grid, which
+is `center` accuracy on a routine job — the common case, and the one that
+should be fastest. Above depth 1 neither matters: the subdivision walk
+costs 600-2300 ms/frame and a 60 ms serial fraction disappears beside it.
+
+So the two are complementary rather than competing, and both are depth-0
+work. That is worth knowing before optimising either further: **the whole
+remaining opportunity in mapping is at depth 0**, because depths above 2
+change no intensity a measurement can resolve.
+
+## What has to be measured first
+
+The stage table in the findings document (load 48.3 / correct 42.3 /
+slice-copy 24.0 / kernel 853.4 / reduce 121.2 ms) predates grouping, the
+prepare pool and the current grid, and was taken in a kernel-dominated
+regime. **It cannot be used to apportion 110 ms/frame today.** Re-measure
+before designing anything:
+
+1. Per-stage wall and CPU time at depth 0 on both grids, inside the real
+   scheduler rather than a harness, distinguishing
+   - HDF5 read and decompression (GIL released inside h5py),
+   - correction arithmetic (several full-detector NumPy passes),
+   - the per-tile `np.stack` / `ascontiguousarray` copy,
+   - time blocked on the GIL itself.
+2. How much of `load` is genuinely I/O-bound. If it is already at the
+   cold-read floor, correction is the only thing left worth attacking.
+
+The GIL-blocked fraction is the number that decides this. If it is small,
+the pipeline is I/O-bound, the ceiling is nearer 1.2x than 2.1x, and this
+whole document should be closed the way phase 4 was.
+
+## Candidate mechanisms
+
+In increasing order of invasiveness. Each carries its own predicted band,
+stated before implementing so it can be wrong on the record.
+
+### A. Drop the per-tile stack and copy
+
+`_map_frame_group` builds `np.ascontiguousarray(np.stack(...))` for
+intensity, variance and mask, per tile, per group. Tiles partition the
+detector, so this copies every corrected frame exactly once — 105 MB per
+frame — in Python, under the GIL, purely to hand the kernel a contiguous
+`(frames, rows, columns)` buffer.
+
+`accumulate_group` could instead take the whole corrected frame plus the
+tile rectangle and read the sub-rectangle itself; it already walks bricks
+within a tile, so a row stride costs it nothing.
+
+**Predicted: 0.80-0.90x** at depth 0, from removing ~24 ms of the ~110.
+Findings open item 1 named this direction; it is the cheapest of the
+three and does not touch any arithmetic, so it should be bit-for-bit.
+
+### B. Fuse and release the correction
+
+Correction is several full-detector NumPy passes — background subtract,
+static factor, exposure and monitor scaling, the finiteness check, mask
+combination. Each is memory-bound over ~50 MB, and NumPy holds the GIL
+across the small ones. Fusing them into a single pass in the existing C++
+extension would both cut memory traffic and release the GIL, letting the
+prepare pool actually run in parallel.
+
+**Predicted: 0.65-0.80x** on top of A. **This is a `phys` change** — it
+moves arithmetic that produces the corrected intensities and variances
+into another language and another association order. It needs the same
+treatment phase 3 got: voxels, contributor counts and totals pinned, and
+the last-bit movement stated. Do not fold it in with A.
+
+### C. Extend the prepare pool to the per-frame path
+
+The grouped scheduler already hoists load and correct into a prepare
+pool; the per-frame scheduler does not, so a job outside the grouping
+regime — a coarse scan, an interlaced one — still queues its GIL-held
+Python behind its own native call. With B done, this is mostly
+bookkeeping.
+
+**Predicted: little on the reference job** (which groups), and up to the
+same 0.87x grouping bought, on jobs that do not.
+
+## Ordering
+
+A, then measure; B, then measure; C last. A is bit-for-bit and cheap. B
+is where the prize is and where the risk is. C only pays for jobs the
+first two do not already cover.
+
+Do not do all three and measure once. Phase 4 is the standing argument:
+a saving of known size need not convert, and finding that out one change
+at a time is what makes it interpretable.
+
+## Risks
+
+**The ceiling is low and the noise is high.** 2.1x is the best case, and
+depth 0 is the regime the findings document flags as dominated by page
+cache state — one arm spread 176.5 to 340.6 ms/frame. Six interleaved
+pairs minimum, and the foreign-load guard on every run. It is entirely
+possible for a real 15% win to be invisible here.
+
+**B changes reconstructed values.** Not in which voxels are reached, but
+in the last bits of the corrected intensities that feed them. That is
+acceptable on the same grounds the phase 3 change was — the feature has
+never been released — but it must be said, and it must not be discovered
+by a user.
+
+**I/O may already be the wall.** If step 1 of the measurement shows load
+sitting at the cold-read floor with correction overlapped, there is
+nothing here worth building, and saying so is the correct outcome.
diff --git a/doc/design/reciprocal_space_performance_findings.md b/doc/design/reciprocal_space_performance_findings.md
index d9057e5..d9726a0 100644
--- a/doc/design/reciprocal_space_performance_findings.md
+++ b/doc/design/reciprocal_space_performance_findings.md
@@ -2,7 +2,9 @@
 
 Companion to `reciprocal_space_scratch_architecture.md`, which describes what
 the pipeline *is*. This one records what was measured, what those measurements
-settled, and what is still open.
+settled, and what is still open. Two work streams have grown out of it into
+plans of their own: `reciprocal_space_mapping_locality_and_subdivision.md`
+(complete) and `reciprocal_space_mapping_serial_fraction.md` (next).
 
 > **Read `reciprocal_space_mapping_locality_and_subdivision.md` alongside
 > this.** Since these numbers were taken, the subdivision was rewritten
@@ -194,6 +196,24 @@ only above one frame per group; the per-frame path still has the serial
 fraction described above, and the per-tile slice/copy is still in Python on
 both paths.
 
+**Now the binding constraint, and measured rather than inferred.** Phase 4
+turned this up by accident. On the current (coarse) reference job at depth
+0 the whole pipeline runs at ~110 ms/frame while using **7-8 of 24 cores** —
+sampled per process during live runs, not derived from a model. Mapping is
+not the limit there at all; loading and correcting are. The Amdahl estimate
+above put the ceiling at 6.1x and measured 4.2x, which understated how
+lopsided this gets once the kernel is cheap: a coarse grid, a fast disk and
+depth 0 leave two thirds of the machine idle. **This is the largest
+remaining win in the mapping phase**, and it is now better evidenced than
+anything in the locality document's remaining ideas.
+
+Worth **up to 2.1x, not more**: cold reads cap the pipeline at 51.5
+ms/frame (49.8 MB at 967 MB/s) however free the CPU work becomes, and an
+earlier draft that reasoned from idle cores alone and claimed 3x was
+wrong. This item is now a plan of its own —
+`reciprocal_space_mapping_serial_fraction.md` — and further work on it
+belongs there rather than here.
+
 ### 2. `bounded_block_size` still clamps memory with a depth-blind constant
 
 The Python side now caps the work block against the arena the kernel will
@@ -220,6 +240,13 @@ same record ceiling.
 The `16384 >> depth` rule was measured at depths 0, 1 and 2 (optima 16384, 8192,
 4096, consistent at 12 and 24 threads). Depths 3-8 follow by extrapolation only.
 
+**Mostly retired by the depth-convergence result.** Depths above 2 change
+no intensity a measurement can resolve, so tuning their block size
+optimises settings nobody should be running for production. Worth doing
+only if someone is using them deliberately to check convergence, and even
+then the runtime is dominated by the subdivision itself rather than by
+block-size choice.
+
 ### 4. `image_workers = min(len(pending_ranges), ...)`
 
 Concurrency is capped by the *scheduling* granularity. Production sizes

From 395a834882fae9d9227bd212940660db7873209c Mon Sep 17 00:00:00 2001
From: "T. Fuchs" <tfuchs@cornell.edu>
Date: Wed, 12 Aug 2026 11:10:48 -0400
Subject: [PATCH 30/33] test: measure mapping stages and run two arms
 interleaved

The stage benchmark splits the mapping phase by stage and reports wall
against on-core time for each, from QueryThreadCycleTime rather than
time.thread_time, which quantises to the 15.6 ms scheduler tick. It also
probes GIL contention directly.

The A/B driver runs two arms alternating, flipping their order each
repeat, and reports paired ratios, per-run foreign CPU load and each
run's checkpoint fingerprint. An arm is a set of files copied into the
checkout, so a native change measures against its own baseline binary.
---
 benchmarks/benchmark_reconstruction_ab.py     | 342 ++++++++
 benchmarks/benchmark_reconstruction_stages.py | 774 ++++++++++++++++++
 2 files changed, 1116 insertions(+)
 create mode 100644 benchmarks/benchmark_reconstruction_ab.py
 create mode 100644 benchmarks/benchmark_reconstruction_stages.py

diff --git a/benchmarks/benchmark_reconstruction_ab.py b/benchmarks/benchmark_reconstruction_ab.py
new file mode 100644
index 0000000..4fc7461
--- /dev/null
+++ b/benchmarks/benchmark_reconstruction_ab.py
@@ -0,0 +1,342 @@
+"""Run two pipeline arms interleaved, and report the paired ratios.
+
+Everything measured about the mapping pipeline on this machine has had to
+be measured this way, and every conclusion drawn any other way has had to
+be withdrawn. The rules, which this script exists to make automatic:
+
+- **Interleave the arms.** A run of one arm followed by a run of the other
+  confounds the arm with drift and with page-cache state. Arms alternate
+  here, and the order flips on every repeat so that neither arm is always
+  the one that inherits the other's warm cache.
+- **Only paired ratios carry.** Absolute values drift between sessions by
+  30%; the same configuration has measured 183, 202 and 342 ms/frame. The
+  summary reports ratios and, more importantly, whether the arms separate
+  cleanly -- every run of one beating every run of the other is worth more
+  than a difference of means.
+- **Verify the machine is quiet.** It carries a persistent 1.8-2.9 cores
+  of background load. Foreign CPU is measured per run as the sum over
+  *other user processes*, not as machine-busy minus the benchmark's own:
+  kernel, interrupt and storage-driver time is charged to System rather
+  than to the process that caused it, and mapping is I/O-heavy, so the
+  total-minus-child definition reported 7.2 cores of interference where
+  per-process accounting showed 2.9.
+- **Check what each run actually mapped.** Roughly one mapping run in
+  twenty routes zero records, writes a checkpoint claiming every frame,
+  and exits 0 at about a fifth of the usual time. A timing-only harness
+  reads that as a win. Every run's checkpoint fingerprint is compared
+  against its arm's first run here, and a mismatch is reported rather
+  than averaged in.
+
+An arm is a set of files copied into the checkout before its runs, so a
+native change can be measured against its own baseline binary without
+rebuilding between runs: capture the baseline ``.pyd`` and ``.py`` before
+changing anything, and name both states as arms.
+
+Plan file (JSON)::
+
+    {
+      "repeats": 6,
+      "command": ["python", "benchmarks/benchmark_reconstruction_pipeline.py",
+                  "<job>.json", "--start", "1638", "--count", "234",
+                  "--depth", "0"],
+      "arms": [
+        {"name": "baseline",
+         "copy": [["saved/baseline.pyd", "orgui/.../_reciprocal_...pyd"]]},
+        {"name": "tile-view",
+         "copy": [["saved/new.pyd", "orgui/.../_reciprocal_...pyd"]],
+         "arguments": ["--group", "4"]}
+      ]
+    }
+
+``PYTHONPATH`` is set to the checkout for every child, which is the trap
+the benchmarks themselves warn about: ``python benchmarks/<script>.py``
+puts ``benchmarks/`` on ``sys.path``, not the repository root.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import os
+from pathlib import Path
+import shutil
+import statistics
+import subprocess
+import sys
+from time import perf_counter
+
+
+def _cpu_snapshot():
+    """``{pid: cpu_seconds}`` for every process the user can see.
+
+    Excludes the two pseudo-processes Windows reports (System Idle and
+    System): the second collects kernel, interrupt and storage-driver
+    time, which for an I/O-heavy benchmark is mostly the benchmark's own
+    and would otherwise be counted as interference.
+    """
+    import psutil
+
+    snapshot = {}
+    for process in psutil.process_iter(["pid", "name"]):
+        pid = process.info["pid"]
+        if pid in (0, 4):
+            continue
+        try:
+            times = process.cpu_times()
+        except (psutil.NoSuchProcess, psutil.AccessDenied):
+            continue
+        snapshot[pid] = times.user + times.system
+    return snapshot
+
+
+def foreign_cores(before, after, elapsed, child_pid):
+    """Cores of CPU spent by processes other than the benchmark.
+
+    A process that appeared during the run is counted in full: it did all
+    of its work inside the window. A process that exited during the run is
+    lost, which understates interference slightly and is the one direction
+    worth being wrong in -- it never manufactures a quiet machine out of a
+    busy one, only the reverse.
+
+    :rtype: float
+    """
+    total = 0.0
+    for pid, cpu in after.items():
+        if pid == child_pid:
+            continue
+        total += max(0.0, cpu - before.get(pid, 0.0))
+    return total / elapsed if elapsed > 0 else 0.0
+
+
+def _apply(arm, root):
+    for source, destination in arm.get("copy", ()):
+        source_path = Path(source)
+        destination_path = root / destination
+        if not source_path.is_file():
+            raise SystemExit(f"Arm {arm['name']}: no such file {source_path}")
+        shutil.copyfile(source_path, destination_path)
+
+
+def _run(arm, plan, root, index, retries=1):
+    """One run of one arm; returns the parsed header, result and load."""
+    command = [
+        argument.replace("{repeat}", str(index))
+        for argument in [*plan["command"], *arm.get("arguments", ())]
+    ]
+    if command[0] == "python":
+        command[0] = sys.executable
+    environment = dict(os.environ, PYTHONPATH=str(root))
+    before = _cpu_snapshot()
+    started = perf_counter()
+    # Popen rather than run(): the child's pid is what separates its own
+    # CPU from the machine's, and CompletedProcess does not carry one.
+    child = subprocess.Popen(
+        command,
+        cwd=root,
+        env=environment,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE,
+        text=True,
+    )
+    try:
+        stdout, stderr = child.communicate(timeout=plan.get("timeout", 1800))
+    except subprocess.TimeoutExpired:
+        # The mapping pipeline hangs about once in a few dozen runs, with
+        # every checkpoint already written and its thread pools still
+        # alive (findings document, measurement traps). Killing and
+        # retrying loses the run, not the sweep -- but a run that hangs
+        # twice is something to go and look at rather than to paper over.
+        child.kill()
+        child.communicate()
+        if retries <= 0:
+            raise SystemExit(f"Arm {arm['name']} run {index} hung twice")
+        print(
+            f"  !! {arm['name']} repeat {index} hung; killed and retrying",
+            flush=True,
+        )
+        return _run(arm, plan, root, index, retries=retries - 1)
+    elapsed = perf_counter() - started
+    after = _cpu_snapshot()
+    if child.returncode != 0:
+        print(stdout[-4000:])
+        print(stderr[-4000:], file=sys.stderr)
+        raise SystemExit(f"Arm {arm['name']} run {index} failed")
+    documents = []
+    for line in stdout.splitlines():
+        line = line.strip()
+        if line.startswith("{") and line.endswith("}"):
+            try:
+                documents.append(json.loads(line))
+            except json.JSONDecodeError:
+                continue
+    if len(documents) < 2:
+        print(stdout[-4000:])
+        raise SystemExit(
+            f"Arm {arm['name']} run {index}: expected a header and a result"
+        )
+    header, result = documents[0], documents[-1]
+    return {
+        "arm": arm["name"],
+        "repeat": index,
+        "ms_per_frame": result["seconds_per_frame"] * 1e3,
+        "routed_records_per_frame": result.get("routed_records_per_frame"),
+        # Voxels reached and contributors apportioned (the fingerprint),
+        # the row count, and the totals to the last bit. An optimisation
+        # that is meant to be bit-for-bit has to hold all of them; a
+        # change that reassociates sums on purpose keeps the first two and
+        # moves the last, which is worth being able to see.
+        "fingerprint": {
+            name: [
+                values["voxel_fingerprint"],
+                values["rows"],
+                values["contributors"],
+                repr(values["weight"]),
+                repr(values["weighted_intensity"]),
+                repr(values["weighted_variance"]),
+            ]
+            for name, values in result.get("grids", {}).items()
+        },
+        "frames_per_group": header.get("frames_per_group"),
+        "final_layout": result.get("final_layout"),
+        "foreign_cores": foreign_cores(before, after, elapsed, child.pid),
+        "wall_seconds": elapsed,
+    }
+
+
+def main():
+    """Run the plan's arms interleaved and summarise the paired ratios."""
+    parser = argparse.ArgumentParser()
+    parser.add_argument("plan", type=Path)
+    parser.add_argument("--repeats", type=int, default=None)
+    parser.add_argument("--output", type=Path)
+    arguments = parser.parse_args()
+
+    plan = json.loads(arguments.plan.read_text(encoding="utf-8"))
+    root = Path(plan.get("root", Path.cwd())).resolve()
+    repeats = arguments.repeats or int(plan.get("repeats", 6))
+    arms = plan["arms"]
+    if len(arms) != 2:
+        raise SystemExit("A comparison needs exactly two arms")
+
+    runs = []
+    for repeat in range(repeats):
+        # Flip the order every repeat: the second arm of a pair inherits
+        # whatever of the window the first left in the page cache, and at
+        # depth 0 that is the single largest source of run-to-run spread.
+        order = arms if repeat % 2 == 0 else arms[::-1]
+        for arm in order:
+            print(f"  [{arm['name']} repeat {repeat}] running...", flush=True)
+            _apply(arm, root)
+            run = _run(arm, plan, root, repeat)
+            runs.append(run)
+            print(
+                f"  {run['arm']:<16} repeat {repeat}  "
+                f"{run['ms_per_frame']:7.1f} ms/frame  "
+                f"{(run['routed_records_per_frame'] or 0):>11,.0f} rec/frame  "
+                f"foreign {run['foreign_cores']:.2f} cores",
+                flush=True,
+            )
+
+    by_arm = {arm["name"]: [] for arm in arms}
+    for run in runs:
+        by_arm[run["arm"]].append(run)
+
+    print()
+    problems = []
+    for name, arm_runs in by_arm.items():
+        reference = arm_runs[0]["fingerprint"]
+        for run in arm_runs[1:]:
+            if run["fingerprint"] != reference:
+                problems.append(
+                    f"{name} repeat {run['repeat']} mapped something else: "
+                    f"{run['fingerprint']} against {reference}"
+                )
+    first, second = (arm["name"] for arm in arms)
+    same_output = by_arm[first][0]["fingerprint"] == by_arm[second][0]["fingerprint"]
+    print(
+        "Checkpoint fingerprints: "
+        + ("identical across arms" if same_output else "DIFFER between arms")
+    )
+    for problem in problems:
+        print(f"  !! {problem}")
+
+    # A pair either of whose runs saw more than the machine's usual
+    # background load is discarded, not corrected: the interference is
+    # not a constant that can be subtracted, and a pair is the unit that
+    # carries meaning.
+    foreign_limit = float(plan.get("foreign_limit", 3.0))
+    pairs = []
+    for index in range(min(len(by_arm[first]), len(by_arm[second]))):
+        left, right = by_arm[first][index], by_arm[second][index]
+        pairs.append(
+            {
+                "repeat": index,
+                "ratio": right["ms_per_frame"] / left["ms_per_frame"],
+                "clean": max(left["foreign_cores"], right["foreign_cores"])
+                <= foreign_limit,
+                "foreign_cores": max(
+                    left["foreign_cores"], right["foreign_cores"]
+                ),
+            }
+        )
+    ratios = [pair["ratio"] for pair in pairs]
+    clean_pairs = [pair for pair in pairs if pair["clean"]]
+    if len(clean_pairs) < len(pairs):
+        print(
+            f"  {len(pairs) - len(clean_pairs)} of {len(pairs)} pairs exceeded "
+            f"{foreign_limit:.1f} cores of foreign load and are discarded: "
+            + ", ".join(
+                f"repeat {pair['repeat']} ({pair['foreign_cores']:.2f})"
+                for pair in pairs
+                if not pair["clean"]
+            )
+        )
+    print()
+    for name, arm_runs in by_arm.items():
+        values = sorted(run["ms_per_frame"] for run in arm_runs)
+        print(
+            f"  {name:<16} median {statistics.median(values):7.1f} ms/frame  "
+            f"range {values[0]:.1f}-{values[-1]:.1f}"
+        )
+    kept = [pair["repeat"] for pair in clean_pairs]
+    separated = bool(kept) and max(
+        by_arm[second][index]["ms_per_frame"] for index in kept
+    ) < min(by_arm[first][index]["ms_per_frame"] for index in kept)
+    print()
+    print(
+        f"  paired ratios ({second} / {first}): "
+        + ", ".join(
+            f"{pair['ratio']:.3f}" + ("" if pair["clean"] else "*")
+            for pair in pairs
+        )
+    )
+    if clean_pairs:
+        print(
+            f"  median ratio {statistics.median(p['ratio'] for p in clean_pairs):.3f}"
+            f" over {len(clean_pairs)} clean pairs"
+        )
+    print(
+        f"  clean separation: {'yes' if separated else 'no'} "
+        f"(every {second} run faster than every {first} run, clean pairs only)"
+    )
+    worst = max(run["foreign_cores"] for run in runs)
+    print(f"  worst foreign load in any run: {worst:.2f} cores")
+
+    if arguments.output:
+        arguments.output.write_text(
+            json.dumps(
+                {
+                    "plan": plan,
+                    "runs": runs,
+                    "pairs": pairs,
+                    "ratios": ratios,
+                    "clean_separation": separated,
+                },
+                indent=2,
+            ),
+            encoding="utf-8",
+        )
+
+
+if __name__ == "__main__":
+    main()
diff --git a/benchmarks/benchmark_reconstruction_stages.py b/benchmarks/benchmark_reconstruction_stages.py
new file mode 100644
index 0000000..26f4226
--- /dev/null
+++ b/benchmarks/benchmark_reconstruction_stages.py
@@ -0,0 +1,774 @@
+"""Where a mapped frame's wall time goes, inside the real scheduler.
+
+``benchmark_reconstruction_pipeline.py`` reports one number for the whole
+mapping phase. This runs the same pipeline and splits that number by stage
+-- HDF5 read, whole-frame correction, per-tile assembly, the native call,
+the cross-tile merge, the checkpoint route -- and reports, for each of
+them, *wall* time and *on-core* time separately.
+
+The difference between the two is the point. A stage's wall time is what
+it costs the thread running it; its on-core time is what it costs the
+machine. Wall minus on-core is time the thread existed but was not
+executing: waiting for the disk, waiting for the GIL, or preempted. At
+depth 0 the mapping pipeline uses 7-8 of 24 cores, and which of those
+three explains the gap decides whether there is anything to win.
+
+On-core time comes from ``QueryThreadCycleTime``, not ``time.thread_time``:
+the latter is ``GetThreadTimes`` on Windows and quantises to the ~15.6 ms
+scheduler tick, which is larger than most of the stages measured here.
+Cycles are converted with a rate calibrated by spinning at start-up.
+
+Every stage is measured by wrapping an existing call site from outside --
+no instrumentation is compiled into the pipeline, and the arithmetic is
+untouched. Stages nest (the kernel, merge and route happen inside
+``_map_frame_group``), so the table reports the nesting and derives the
+per-tile assembly cost, which is the one stage with no call of its own,
+as its parent's residual.
+
+**Run this with ``PYTHONPATH`` pointing at the checkout**, for the reason
+the pipeline benchmark gives; the resolved extension path is printed.
+"""
+
+from __future__ import annotations
+
+import argparse
+import ctypes
+from dataclasses import replace
+import json
+from pathlib import Path
+import shutil
+import tempfile
+import threading
+from time import perf_counter
+
+import numpy as np
+
+from benchmark_reconstruction_pipeline import (
+    _summarize_checkpoints,
+    _window_ranges,
+)
+
+from orgui.datautils.xrayutils import reconstruction
+from orgui.datautils.xrayutils.reconstruction import (
+    _CheckpointRouter,
+    _native_module,
+)
+import orgui.reconstruction_job as reconstruction_job
+from orgui.reconstruction_job import (
+    _base_provenance,
+    _correction_pipeline,
+    _execution_layout,
+    _load_assets,
+    _map_pending_ranges,
+    read_job,
+    resolve_work_block_pixels,
+    split_memory_budget,
+)
+
+
+_kernel32 = ctypes.WinDLL("kernel32", use_last_error=True)
+_kernel32.QueryThreadCycleTime.argtypes = [
+    ctypes.c_void_p,
+    ctypes.POINTER(ctypes.c_ulonglong),
+]
+_kernel32.QueryThreadCycleTime.restype = ctypes.c_int
+_PSEUDO_CURRENT_THREAD = ctypes.c_void_p(_kernel32.GetCurrentThread())
+
+
+def thread_cycles():
+    """Cycles this thread has executed, including stall cycles.
+
+    Unlike ``time.thread_time()`` (``GetThreadTimes``, quantised to the
+    scheduler tick) this is exact enough to attribute a single millisecond
+    stage, and unlike wall time it does not count a thread that is waiting
+    for the GIL or for the disk.
+
+    :rtype: int
+    """
+    value = ctypes.c_ulonglong()
+    if not _kernel32.QueryThreadCycleTime(
+        _PSEUDO_CURRENT_THREAD, ctypes.byref(value)
+    ):
+        raise OSError(ctypes.get_last_error(), "QueryThreadCycleTime failed")
+    return value.value
+
+
+def calibrate_cycle_rate(bursts=3, seconds=0.05):
+    """Cycles per second of on-core time, measured by spinning.
+
+    Taken as the maximum over several bursts: a burst that happened to be
+    preempted measures too low, never too high.
+
+    :rtype: float
+    """
+    best = 0.0
+    for _burst in range(bursts):
+        start_wall = perf_counter()
+        start_cycles = thread_cycles()
+        while perf_counter() - start_wall < seconds:
+            pass
+        elapsed = perf_counter() - start_wall
+        best = max(best, (thread_cycles() - start_cycles) / elapsed)
+    return best
+
+
+_registry: list[dict] = []
+_registry_lock = threading.Lock()
+_local = threading.local()
+
+
+def _bucket(name):
+    """This thread's ``[wall, cycles, calls]`` accumulator for a stage.
+
+    Per thread and merged at the end, so the hot path takes no lock: a
+    shared counter would contend on exactly the resource being measured.
+    """
+    buckets = getattr(_local, "buckets", None)
+    if buckets is None:
+        buckets = _local.buckets = {}
+        with _registry_lock:
+            _registry.append(buckets)
+    bucket = buckets.get(name)
+    if bucket is None:
+        bucket = buckets[name] = [0.0, 0, 0]
+    return bucket
+
+
+class timed:
+    """Context manager accumulating wall time and cycles for one stage.
+
+    A class rather than ``@contextmanager`` to keep the per-entry cost at
+    two attribute lookups; the stages measured here are milliseconds, but
+    ``np.stack`` is called several times per tile.
+    """
+
+    __slots__ = ("_bucket", "_wall", "_cycles")
+
+    def __init__(self, name):
+        self._bucket = _bucket(name)
+
+    def __enter__(self):
+        self._wall = perf_counter()
+        self._cycles = thread_cycles()
+        return self
+
+    def __exit__(self, *exception):
+        bucket = self._bucket
+        bucket[0] += perf_counter() - self._wall
+        bucket[1] += thread_cycles() - self._cycles
+        bucket[2] += 1
+        return False
+
+
+def merged_stages():
+    """Every thread's buckets, summed per stage.
+
+    :rtype: dict[str, dict]
+    """
+    totals: dict[str, list] = {}
+    with _registry_lock:
+        snapshots = [dict(buckets) for buckets in _registry]
+    for buckets in snapshots:
+        for name, (wall, cycles, calls) in buckets.items():
+            total = totals.setdefault(name, [0.0, 0, 0])
+            total[0] += wall
+            total[1] += cycles
+            total[2] += calls
+    return totals
+
+
+def minimal_correction(correct, static_mask):
+    """A correction that keeps the mask and drops the scaling arithmetic.
+
+    A ceiling probe, not a proposal. Fusing correction into the native
+    extension would remove the full-detector NumPy passes that apply the
+    static factor, the exposure and the monitors -- it would not remove
+    the raw copy, the repair, or the finiteness check, and it cannot make
+    the stage cost less than nothing. So the wall-time difference between
+    this and the real correction bounds from above what fusing could
+    possibly buy, and it can be measured without writing the fused kernel.
+
+    Which voxels a pixel reaches depends on geometry and the mask, never
+    on the recorded intensity, so this arm routes the identical records
+    and keeps the checkpoint fingerprint comparable; only the intensities
+    and variances it carries are wrong, which is why it is a probe and
+    nothing else.
+
+    :param correct:
+        The real pipeline, for its repair plan.
+    :param static_mask:
+        The job's static mask, or ``None``.
+    """
+    repair_plan = getattr(correct, "repair_plan", None)
+
+    def correct_frame(payload, raw, frame_index):
+        source_variance = getattr(payload, "variance", None)
+        if source_variance is None:
+            variance = np.maximum(raw, 0.0)
+        else:
+            variance = np.asarray(source_variance, dtype=np.float64).copy()
+        intensity = np.asarray(raw, dtype=np.float64).copy()
+        if static_mask is None:
+            mask = np.zeros(intensity.shape, dtype=bool)
+        else:
+            mask = static_mask.copy()
+        if repair_plan is not None:
+            mask, _ = repair_plan.apply_inplace(intensity, variance)
+        mask |= ~np.isfinite(intensity) | ~np.isfinite(variance)
+        return intensity, variance, mask
+
+    def pipeline(payload, raw, frame_index, tile):
+        row_start, row_stop, column_start, column_stop = tile
+        full = np.asarray(getattr(payload, "img", raw))
+        intensity, variance, mask = correct_frame(payload, full, frame_index)
+        selection = np.s_[row_start:row_stop, column_start:column_stop]
+        return intensity[selection], variance[selection], mask[selection]
+
+    pipeline.correct_frame = correct_frame
+    pipeline.repair_plan = repair_plan
+    return pipeline
+
+
+class GilProbe(threading.Thread):
+    """What share of its wall time a Python thread spends waiting for the GIL.
+
+    Every stage's ``blocked`` column mixes GIL waiting with disk waiting
+    and preemption, and no arithmetic separates them. This does: it runs
+    bursts of pure Python -- no I/O, no NumPy, nothing that could release
+    the GIL voluntarily -- and compares each burst's wall time against the
+    cycles it actually executed. A burst is deliberately several times the
+    interpreter's switch interval, so the GIL is dropped and recontended
+    for during it rather than held straight through.
+
+    Blocked fraction 0 means the GIL was there whenever this thread wanted
+    it. Near 1 means a Python thread wanting to run mostly cannot, which
+    is what "the pipeline is GIL-bound" would look like.
+
+    The probe competes for the same GIL it measures, so it runs at a low
+    duty cycle and its own cost is reported alongside its answer.
+    """
+
+    def __init__(self, cycle_rate, *, burst_seconds=0.02, period_seconds=0.25):
+        super().__init__(name="orgui-gil-probe", daemon=True)
+        self._cycle_rate = cycle_rate
+        self._burst_cycles = int(burst_seconds * cycle_rate)
+        self._period = period_seconds
+        self._halt = threading.Event()
+        self.wall = 0.0
+        self.cycles = 0
+        self.bursts = 0
+
+    def halt(self):
+        self._halt.set()
+
+    def run(self):
+        while not self._halt.is_set():
+            start_wall = perf_counter()
+            start_cycles = thread_cycles()
+            value = 0
+            while thread_cycles() - start_cycles < self._burst_cycles:
+                for _iteration in range(2000):
+                    value += 1
+            self.wall += perf_counter() - start_wall
+            self.cycles += thread_cycles() - start_cycles
+            self.bursts += 1
+            self._halt.wait(self._period)
+
+    def summary(self):
+        """:rtype: dict"""
+        on_core = self.cycles / self._cycle_rate if self._cycle_rate else 0.0
+        return {
+            "bursts": self.bursts,
+            "wall_seconds": self.wall,
+            "on_core_seconds": on_core,
+            "blocked_fraction": (
+                (self.wall - on_core) / self.wall if self.wall else None
+            ),
+        }
+
+
+def uncontended(scan, correct, frame_indices, cycle_rate):
+    """Load and correct a few frames serially, before the pipeline starts.
+
+    The reference the in-pipeline numbers are read against: one thread, no
+    GIL competition, cold cache. A stage whose in-pipeline on-core time
+    matches this is contending for the GIL or the disk but not for memory
+    bandwidth; one that inflates is doing the same arithmetic more slowly
+    because 24 threads are sharing the same cache and controller.
+
+    :param frame_indices:
+        Frames outside the measured window -- reading a frame here would
+        otherwise leave it warm in the page cache for the run.
+    :rtype: dict
+    """
+    correct_frame = correct.correct_frame
+    totals = {"load": [0.0, 0], "correct": [0.0, 0]}
+    for frame_index in frame_indices:
+        wall = perf_counter()
+        cycles = thread_cycles()
+        payload = scan.get_raw_img(frame_index)
+        totals["load"][0] += perf_counter() - wall
+        totals["load"][1] += thread_cycles() - cycles
+        raw = np.asarray(payload.img)
+        wall = perf_counter()
+        cycles = thread_cycles()
+        correct_frame(payload, raw, frame_index)
+        totals["correct"][0] += perf_counter() - wall
+        totals["correct"][1] += thread_cycles() - cycles
+    count = max(1, len(frame_indices))
+    return {
+        name: {
+            "wall_ms_per_frame": wall / count * 1e3,
+            "on_core_ms_per_frame": cycles / cycle_rate / count * 1e3,
+        }
+        for name, (wall, cycles) in totals.items()
+    }
+
+
+class _TimedKernel:
+    """Forwarding proxy timing a kernel's accumulate calls."""
+
+    def __init__(self, kernel):
+        self._kernel = kernel
+
+    def __getattr__(self, name):
+        return getattr(self._kernel, name)
+
+    def accumulate_group(self, *arguments):
+        with timed("kernel"):
+            return self._kernel.accumulate_group(*arguments)
+
+    def accumulate_group_tile(self, *arguments):
+        with timed("kernel"):
+            return self._kernel.accumulate_group_tile(*arguments)
+
+    def accumulate(self, *arguments):
+        with timed("kernel"):
+            return self._kernel.accumulate(*arguments)
+
+
+def _install(scan, correct, router, *, stack_threshold_bytes=1 << 20):
+    """Wrap every stage boundary in place; returns the patched pipeline.
+
+    Patches module globals rather than editing the pipeline, so the code
+    being measured is exactly the code that ships. ``np.stack`` is patched
+    globally but attributed only above ``stack_threshold_bytes``, which
+    separates ``_map_frame_group``'s per-tile buffers from the small
+    stacks the rest of the process makes.
+    """
+    scan_read = scan.get_raw_img
+
+    def get_raw_img(frame_index, *arguments, **keywords):
+        with timed("load"):
+            return scan_read(frame_index, *arguments, **keywords)
+
+    scan.get_raw_img = get_raw_img
+
+    correct_frame = correct.correct_frame
+
+    def timed_correct_frame(payload, raw, frame_index):
+        with timed("correct"):
+            return correct_frame(payload, raw, frame_index)
+
+    def timed_correct(payload, raw, frame_index, tile):
+        with timed("correct_tile"):
+            return correct(payload, raw, frame_index, tile)
+
+    timed_correct.correct_frame = timed_correct_frame
+    timed_correct.repair_plan = correct.repair_plan
+
+    build_kernels = reconstruction_job._build_kernels
+
+    def timed_build_kernels(*arguments, **keywords):
+        return {
+            name: _TimedKernel(kernel)
+            for name, kernel in build_kernels(*arguments, **keywords).items()
+        }
+
+    reconstruction_job._build_kernels = timed_build_kernels
+
+    reduce_batches = reconstruction._reduce_batches
+
+    def timed_reduce_batches(batches):
+        with timed("merge_tiles"):
+            return reduce_batches(batches)
+
+    reconstruction._reduce_batches = timed_reduce_batches
+
+    map_frame_group = reconstruction_job._map_frame_group
+
+    def timed_map_frame_group(*arguments, **keywords):
+        with timed("map_frame_group"):
+            return map_frame_group(*arguments, **keywords)
+
+    reconstruction_job._map_frame_group = timed_map_frame_group
+
+    numpy_stack = np.stack
+
+    def timed_stack(arrays, *arguments, **keywords):
+        # Sized from the inputs, so a small stack anywhere else in the
+        # process is never timed rather than timed and then discounted.
+        first = arrays[0] if isinstance(arrays, list | tuple) and arrays else None
+        if getattr(first, "nbytes", 0) * len(arrays) < stack_threshold_bytes:
+            return numpy_stack(arrays, *arguments, **keywords)
+        with timed("tile_stack"):
+            return numpy_stack(arrays, *arguments, **keywords)
+
+    np.stack = timed_stack
+
+    route = router.route
+    routed = {"records": 0, "calls": 0}
+
+    def timed_route(grid_name, frame_index, batch, **keywords):
+        routed["records"] += int(batch["chunk_id"].size)
+        routed["calls"] += 1
+        with timed("route"):
+            return route(grid_name, frame_index, batch, **keywords)
+
+    router.route = timed_route
+    return timed_correct, routed
+
+
+def _arguments():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("job", type=Path)
+    parser.add_argument("--start", type=int, default=1638)
+    parser.add_argument("--count", type=int, default=234)
+    parser.add_argument("--ranges-per-checkpoint", type=int, default=2)
+    parser.add_argument("--group", type=int, default=None)
+    parser.add_argument("--threads-per-image", type=int, default=None)
+    parser.add_argument("--memory-gib", type=float, default=None)
+    parser.add_argument("--tile-rows", type=int, default=None)
+    parser.add_argument("--depth", type=int, default=None)
+    parser.add_argument(
+        "--baseline-frames",
+        type=int,
+        default=3,
+        help=(
+            "Frames to load and correct serially before the run, as the "
+            "uncontended reference. Taken from just before the window."
+        ),
+    )
+    parser.add_argument(
+        "--correction",
+        choices=("full", "minimal"),
+        default="full",
+        help=(
+            "'minimal' keeps the mask and drops the scaling arithmetic, as "
+            "an upper bound on what fusing correction could buy. It routes "
+            "the same records and carries wrong intensities."
+        ),
+    )
+    parser.add_argument("--scratch", type=Path)
+    parser.add_argument("--output", type=Path)
+    return parser.parse_args()
+
+
+def main():
+    """Run the real mapping pipeline and report where its time went."""
+    arguments = _arguments()
+    cycle_rate = calibrate_cycle_rate()
+
+    job = read_job(arguments.job)
+    config = job.config_data
+    scan = job.scan
+    spec = job.internal_spec()
+    if arguments.depth is not None:
+        spec = replace(
+            spec,
+            max_depth=arguments.depth,
+            work_block_pixels=resolve_work_block_pixels(
+                job.work_block_pixels,
+                arguments.depth,
+                spec.memory_budget_bytes,
+                spec.threads,
+            ),
+        )
+    if arguments.group is not None:
+        spec = replace(spec, frames_per_group=arguments.group)
+    bounds = scan.exposure_angle_bounds(config, fallback=job.angle_fallback)
+    ranges, tiles = _execution_layout(job, scan, config)
+    if arguments.tile_rows:
+        detector_rows, detector_columns = config.detector.detector.shape
+        tiles = [
+            (row, min(row + arguments.tile_rows, detector_rows), 0, detector_columns)
+            for row in range(0, detector_rows, arguments.tile_rows)
+        ]
+    kept, boundaries = _window_ranges(
+        ranges,
+        arguments.start,
+        arguments.start + arguments.count,
+        arguments.ranges_per_checkpoint,
+    )
+    grid_names = [grid.grid_name for grid in spec.grids]
+    plan = {grid_name: list(boundaries) for grid_name in grid_names}
+
+    effective_memory = (
+        int(arguments.memory_gib * 1024**3)
+        if arguments.memory_gib
+        else job.memory_override_bytes or job.runtime_memory_bytes
+    )
+    active_budget_bytes, effective_memory = split_memory_budget(
+        effective_memory, max(1, len(spec.grids))
+    )
+
+    scratch = Path(
+        arguments.scratch or tempfile.mkdtemp(prefix="orgui-rsmap-stages-")
+    )
+    checkpoint_dir = scratch / "checkpoints"
+    if checkpoint_dir.exists():
+        shutil.rmtree(checkpoint_dir)
+    router = _CheckpointRouter(
+        plan,
+        spec_digest=job.digest,
+        checkpoint_dir=checkpoint_dir,
+        active_budget_bytes=active_budget_bytes,
+    )
+    assets = _load_assets(job)
+    correct = _correction_pipeline(
+        config, scan, assets, _base_provenance(job, config)
+    )
+    if arguments.correction == "minimal":
+        correct = minimal_correction(
+            correct,
+            np.ascontiguousarray(assets["mask"], dtype=bool)
+            if "mask" in assets
+            else None,
+        )
+    baseline = (
+        uncontended(
+            scan,
+            correct,
+            [
+                frame
+                for frame in range(
+                    kept[0][0] - arguments.baseline_frames, kept[0][0]
+                )
+                if frame >= 0
+            ],
+            cycle_rate,
+        )
+        if arguments.baseline_frames
+        else None
+    )
+    timed_correct, routed = _install(scan, correct, router)
+    total_images = sum(stop - start for start, stop in kept)
+
+    grid = spec.grids[0]
+    header = {
+        "job": str(arguments.job),
+        "frames": [kept[0][0], kept[-1][1]],
+        "total_images": total_images,
+        "scheduling_ranges": len(kept),
+        "checkpoints": len(boundaries),
+        "tiles": len(tiles),
+        "grid_shape": list(grid.shape),
+        "grid_step": [float(step) for step in grid.step],
+        "frames_per_group": getattr(spec, "frames_per_group", None),
+        "max_depth": spec.max_depth,
+        "work_block_pixels": spec.work_block_pixels,
+        "threads": spec.threads,
+        "angle_fallback": job.angle_fallback,
+        "effective_memory_bytes": int(effective_memory),
+        "correction": arguments.correction,
+        "cycle_rate_ghz": round(cycle_rate / 1e9, 4),
+        "native": _native_module().__file__,
+        "scratch": str(scratch),
+    }
+    print(json.dumps(header, sort_keys=True), flush=True)
+
+    messages = []
+
+    def capture(value, maximum, message):
+        messages.append(message)
+
+    process = None
+    try:
+        import psutil
+
+        process = psutil.Process()
+        cpu_before = process.cpu_times()
+    except ImportError:
+        cpu_before = None
+
+    idle_probe = GilProbe(cycle_rate)
+    idle_probe.start()
+    idle_probe._halt.wait(1.0)
+    idle_probe.halt()
+    idle_probe.join(timeout=5.0)
+    probe = GilProbe(cycle_rate)
+    probe.start()
+
+    started = perf_counter()
+    _map_pending_ranges(
+        spec,
+        scan,
+        config,
+        bounds,
+        tiles,
+        kept,
+        router,
+        correction_pipeline=timed_correct,
+        effective_memory=effective_memory,
+        threads_per_image=(
+            arguments.threads_per_image
+            if arguments.threads_per_image is not None
+            else job.threads_per_image
+        ),
+        accumulation_budget_bytes=job.accumulation_budget_bytes,
+        total_images=total_images,
+        completed_images=0,
+        progress=capture,
+    )
+    elapsed = perf_counter() - started
+    probe.halt()
+    probe.join(timeout=5.0)
+    process_cpu = None
+    if process is not None:
+        cpu_after = process.cpu_times()
+        process_cpu = (
+            cpu_after.user - cpu_before.user + cpu_after.system - cpu_before.system
+        )
+
+    stages = {}
+    for name, (wall, cycles, calls) in merged_stages().items():
+        on_core = cycles / cycle_rate if cycle_rate else float("nan")
+        stages[name] = {
+            "calls": calls,
+            "wall_ms_per_frame": wall / total_images * 1e3,
+            "on_core_ms_per_frame": on_core / total_images * 1e3,
+            "blocked_ms_per_frame": (wall - on_core) / total_images * 1e3,
+        }
+    # The one stage with no call of its own: what _map_frame_group spends
+    # outside the kernel, the merge and the route is the per-tile slice
+    # and stack, plus the Python around them.
+    parent = stages.get("map_frame_group")
+    if parent is not None:
+        # Correction is deliberately absent: on the grouped scheduler it
+        # runs in the prepare pool, outside this call entirely.
+        children = ("kernel", "merge_tiles", "route", "correct_tile")
+        assembly = {
+            key: parent[key]
+            - sum(stages.get(child, {}).get(key, 0.0) for child in children)
+            for key in (
+                "wall_ms_per_frame",
+                "on_core_ms_per_frame",
+                "blocked_ms_per_frame",
+            )
+        }
+        assembly["calls"] = parent["calls"]
+        stages["tile_assembly (derived)"] = assembly
+
+    # Every stage's on-core time is the *calling* thread's. The kernel
+    # spawns its own native worker pool, whose cycles no Python thread
+    # ever accounts for, so what the process spent minus what the Python
+    # threads spent is (to within the instrument's reach) the native
+    # pool's own CPU.
+    python_cpu_ms_per_frame = sum(
+        values["on_core_ms_per_frame"]
+        for name, values in stages.items()
+        if name in {"load", "correct", "correct_tile", "map_frame_group"}
+    )
+    result = {
+        "seconds": elapsed,
+        "seconds_per_frame": elapsed / total_images,
+        "ms_per_frame": elapsed / total_images * 1e3,
+        "routed_records_per_frame": routed["records"] / total_images,
+        "process_cpu_seconds": process_cpu,
+        "process_cpu_ms_per_frame": (
+            process_cpu / total_images * 1e3 if process_cpu else None
+        ),
+        "python_thread_cpu_ms_per_frame": python_cpu_ms_per_frame,
+        "cores_in_use": (process_cpu / elapsed) if process_cpu else None,
+        "final_layout": messages[-1] if messages else None,
+        "gil_probe": probe.summary(),
+        "gil_probe_idle": idle_probe.summary(),
+        "uncontended": baseline,
+        "stages": stages,
+        "grids": _summarize_checkpoints(checkpoint_dir, grid_names),
+    }
+    print(json.dumps(result, sort_keys=True), flush=True)
+
+    print()
+    print(
+        f"{total_images} frames in {elapsed:.1f} s "
+        f"({elapsed / total_images * 1e3:.1f} ms/frame)"
+    )
+    if process_cpu:
+        print(
+            f"  {process_cpu:.1f} s of CPU -> "
+            f"{process_cpu / elapsed:.2f} cores in use "
+            f"({process_cpu / total_images * 1e3:.0f} ms of CPU per frame, "
+            f"{python_cpu_ms_per_frame:.0f} of it on Python threads)"
+        )
+    during = probe.summary()
+    idle = idle_probe.summary()
+    if during["blocked_fraction"] is not None:
+        print(
+            f"  GIL probe: a Python thread waited "
+            f"{during['blocked_fraction'] * 100:.0f}% of its wall time during "
+            f"the run ({idle['blocked_fraction'] * 100:.0f}% idle), over "
+            f"{during['bursts']} bursts"
+        )
+    if baseline:
+        print(
+            "  uncontended, one thread: "
+            + ", ".join(
+                f"{name} {values['wall_ms_per_frame']:.1f} ms wall / "
+                f"{values['on_core_ms_per_frame']:.1f} on-core"
+                for name, values in baseline.items()
+            )
+        )
+    if messages:
+        print(f"  {messages[-1]}")
+    print()
+    print(
+        f"  {'stage':<26}{'calls':>8}{'wall':>10}{'on-core':>10}"
+        f"{'blocked':>10}   (ms per frame)"
+    )
+    order = [
+        "load",
+        "correct",
+        "correct_tile",
+        "map_frame_group",
+        "tile_assembly (derived)",
+        "tile_stack",
+        "kernel",
+        "merge_tiles",
+        "route",
+    ]
+    for name in order:
+        values = stages.get(name)
+        if values is None:
+            continue
+        indent = "    " if name in {
+            "tile_assembly (derived)",
+            "tile_stack",
+            "kernel",
+            "merge_tiles",
+            "route",
+        } else "  "
+        print(
+            f"  {indent}{name:<{26 - len(indent) + 2}}{values['calls']:>8}"
+            f"{values['wall_ms_per_frame']:>10.1f}"
+            f"{values['on_core_ms_per_frame']:>10.1f}"
+            f"{values['blocked_ms_per_frame']:>10.1f}"
+        )
+    for grid_name, values in result["grids"].items():
+        print(
+            f"  {grid_name}: {values['rows']:,} rows, "
+            f"{values['contributors']:,} contributors, "
+            f"fingerprint {values['voxel_fingerprint']}"
+        )
+
+    if arguments.output:
+        arguments.output.write_text(
+            json.dumps({"header": header, "result": result}, indent=2),
+            encoding="utf-8",
+        )
+    if arguments.scratch is None:
+        shutil.rmtree(scratch, ignore_errors=True)
+
+
+if __name__ == "__main__":
+    main()

From 6a873bcdbaa808132afcaa498d5e45865219db4b Mon Sep 17 00:00:00 2001
From: "T. Fuchs" <tfuchs@cornell.edu>
Date: Wed, 12 Aug 2026 11:12:10 -0400
Subject: [PATCH 31/33] perf: read a detector tile in place instead of copying
 it

Detector tiles partition the detector, so gathering each tile into its
own (frames, rows, columns) buffer copied every corrected frame exactly
once per group -- ~105 MB a frame, in Python, on the compute worker --
purely to give accumulate_group a shape it did not need.

accumulate_group_tile takes the whole frames plus the tile rectangle and
reads it through the frame's own row stride. Both entry points build the
same GroupPixels view and run the identical brick loop, so this is
bit-for-bit: measured 0.840x end to end at depth 0 over six interleaved
pairs, with identical voxels, contributors and totals.
---
 .../cpp/reciprocal_reconstruction_cpp.cpp     | 271 +++++++++++++++++-
 orgui/datautils/xrayutils/reconstruction.py   |  71 +++--
 .../test/test_reconstruction_group.py         | 135 +++++++++
 3 files changed, 442 insertions(+), 35 deletions(-)

diff --git a/orgui/datautils/xrayutils/cpp/reciprocal_reconstruction_cpp.cpp b/orgui/datautils/xrayutils/cpp/reciprocal_reconstruction_cpp.cpp
index 7d8b52a..fe8978a 100644
--- a/orgui/datautils/xrayutils/cpp/reciprocal_reconstruction_cpp.cpp
+++ b/orgui/datautils/xrayutils/cpp/reciprocal_reconstruction_cpp.cpp
@@ -554,6 +554,23 @@ struct BrickExtent {
     std::size_t column_end;
 };
 
+// A frame group's pixel data as the caller already holds it: one pointer
+// per frame at the tile's first pixel, plus the distance between two
+// rows of the caller's own array.
+//
+// The stride is what makes the tile a view rather than a copy. A tile cut
+// out of whole corrected frames strides by the frame's row length and is
+// read in place; a tile that has been gathered into its own
+// ``(frames, rows, columns)`` buffer strides by the tile's own row
+// length. The brick loop cannot tell the two apart, and reads the same
+// values in the same order either way.
+struct GroupPixels {
+    std::vector<const double *> intensity;
+    std::vector<const double *> variance;
+    std::vector<const bool *> mask;
+    std::size_t row_stride = 0;
+};
+
 struct BlockProfile {
     std::uint64_t pixels_seen = 0;
     std::uint64_t valid_pixels = 0;
@@ -846,6 +863,12 @@ class ReconstructionKernel {
         //
         // The caller chooses the group size by how many frames it passes;
         // one frame reproduces accumulate() exactly.
+        //
+        // This form takes one contiguous ``(frames, rows, columns)``
+        // buffer per array. :meth:`accumulate_group_tile` takes the same
+        // data as whole frames plus a rectangle, and is what the mapping
+        // pipeline uses; both run the identical brick loop over the
+        // identical values.
         const auto total_started = std::chrono::steady_clock::now();
         const py::buffer_info intensity_info = intensity.request();
         const py::buffer_info variance_info = variance.request();
@@ -867,9 +890,6 @@ class ReconstructionKernel {
         const auto *variance_data =
             static_cast<const double *>(variance_info.ptr);
         const auto *mask_data = static_cast<const bool *>(mask_info.ptr);
-        const auto *ray_data = static_cast<const double *>(ray_info.ptr);
-        const auto *start_data = static_cast<const double *>(start_info.ptr);
-        const auto *end_data = static_cast<const double *>(end_info.ptr);
 
         const std::size_t frames =
             static_cast<std::size_t>(intensity_info.shape[0]);
@@ -877,6 +897,148 @@ class ReconstructionKernel {
             static_cast<std::size_t>(intensity_info.shape[1]);
         const std::size_t cols =
             static_cast<std::size_t>(intensity_info.shape[2]);
+
+        GroupPixels pixels;
+        pixels.row_stride = cols;
+        pixels.intensity.reserve(frames);
+        pixels.variance.reserve(frames);
+        pixels.mask.reserve(frames);
+        for (std::size_t frame = 0; frame < frames; ++frame) {
+            const std::size_t offset = frame * rows * cols;
+            pixels.intensity.push_back(intensity_data + offset);
+            pixels.variance.push_back(variance_data + offset);
+            pixels.mask.push_back(mask_data + offset);
+        }
+        return accumulate_group_impl(
+            pixels,
+            frames,
+            rows,
+            cols,
+            static_cast<const double *>(ray_info.ptr),
+            static_cast<const double *>(start_info.ptr),
+            static_cast<const double *>(end_info.ptr),
+            profile,
+            total_started
+        );
+    }
+
+    py::dict accumulate_group_tile(
+        const py::sequence &intensity,
+        const py::sequence &variance,
+        const py::sequence &mask,
+        const FloatArray &corner_rays,
+        const FloatArray &angles_start,
+        const FloatArray &angles_end,
+        const std::size_t row_start,
+        const std::size_t row_stop,
+        const std::size_t column_start,
+        const std::size_t column_stop,
+        const bool profile = false
+    ) const {
+        // The same call as :meth:`accumulate_group`, taking whole frames
+        // and the rectangle to map instead of a buffer already cut down
+        // to it.
+        //
+        // Detector tiles partition the detector, so gathering each tile
+        // into its own contiguous buffer copies every corrected frame
+        // exactly once per group -- around 105 MB a frame, in Python,
+        // holding the GIL, purely to give this call a shape it does not
+        // need. The brick loop walks rows within a tile in either form,
+        // so reading through the frame's own row stride costs it nothing.
+        const auto total_started = std::chrono::steady_clock::now();
+        const py::buffer_info ray_info = corner_rays.request();
+        const py::buffer_info start_info = angles_start.request();
+        const py::buffer_info end_info = angles_end.request();
+
+        const std::size_t frames = static_cast<std::size_t>(py::len(intensity));
+        if (frames < 1) {
+            throw py::value_error("A frame group needs at least one frame");
+        }
+        if (
+            static_cast<std::size_t>(py::len(variance)) != frames
+            || static_cast<std::size_t>(py::len(mask)) != frames
+        ) {
+            throw py::value_error(
+                "variance and mask must hold one frame each, as intensity does"
+            );
+        }
+        if (row_stop <= row_start || column_stop <= column_start) {
+            throw py::value_error("The tile rectangle must be non-empty");
+        }
+        const std::size_t rows = row_stop - row_start;
+        const std::size_t cols = column_stop - column_start;
+
+        // The casts hold references for as long as this call runs, which
+        // is what keeps the frames alive while the GIL is released.
+        std::vector<FloatArray> intensity_frames;
+        std::vector<FloatArray> variance_frames;
+        std::vector<BoolArray> mask_frames;
+        intensity_frames.reserve(frames);
+        variance_frames.reserve(frames);
+        mask_frames.reserve(frames);
+        GroupPixels pixels;
+        pixels.intensity.reserve(frames);
+        pixels.variance.reserve(frames);
+        pixels.mask.reserve(frames);
+        std::size_t frame_columns = 0;
+        for (std::size_t frame = 0; frame < frames; ++frame) {
+            intensity_frames.push_back(intensity[frame].cast<FloatArray>());
+            variance_frames.push_back(variance[frame].cast<FloatArray>());
+            mask_frames.push_back(mask[frame].cast<BoolArray>());
+            const py::buffer_info intensity_info =
+                intensity_frames.back().request();
+            const py::buffer_info variance_info =
+                variance_frames.back().request();
+            const py::buffer_info mask_info = mask_frames.back().request();
+            validate_tile_frame(
+                intensity_info, variance_info, mask_info, row_stop, column_stop
+            );
+            const std::size_t columns_here =
+                static_cast<std::size_t>(intensity_info.shape[1]);
+            if (frame == 0) {
+                frame_columns = columns_here;
+            } else if (columns_here != frame_columns) {
+                throw py::value_error(
+                    "every frame in a group must have the same shape"
+                );
+            }
+            const std::size_t offset = row_start * frame_columns + column_start;
+            pixels.intensity.push_back(
+                static_cast<const double *>(intensity_info.ptr) + offset
+            );
+            pixels.variance.push_back(
+                static_cast<const double *>(variance_info.ptr) + offset
+            );
+            pixels.mask.push_back(
+                static_cast<const bool *>(mask_info.ptr) + offset
+            );
+        }
+        pixels.row_stride = frame_columns;
+        validate_group_geometry(ray_info, start_info, end_info, frames, rows, cols);
+        return accumulate_group_impl(
+            pixels,
+            frames,
+            rows,
+            cols,
+            static_cast<const double *>(ray_info.ptr),
+            static_cast<const double *>(start_info.ptr),
+            static_cast<const double *>(end_info.ptr),
+            profile,
+            total_started
+        );
+    }
+
+    py::dict accumulate_group_impl(
+        const GroupPixels &pixels,
+        const std::size_t frames,
+        const std::size_t rows,
+        const std::size_t cols,
+        const double *ray_data,
+        const double *start_data,
+        const double *end_data,
+        const bool profile,
+        const std::chrono::steady_clock::time_point total_started
+    ) const {
         const std::size_t frame_pixels = rows * cols;
         const std::size_t samples = frames * frame_pixels;
 
@@ -992,11 +1154,8 @@ class ReconstructionKernel {
                         block_results[index] = accumulate_brick(
                             bricks[index],
                             frames,
-                            rows,
                             cols,
-                            intensity_data,
-                            variance_data,
-                            mask_data,
+                            pixels,
                             ray_data,
                             transforms,
                             stationary,
@@ -1192,6 +1351,75 @@ class ReconstructionKernel {
         return details;
     }
 
+    static void validate_tile_frame(
+        const py::buffer_info &intensity,
+        const py::buffer_info &variance,
+        const py::buffer_info &mask,
+        const std::size_t row_stop,
+        const std::size_t column_stop
+    ) {
+        // The frames are read in place, so their layout is part of the
+        // contract rather than something a cast quietly repairs: one row
+        // stride has to serve every frame and every array.
+        if (intensity.ndim != 2) {
+            throw py::value_error(
+                "each frame's intensity must be a two-dimensional "
+                "(rows, columns) array"
+            );
+        }
+        for (const py::buffer_info *other : {&variance, &mask}) {
+            if (
+                other->ndim != 2
+                || other->shape[0] != intensity.shape[0]
+                || other->shape[1] != intensity.shape[1]
+            ) {
+                throw py::value_error(
+                    "each frame's variance and mask must match its intensity "
+                    "shape"
+                );
+            }
+        }
+        if (
+            static_cast<std::size_t>(intensity.shape[0]) < row_stop
+            || static_cast<std::size_t>(intensity.shape[1]) < column_stop
+        ) {
+            throw py::value_error(
+                "the tile rectangle reaches outside the frame"
+            );
+        }
+    }
+
+    static void validate_group_geometry(
+        const py::buffer_info &rays,
+        const py::buffer_info &start,
+        const py::buffer_info &end,
+        const std::size_t frames,
+        const std::size_t rows,
+        const std::size_t cols
+    ) {
+        if (
+            rays.ndim != 3
+            || static_cast<std::size_t>(rays.shape[0]) != rows + 1
+            || static_cast<std::size_t>(rays.shape[1]) != cols + 1
+            || rays.shape[2] != 3
+        ) {
+            throw py::value_error(
+                "corner_rays must have shape (rows + 1, columns + 1, 3)"
+            );
+        }
+        for (const py::buffer_info *angles : {&start, &end}) {
+            if (
+                angles->ndim != 2
+                || static_cast<std::size_t>(angles->shape[0]) != frames
+                || angles->shape[1] != 4
+            ) {
+                throw py::value_error(
+                    "angles_start and angles_end must have shape (frames, 4)"
+                );
+            }
+        }
+    }
+
     static void validate_group_inputs(
         const py::buffer_info &intensity,
         const py::buffer_info &variance,
@@ -2071,11 +2299,8 @@ class ReconstructionKernel {
     std::vector<Record> accumulate_brick(
         const BrickExtent &brick,
         const std::size_t frames,
-        const std::size_t rows,
         const std::size_t columns,
-        const double *intensity,
-        const double *variance,
-        const bool *mask,
+        const GroupPixels &pixels,
         const double *rays,
         const std::vector<std::vector<CoordinateTransform>> &transforms,
         const std::vector<char> &stationary,
@@ -2084,7 +2309,6 @@ class ReconstructionKernel {
         BlockProfile *profile,
         std::pmr::monotonic_buffer_resource &arena
     ) const {
-        (void)rows;
         arena.release();
         std::pmr::map<RecordKey, RecordAccum> tree(&arena);
         std::uint64_t unreduced_count = 0;
@@ -2103,7 +2327,7 @@ class ReconstructionKernel {
         };
         const std::size_t brick_rows = brick.row_end - brick.row_begin;
         const std::size_t brick_columns = brick.column_end - brick.column_begin;
-        const std::size_t frame_stride = rows * columns;
+        const std::size_t row_stride = pixels.row_stride;
 
         if (max_depth_ == 0) {
             // A pixel's centre ray is pure detector geometry -- the frame's
@@ -2132,9 +2356,11 @@ class ReconstructionKernel {
                 transforms[frame];
             const CoordinateTransform &centre_transform =
                 frame_transforms[frame_transforms.size() / 2];
+            const double *const intensity = pixels.intensity[frame];
+            const double *const variance = pixels.variance[frame];
+            const bool *const mask = pixels.mask[frame];
             for (std::size_t row = brick.row_begin; row < brick.row_end; ++row) {
-                const std::size_t row_offset =
-                    frame * frame_stride + row * columns;
+                const std::size_t row_offset = row * row_stride;
                 for (
                     std::size_t column = brick.column_begin;
                     column < brick.column_end;
@@ -2744,6 +2970,21 @@ PYBIND11_MODULE(_reciprocal_reconstruction_cpp, module) {
             py::arg("angles_end"),
             py::arg("profile") = false
         )
+        .def(
+            "accumulate_group_tile",
+            &ReconstructionKernel::accumulate_group_tile,
+            py::arg("intensity"),
+            py::arg("variance"),
+            py::arg("mask"),
+            py::arg("corner_rays"),
+            py::arg("angles_start"),
+            py::arg("angles_end"),
+            py::arg("row_start"),
+            py::arg("row_stop"),
+            py::arg("column_start"),
+            py::arg("column_stop"),
+            py::arg("profile") = false
+        )
         .def(
             "coordinate",
             &ReconstructionKernel::coordinate,
diff --git a/orgui/datautils/xrayutils/reconstruction.py b/orgui/datautils/xrayutils/reconstruction.py
index 866fe41..892c458 100644
--- a/orgui/datautils/xrayutils/reconstruction.py
+++ b/orgui/datautils/xrayutils/reconstruction.py
@@ -1585,29 +1585,60 @@ def _map_frame_group(
     tile_batches: dict[str, list[Mapping[str, np.ndarray]]] = {
         grid.grid_name: [] for grid in spec.grids
     }
+    if corrected_frames is not None:
+        # Checked once for the whole group rather than once per tile, and
+        # a no-op on what ``correct_frame`` returns. The kernel reads
+        # these in place, so the layout it needs is settled here instead
+        # of being repaired six times over by the tile loop below.
+        corrected_frames = [
+            (
+                np.ascontiguousarray(intensity, dtype=np.float64),
+                np.ascontiguousarray(variance, dtype=np.float64),
+                np.ascontiguousarray(mask, dtype=bool),
+            )
+            for intensity, variance, mask in corrected_frames
+        ]
     for detector_tile in detector_tiles:
         row_start, row_stop, column_start, column_stop = detector_tile
-        selection = np.s_[row_start:row_stop, column_start:column_stop]
-        if corrected_frames is None:
-            tile_values = [
-                correction_pipeline(
-                    payload,
-                    image[selection],
-                    frame_index,
-                    detector_tile,
+        if corrected_frames is not None:
+            # Whole frames plus the rectangle to map. Detector tiles
+            # partition the detector, so gathering each tile into its own
+            # contiguous buffer would copy every corrected frame exactly
+            # once per group -- ~105 MB a frame, in Python, under the GIL
+            # -- purely to give the call a shape it does not need. The
+            # kernel already walks rows within a tile, so it reads the
+            # rectangle through the frame's own row stride instead.
+            for grid in spec.grids:
+                batch = kernels[grid.grid_name].accumulate_group_tile(
+                    [values[0] for values in corrected_frames],
+                    [values[1] for values in corrected_frames],
+                    [values[2] for values in corrected_frames],
+                    ray_arrays[detector_tile],
+                    angles_start,
+                    angles_end,
+                    row_start,
+                    row_stop,
+                    column_start,
+                    column_stop,
                 )
-                for payload, image, frame_index in zip(
-                    image_payloads, images, frame_indices
-                )
-            ]
-        else:
-            tile_values = [
-                tuple(values[selection] for values in corrected)
-                for corrected in corrected_frames
-            ]
-        # One contiguous (frames, rows, columns) buffer per array, which
-        # is the copy the per-tile ascontiguousarray already made for a
-        # single frame -- the group spends it over F frames instead of 1.
+                tile_batches[grid.grid_name].append(batch)
+            continue
+        # A pipeline with no whole-frame ``correct_frame`` step corrects
+        # per tile, and its output is tile-sized already: there is no
+        # whole frame to take a view of, so the group's frames are
+        # stacked into one contiguous buffer as before.
+        selection = np.s_[row_start:row_stop, column_start:column_stop]
+        tile_values = [
+            correction_pipeline(
+                payload,
+                image[selection],
+                frame_index,
+                detector_tile,
+            )
+            for payload, image, frame_index in zip(
+                image_payloads, images, frame_indices
+            )
+        ]
         intensity = np.ascontiguousarray(
             np.stack([values[0] for values in tile_values]), dtype=np.float64
         )
diff --git a/orgui/datautils/xrayutils/test/test_reconstruction_group.py b/orgui/datautils/xrayutils/test/test_reconstruction_group.py
index 87feecd..e7c85b9 100644
--- a/orgui/datautils/xrayutils/test/test_reconstruction_group.py
+++ b/orgui/datautils/xrayutils/test/test_reconstruction_group.py
@@ -293,6 +293,141 @@ def test_group_emits_fewer_records_than_the_frames_do_apart():
     assert brick_records < 0.85 * apart
 
 
+def _tile_arguments(rays, intensity, variance, mask, tile):
+    """The same group, as whole frames plus a rectangle."""
+    row_start, row_stop, column_start, column_stop = tile
+    return (
+        [np.ascontiguousarray(frame) for frame in intensity],
+        [np.ascontiguousarray(frame) for frame in variance],
+        [np.ascontiguousarray(frame) for frame in mask],
+        np.ascontiguousarray(
+            rays[row_start : row_stop + 1, column_start : column_stop + 1]
+        ),
+    )
+
+
+@pytest.mark.parametrize("moving", [False, True])
+@pytest.mark.parametrize("max_depth", [0, 1, 2])
+@pytest.mark.parametrize(
+    "tile", [(0, ROWS, 0, COLUMNS), (5, 19, 3, 20), (0, 7, 21, COLUMNS)]
+)
+def test_tile_view_is_bit_for_bit_with_the_copied_tile(tile, max_depth, moving):
+    """Reading a tile in place must be the identical computation.
+
+    The mapping pipeline hands the kernel whole corrected frames and the
+    rectangle to map, rather than gathering each tile into its own
+    ``(frames, rows, columns)`` buffer first. That is a pure optimisation:
+    the same values, in the same order, into the same accumulators. Not
+    "agrees to rounding" -- bit for bit, because nothing about the
+    arithmetic or its association has changed.
+    """
+    frames = 4
+    case = _case(frames, moving, max_depth)
+    kernel, rays, start, end, intensity, variance, mask = case
+    row_start, row_stop, column_start, column_stop = tile
+    selection = np.s_[:, row_start:row_stop, column_start:column_stop]
+    tile_rays = np.ascontiguousarray(
+        rays[row_start : row_stop + 1, column_start : column_stop + 1]
+    )
+    copied = kernel.accumulate_group(
+        np.ascontiguousarray(intensity[selection]),
+        np.ascontiguousarray(variance[selection]),
+        np.ascontiguousarray(mask[selection]),
+        tile_rays,
+        start,
+        end,
+    )
+    frame_intensity, frame_variance, frame_mask, _rays_again = _tile_arguments(
+        rays, intensity, variance, mask, tile
+    )
+    viewed = kernel.accumulate_group_tile(
+        frame_intensity,
+        frame_variance,
+        frame_mask,
+        tile_rays,
+        start,
+        end,
+        row_start,
+        row_stop,
+        column_start,
+        column_stop,
+    )
+    assert copied["chunk_id"].size > 1
+    for name in (
+        "chunk_id",
+        "local_voxel_id",
+        "contributors",
+        "weighted_intensity",
+        "weighted_variance",
+        "weight",
+    ):
+        np.testing.assert_array_equal(viewed[name], copied[name], name)
+
+
+def test_tiles_partition_the_detector():
+    """Row bands mapped separately and merged reach the whole frame's voxels.
+
+    The tiling the mapping pipeline uses is a set of row bands, and their
+    merged batches have to reproduce the ungrouped whole-frame call --
+    which is what makes reading a band in place safe at the band's edges.
+    """
+    case = _case(4, True, 1)
+    kernel, rays, start, end, intensity, variance, mask = case
+    whole = kernel.accumulate_group(intensity, variance, mask, rays, start, end)
+    bands = []
+    for row_start in range(0, ROWS, 9):
+        row_stop = min(row_start + 9, ROWS)
+        frame_intensity, frame_variance, frame_mask, band_rays = _tile_arguments(
+            rays, intensity, variance, mask, (row_start, row_stop, 0, COLUMNS)
+        )
+        bands.append(
+            kernel.accumulate_group_tile(
+                frame_intensity,
+                frame_variance,
+                frame_mask,
+                band_rays,
+                start,
+                end,
+                row_start,
+                row_stop,
+                0,
+                COLUMNS,
+            )
+        )
+    merged = _reduce_batches(bands)
+    assert len(bands) > 1
+    assert set(_as_mapping(merged)) == set(_as_mapping(whole))
+    for name in ("contributors",):
+        assert merged[name].sum() == whole[name].sum()
+    for name in ("weighted_intensity", "weight"):
+        np.testing.assert_allclose(
+            merged[name].sum(), whole[name].sum(), rtol=1e-12, err_msg=name
+        )
+
+
+def test_tile_view_rejects_impossible_requests():
+    kernel, rays, start, end, intensity, variance, mask = _case(4, False, 0)
+    frames = [np.ascontiguousarray(frame) for frame in intensity]
+    variances = [np.ascontiguousarray(frame) for frame in variance]
+    masks = [np.ascontiguousarray(frame) for frame in mask]
+    with pytest.raises(ValueError, match="outside the frame"):
+        kernel.accumulate_group_tile(
+            frames, variances, masks, rays, start, end, 0, ROWS + 1, 0, COLUMNS
+        )
+    with pytest.raises(ValueError, match="non-empty"):
+        kernel.accumulate_group_tile(
+            frames, variances, masks, rays, start, end, 4, 4, 0, COLUMNS
+        )
+    with pytest.raises(ValueError, match="one frame each"):
+        kernel.accumulate_group_tile(
+            frames, variances[:2], masks, rays, start, end, 0, ROWS, 0, COLUMNS
+        )
+    with pytest.raises(ValueError, match=r"\(rows \+ 1, columns \+ 1, 3\)"):
+        kernel.accumulate_group_tile(
+            frames, variances, masks, rays, start, end, 0, 6, 0, COLUMNS
+        )
+
+
 def test_group_rejects_mismatched_shapes():
     kernel, rays, start, end, intensity, variance, mask = _case(4, False, 0)
     with pytest.raises(ValueError, match="three-dimensional"):

From 4238288eef315fcd5253b430a994133fdc82ffae Mon Sep 17 00:00:00 2001
From: "T. Fuchs" <tfuchs@cornell.edu>
Date: Wed, 12 Aug 2026 11:13:17 -0400
Subject: [PATCH 32/33] perf(phys): apply the correction factors in one native
 pass

The per-pixel static factor, each scalar factor in turn and the
finiteness check that masks non-finite pixels were eight or nine
full-detector NumPy passes over ~50 MB each, taking and dropping the GIL
between them. They are now one native pass, GIL released throughout.

Bit-for-bit, not merely equivalent: correction is entirely element-wise,
so there is nothing to reassociate. The caller passes the squared factor
it already computed rather than letting the kernel recompute it, since
x ** 2 is pow in Python and a multiplication in C++, and the propagated
branch keeps its order -- scale the variance, use the intensity from
before the factor, then scale the intensity. A test pins both branches.

Measured 0.923x end to end at depth 0, against an independently measured
ceiling of 0.928x for removing the arithmetic entirely. The saving is
memory traffic, not GIL: the pool that runs correction is idle 87% of
the time either way.
---
 .../cpp/reciprocal_reconstruction_cpp.cpp     | 119 ++++++++++++++
 .../xrayutils/test/test_reconstruction.py     | 105 ++++++++++++
 orgui/reconstruction_job.py                   | 149 ++++++++++++------
 3 files changed, 327 insertions(+), 46 deletions(-)

diff --git a/orgui/datautils/xrayutils/cpp/reciprocal_reconstruction_cpp.cpp b/orgui/datautils/xrayutils/cpp/reciprocal_reconstruction_cpp.cpp
index fe8978a..4858d62 100644
--- a/orgui/datautils/xrayutils/cpp/reciprocal_reconstruction_cpp.cpp
+++ b/orgui/datautils/xrayutils/cpp/reciprocal_reconstruction_cpp.cpp
@@ -73,6 +73,111 @@ std::string xxh3_128_buffer(const py::buffer &buffer) {
     return xxh128_hex(hash);
 }
 
+void apply_correction_factors(
+    py::array_t<double, py::array::c_style> intensity,
+    py::array_t<double, py::array::c_style> variance,
+    py::array_t<bool, py::array::c_style> mask,
+    const py::object &static_factor,
+    const py::object &static_factor_squared,
+    const std::vector<double> &factors,
+    const std::vector<double> &factors_squared,
+    const std::vector<double> &factor_variances
+) {
+    // The scaling half of ``_correction_pipeline.correct_frame``, in one
+    // pass: the per-pixel static factor, then each scalar factor in turn,
+    // then the finiteness check folded into the mask.
+    //
+    // NumPy does this in eight or nine full-detector passes over ~50 MB
+    // each, which on a detector this size is around 800 MB of memory
+    // traffic per frame, and it takes and drops the GIL between them.
+    // The arithmetic here is unchanged: the same operations, on the same
+    // values, in the same order, one pixel at a time. Correction is
+    // entirely element-wise -- no reduction, nothing that could
+    // reassociate -- so this is bit-for-bit with the NumPy form, and a
+    // test pins that rather than assuming it.
+    //
+    // ``factors_squared`` and ``factor_variances`` are computed by the
+    // caller rather than here, deliberately: ``factor ** 2`` in Python is
+    // ``pow``, and pow's last bit is not guaranteed to equal a
+    // multiplication's on every platform. The caller already has the
+    // value it used before, so it passes that.
+    const py::buffer_info intensity_info = intensity.request();
+    const py::buffer_info variance_info = variance.request();
+    const py::buffer_info mask_info = mask.request();
+    if (
+        variance_info.size != intensity_info.size
+        || mask_info.size != intensity_info.size
+    ) {
+        throw py::value_error(
+            "intensity, variance and mask must have the same size"
+        );
+    }
+    if (
+        factors_squared.size() != factors.size()
+        || factor_variances.size() != factors.size()
+    ) {
+        throw py::value_error(
+            "factors, factors_squared and factor_variances must match"
+        );
+    }
+    const bool has_static = !static_factor.is_none();
+    py::array_t<double, py::array::c_style> static_array;
+    py::array_t<double, py::array::c_style> static_squared_array;
+    const double *static_data = nullptr;
+    const double *static_squared_data = nullptr;
+    if (has_static) {
+        static_array = static_factor.cast<py::array_t<double, py::array::c_style>>();
+        static_squared_array =
+            static_factor_squared.cast<py::array_t<double, py::array::c_style>>();
+        const py::buffer_info static_info = static_array.request();
+        const py::buffer_info static_squared_info = static_squared_array.request();
+        if (
+            static_info.size != intensity_info.size
+            || static_squared_info.size != intensity_info.size
+        ) {
+            throw py::value_error(
+                "static_factor must have the same size as intensity"
+            );
+        }
+        static_data = static_cast<const double *>(static_info.ptr);
+        static_squared_data =
+            static_cast<const double *>(static_squared_info.ptr);
+    }
+    auto *intensity_data = static_cast<double *>(intensity_info.ptr);
+    auto *variance_data = static_cast<double *>(variance_info.ptr);
+    auto *mask_data = static_cast<bool *>(mask_info.ptr);
+    const std::size_t pixels = static_cast<std::size_t>(intensity_info.size);
+    const std::size_t factor_count = factors.size();
+    {
+        py::gil_scoped_release release;
+        for (std::size_t pixel = 0; pixel < pixels; ++pixel) {
+            double value = intensity_data[pixel];
+            double spread = variance_data[pixel];
+            if (static_data != nullptr) {
+                value *= static_data[pixel];
+                spread *= static_squared_data[pixel];
+            }
+            for (std::size_t index = 0; index < factor_count; ++index) {
+                const double factor_variance = factor_variances[index];
+                if (std::isnan(factor_variance)) {
+                    value *= factors[index];
+                    spread *= factors_squared[index];
+                } else {
+                    // Order matters: the propagated term uses the
+                    // intensity from *before* this factor is applied.
+                    spread *= factors_squared[index];
+                    spread += value * value * factor_variance;
+                    value *= factors[index];
+                }
+            }
+            intensity_data[pixel] = value;
+            variance_data[pixel] = spread;
+            mask_data[pixel] =
+                mask_data[pixel] || !std::isfinite(value) || !std::isfinite(spread);
+        }
+    }
+}
+
 py::dict merge_sorted_batches(
     const ContiguousUInt32Array &left_chunk,
     const ContiguousUInt32Array &left_local,
@@ -2902,6 +3007,20 @@ PYBIND11_MODULE(_reciprocal_reconstruction_cpp, module) {
         py::arg("buffer"),
         "Return the canonical XXH3-128 digest of a C-contiguous buffer."
     );
+    module.def(
+        "apply_correction_factors",
+        &apply_correction_factors,
+        py::arg("intensity"),
+        py::arg("variance"),
+        py::arg("mask"),
+        py::arg("static_factor"),
+        py::arg("static_factor_squared"),
+        py::arg("factors"),
+        py::arg("factors_squared"),
+        py::arg("factor_variances"),
+        "Apply the per-pixel and scalar correction factors in one pass, "
+        "in place, and fold the finiteness check into the mask."
+    );
     module.def(
         "merge_sorted_batches",
         &merge_sorted_batches,
diff --git a/orgui/datautils/xrayutils/test/test_reconstruction.py b/orgui/datautils/xrayutils/test/test_reconstruction.py
index 045e662..34a4c0b 100644
--- a/orgui/datautils/xrayutils/test/test_reconstruction.py
+++ b/orgui/datautils/xrayutils/test/test_reconstruction.py
@@ -666,6 +666,111 @@ def test_shared_correction_pipeline_propagates_factor_uncertainty():
     assert provenance["factor_uncertainty"]["exposure"] == "propagated"
 
 
+class _CorrectionScan:
+    """A scan carrying an exposure time and one monitor, with variances."""
+
+    def __init__(self, image):
+        self.image = image
+        self.exposure_time = np.array([0.37])
+        self.exposure_time_variance = np.array([0.011])
+        self.ic2 = np.array([1.7e5])
+        self.ic2_variance = np.array([913.0])
+
+    def __len__(self):
+        return 1
+
+    def get_raw_img(self, index):
+        return h5_Image(self.image)
+
+
+class _SolidAngleDetector:
+    """Just enough detector for the static per-pixel correction factor."""
+
+    _polFactor = 0.93
+    _polAxis = 0.11
+
+    def __init__(self, shape):
+        self.shape = shape
+
+    def solidAngleArray(self):
+        rows, columns = self.shape
+        grid = np.arange(rows * columns, dtype=np.float64).reshape(rows, columns)
+        return 1.0 + grid / (rows * columns)
+
+    def polarization(self, factor, axis_offset):
+        rows, columns = self.shape
+        grid = np.arange(rows * columns, dtype=np.float64).reshape(rows, columns)
+        return 0.5 + factor * 0.25 + axis_offset + grid / (2 * rows * columns)
+
+
+@pytest.mark.parametrize("propagate", [False, True])
+def test_native_correction_is_bit_for_bit_with_the_numpy_form(
+    monkeypatch, propagate
+):
+    """The fused native pass must not move a single bit.
+
+    Correction applies a per-pixel factor and then each scalar factor in
+    turn, and NumPy does it in eight or nine full-detector passes. Doing
+    the same arithmetic one pixel at a time in the extension is a pure
+    memory-traffic change: every operation is element-wise, so there is
+    no reduction to reassociate and no reason for a result to move.
+
+    Both variance branches are exercised, because they differ in *order*
+    -- a propagated factor scales the variance, then uses the intensity
+    from before the factor was applied, then scales the intensity -- and
+    an implementation that got that order wrong would still look
+    plausible on the deterministic branch.
+    """
+    import orgui.reconstruction_job as job_module
+
+    rows, columns = 9, 11
+    rng = np.random.default_rng(20260811)
+    image = rng.uniform(0.0, 5000.0, size=(rows, columns))
+    # A pixel that must come back masked whatever else happens.
+    image[4, 5] = np.inf
+    scan = _CorrectionScan(image)
+    if not propagate:
+        del scan.exposure_time_variance
+        del scan.ic2_variance
+    config = type(
+        "Config",
+        (),
+        {
+            "corrections": CorrectionState(
+                use_mask=True,
+                use_solid_angle=True,
+                use_polarization=True,
+                normalize_exposure=True,
+                monitor_corrections=("ic2",),
+            ),
+            "detector": _SolidAngleDetector((rows, columns)),
+        },
+    )()
+    static_mask = np.zeros((rows, columns), dtype=bool)
+    static_mask[0, 0] = True
+    assets = {"mask": static_mask}
+
+    def corrected():
+        pipeline = _correction_pipeline(config, scan, assets, {})
+        return pipeline.correct_frame(h5_Image(image), image, 0)
+
+    native_intensity, native_variance, native_mask = corrected()
+    monkeypatch.setattr(job_module, "_correction_extension", lambda: None)
+    numpy_intensity, numpy_variance, numpy_mask = corrected()
+
+    assert job_module._correction_extension() is None
+    np.testing.assert_array_equal(native_intensity, numpy_intensity)
+    np.testing.assert_array_equal(native_variance, numpy_variance)
+    np.testing.assert_array_equal(native_mask, numpy_mask)
+    # ... and the comparison has to be worth something: the factors must
+    # actually have changed the values, and the non-finite pixel must be
+    # masked while its neighbours are not.
+    assert not np.array_equal(native_intensity, image)
+    assert native_mask[4, 5]
+    assert native_mask[0, 0]
+    assert native_mask.sum() == 2
+
+
 def test_shared_correction_repairs_across_detector_tile_boundaries():
     scan = _FakeScan([0.0])
     config = type(
diff --git a/orgui/reconstruction_job.py b/orgui/reconstruction_job.py
index f64e42d..d88f354 100644
--- a/orgui/reconstruction_job.py
+++ b/orgui/reconstruction_job.py
@@ -3,7 +3,9 @@
 from __future__ import annotations
 
 from dataclasses import asdict, dataclass, field, replace
+from functools import lru_cache
 from hashlib import sha256
+import importlib
 import json
 import math
 from pathlib import Path
@@ -1031,6 +1033,22 @@ def _load_assets(job):
     return result
 
 
+@lru_cache(maxsize=1)
+def _correction_extension():
+    """The native extension, or ``None`` where it is not installed.
+
+    Correction has always had a pure-NumPy implementation and keeps one:
+    unlike mapping, it is not a place the pipeline refuses to run without
+    the extension, and the two must stay numerically interchangeable.
+    """
+    try:
+        return importlib.import_module(
+            "orgui.datautils.xrayutils._reciprocal_reconstruction_cpp"
+        )
+    except ImportError:
+        return None
+
+
 def _correction_pipeline(config, scan, assets, provenance):
     correction = config.corrections
     detector = config.detector
@@ -1124,15 +1142,90 @@ def apply_factor(intensity, variance, factor, factor_variance, name):
             variance *= factor**2
             variance += intensity**2 * factor_variance
             intensity *= factor
-            provenance.setdefault("factor_uncertainty", {})[
-                name
-            ] = "propagated"
         else:
             intensity *= factor
             variance *= factor**2
-            provenance.setdefault("factor_uncertainty", {})[
-                name
-            ] = "deterministic-no-uncertainty"
+
+    def record_factor(name, factor_variance):
+        provenance.setdefault("factor_uncertainty", {})[name] = (
+            "propagated"
+            if factor_variance is not None
+            else "deterministic-no-uncertainty"
+        )
+
+    def scalar_factors(frame_index):
+        """This frame's exposure and monitor factors, in application order.
+
+        :returns:
+            ``[(factor, factor_squared, factor_variance_or_None, name)]``.
+            ``factor_squared`` is carried rather than recomputed because
+            it is what the NumPy form multiplied by, and ``x ** 2`` is
+            ``pow`` -- the native pass must scale by the same value, not
+            by one that agrees to the last bit on most platforms.
+        :rtype: list[tuple]
+        """
+        found = []
+        if exposure is not None:
+            value = frame_value(exposure, frame_index)
+            if value <= 0 or not math.isfinite(value):
+                raise ValueError("Exposure time must be finite and positive")
+            factor = 1.0 / value
+            factor_variance = None
+            if exposure_variance is not None:
+                value_variance = frame_value(exposure_variance, frame_index)
+                factor_variance = value_variance / value**4
+            found.append((factor, factor**2, factor_variance, "exposure"))
+        elif correction.normalize_exposure:
+            provenance["exposure_normalization"] = "unavailable"
+        for name, values in monitor_values.items():
+            value = frame_value(values, frame_index)
+            if value == 0 or not math.isfinite(value):
+                raise ValueError(f"Monitor {name} must be finite and nonzero")
+            monitor_variance = monitor_variances[name]
+            factor_variance = None
+            if monitor_variance is not None:
+                value_variance = frame_value(monitor_variance, frame_index)
+                factor_variance = value_variance / value**4
+            factor = 1.0 / value
+            found.append((factor, factor**2, factor_variance, f"monitor:{name}"))
+        return found
+
+    def apply_scaling(intensity, variance, mask, factors):
+        """Scale, propagate and mask off non-finite pixels.
+
+        Fused into one native pass where the extension is available. It
+        is the same arithmetic on the same values in the same order --
+        correction is element-wise, with nothing to reassociate -- so the
+        two paths are bit-for-bit, and a test pins that. What changes is
+        that eight or nine full-detector NumPy passes over ~50 MB each
+        become one, and that the pass releases the GIL for its whole
+        duration instead of taking and dropping it between operations.
+        """
+        native = _correction_extension()
+        if native is not None and all(
+            array.flags.c_contiguous for array in (intensity, variance, mask)
+        ):
+            native.apply_correction_factors(
+                intensity,
+                variance,
+                mask,
+                static_factor,
+                static_factor_squared,
+                [factor for factor, _squared, _variance, _name in factors],
+                [squared for _factor, squared, _variance, _name in factors],
+                [
+                    math.nan if variance_ is None else variance_
+                    for _factor, _squared, variance_, _name in factors
+                ],
+            )
+            return mask
+        if static_factor is not None:
+            intensity *= static_factor
+            variance *= static_factor_squared
+        for factor, _squared, factor_variance, name in factors:
+            apply_factor(intensity, variance, factor, factor_variance, name)
+        mask |= ~np.isfinite(intensity) | ~np.isfinite(variance)
+        return mask
 
     def correct_frame(payload, raw, frame_index):
         source_variance = getattr(payload, "variance", None)
@@ -1159,46 +1252,10 @@ def correct_frame(payload, raw, frame_index):
             mask = static_mask.copy()
         if repair_plan is not None:
             mask, _ = repair_plan.apply_inplace(intensity, variance)
-        if static_factor is not None:
-            intensity *= static_factor
-            variance *= static_factor_squared
-        if exposure is not None:
-            value = frame_value(exposure, frame_index)
-            if value <= 0 or not math.isfinite(value):
-                raise ValueError("Exposure time must be finite and positive")
-            factor = 1.0 / value
-            factor_variance = None
-            if exposure_variance is not None:
-                value_variance = frame_value(
-                    exposure_variance, frame_index
-                )
-                factor_variance = value_variance / value**4
-            apply_factor(
-                intensity,
-                variance,
-                factor,
-                factor_variance,
-                "exposure",
-            )
-        elif correction.normalize_exposure:
-            provenance["exposure_normalization"] = "unavailable"
-        for name, values in monitor_values.items():
-            value = frame_value(values, frame_index)
-            if value == 0 or not math.isfinite(value):
-                raise ValueError(f"Monitor {name} must be finite and nonzero")
-            monitor_variance = monitor_variances[name]
-            factor_variance = None
-            if monitor_variance is not None:
-                value_variance = frame_value(monitor_variance, frame_index)
-                factor_variance = value_variance / value**4
-            apply_factor(
-                intensity,
-                variance,
-                1.0 / value,
-                factor_variance,
-                f"monitor:{name}",
-            )
-        mask |= ~np.isfinite(intensity) | ~np.isfinite(variance)
+        factors = scalar_factors(frame_index)
+        for _factor, _squared, factor_variance, name in factors:
+            record_factor(name, factor_variance)
+        mask = apply_scaling(intensity, variance, mask, factors)
         return intensity, variance, mask
 
     def correct(payload, raw, frame_index, tile):

From c5f5a54928c2bbae7f84e12018cee6c4e05ea5a5 Mon Sep 17 00:00:00 2001
From: "T. Fuchs" <tfuchs@cornell.edu>
Date: Wed, 12 Aug 2026 11:13:29 -0400
Subject: [PATCH 33/33] docs: record what the serial-fraction measurement found

The premise was wrong. A GIL probe measures 23% contention at depth 0
and the prepare pool is 13% occupied, so correction is already
overlapped and nothing is serialised behind it. What holds the pipeline
at 6.4 of 24 cores is memory: the same correction costs 2.3x the cycles
in-pipeline that it costs alone on one thread.

So work removed anywhere pays, including off the critical path, and
moving work between threads pays only where it was queued. A (0.840x)
and B (0.923x) are both traffic reductions and both landed; C was
scheduling only, measured 1.035x on the arm it was written for, and is
reverted. Two of four predicted bands were missed and are recorded as
such.

Also records three things the sweeps caught: a zero-record run on an
unchanged arm at 46.4 ms/frame rather than the documented fifth-of-normal
speed, a run that mapped identical voxels and contributors while
carrying 22% more total intensity, and the pipeline hang caught alive
with every checkpoint written and 58 threads at zero CPU.

Includes the uncommitted zero-record investigation write-up that was
already in the working tree.
---
 CHANGELOG.md                                  |  16 +
 ..._space_mapping_locality_and_subdivision.md |  10 +
 ...eciprocal_space_mapping_serial_fraction.md | 402 ++++++++++++++++--
 .../reciprocal_space_performance_findings.md  | 166 +++++++-
 doc/source/release_notes.rst                  |   2 +
 5 files changed, 531 insertions(+), 65 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 487939b..285b66a 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -102,6 +102,22 @@ Scientific correctness and performance fixes:
   giving 2527 native calls per image instead of six. Band height no longer
   depends on the accuracy setting. Measured throughput at ``balanced``
   accuracy is unchanged; the fix matters at the two highest settings.
+- Reciprocal-space mapping no longer copies each detector tile out of the
+  corrected images before mapping it. Detector tiles partition the image,
+  so gathering every tile into its own buffer copied each corrected image
+  exactly once — about 105 MB per image on a Pilatus 6M — purely to give
+  the native call a shape it did not need. It now takes the whole images
+  plus the rectangle to map and reads that rectangle in place. Measured
+  0.84x the mapping wall time at ``Center only`` accuracy on a 3651-frame
+  rotation scan, with every reconstructed value identical bit for bit.
+- Image correction now applies the per-pixel solid-angle and polarization
+  factor, the exposure and monitor normalizations, and the non-finite
+  check that masks bad pixels in a single native pass, instead of eight or
+  nine full-image NumPy passes over the same arrays. The arithmetic, its
+  order and its uncertainty propagation are unchanged, and the results are
+  identical bit for bit. Measured 0.92x the mapping wall time at ``Center
+  only`` accuracy on the same scan; the saving is memory traffic, so it is
+  largest where the mapping itself is cheap.
 - The reciprocal-space reconstruction documentation now says which footprint
   accuracy setting to choose, and why. Each setting's effect on a
   reconstructed intensity was measured against that voxel's own counting
diff --git a/doc/design/reciprocal_space_mapping_locality_and_subdivision.md b/doc/design/reciprocal_space_mapping_locality_and_subdivision.md
index ee11442..c2466b9 100644
--- a/doc/design/reciprocal_space_mapping_locality_and_subdivision.md
+++ b/doc/design/reciprocal_space_mapping_locality_and_subdivision.md
@@ -442,6 +442,16 @@ because of it.
 | `benchmark_reconstruction_group.py` | one `accumulate_group` call against per-image `accumulate`, by group size | seconds |
 | `benchmark_reconstruction_pipeline.py` | the whole mapping phase — reader pool, correction, kernel, router, checkpoint writes | ~1 min per 234-frame arm at depth 0 |
 | `benchmark_reconstruction_depth_convergence.py` | reconstructed intensities across depths, in units of each voxel's own error bar | ~7 min for depths 0–5 on one tile |
+| `benchmark_reconstruction_stages.py` | the same pipeline split by stage: wall against on-core time per stage, plus a direct GIL-contention probe | as the pipeline benchmark |
+| `benchmark_reconstruction_ab.py` | two arms interleaved, with per-run foreign-load accounting, checkpoint-fingerprint checking and hang retry | ~2 min per pair at depth 0 |
+
+The last two exist because the rules below kept having to be applied by
+hand. `benchmark_reconstruction_ab.py` takes a plan naming two arms as
+sets of files to copy into the checkout, so a native change can be
+measured against its own baseline binary without rebuilding between
+runs; it alternates the arms, flips their order every repeat, and
+reports the paired ratios and whether they separate cleanly. Its
+fingerprint check caught a zero-record run on its first outing.
 
 `benchmark_reconstruction_mapping.py` is dead: it imports `_map_frame_range`
 and the Parquet writers, both retired. Do not resurrect it; the pipeline
diff --git a/doc/design/reciprocal_space_mapping_serial_fraction.md b/doc/design/reciprocal_space_mapping_serial_fraction.md
index 9d0c33c..b0a1ec9 100644
--- a/doc/design/reciprocal_space_mapping_serial_fraction.md
+++ b/doc/design/reciprocal_space_mapping_serial_fraction.md
@@ -1,7 +1,15 @@
 # Reciprocal-space mapping: the per-frame serial fraction
 
-> **Status: planned, nothing implemented.** This is the next feature in
-> the mapping phase. It exists because phase 4 of
+> **Status: measured and complete. A and B are implemented (0.84x and
+> 0.92x, compounding to about 0.77x at depth 0); C was built, measured as
+> a regression, and reverted.** The premise the document was written on —
+> that the pipeline is held back by GIL-serialised per-frame Python — was
+> measured false at step 1, and what replaced it is that this machine is
+> short of memory bandwidth rather than of cores. Both changes that
+> survived are traffic reductions, and the one that was purely a
+> scheduling change is the one that failed.
+>
+> It exists because phase 4 of
 > `reciprocal_space_mapping_locality_and_subdivision.md` measured
 > something it was not looking for: at low adaptive depth the pipeline
 > leaves two thirds of the machine idle, and the kernel is not what is
@@ -50,62 +58,279 @@ work. That is worth knowing before optimising either further: **the whole
 remaining opportunity in mapping is at depth 0**, because depths above 2
 change no intensity a measurement can resolve.
 
-## What has to be measured first
+## What had to be measured first
 
 The stage table in the findings document (load 48.3 / correct 42.3 /
 slice-copy 24.0 / kernel 853.4 / reduce 121.2 ms) predates grouping, the
 prepare pool and the current grid, and was taken in a kernel-dominated
-regime. **It cannot be used to apportion 110 ms/frame today.** Re-measure
-before designing anything:
-
-1. Per-stage wall and CPU time at depth 0 on both grids, inside the real
-   scheduler rather than a harness, distinguishing
-   - HDF5 read and decompression (GIL released inside h5py),
-   - correction arithmetic (several full-detector NumPy passes),
-   - the per-tile `np.stack` / `ascontiguousarray` copy,
-   - time blocked on the GIL itself.
-2. How much of `load` is genuinely I/O-bound. If it is already at the
-   cold-read floor, correction is the only thing left worth attacking.
-
-The GIL-blocked fraction is the number that decides this. If it is small,
-the pipeline is I/O-bound, the ceiling is nearer 1.2x than 2.1x, and this
-whole document should be closed the way phase 4 was.
+regime. **It cannot be used to apportion 110 ms/frame today.** So the
+first step was to re-measure, inside the real scheduler rather than a
+harness, distinguishing HDF5 read and decompression, correction
+arithmetic, the per-tile copy, the native call, the cross-tile merge and
+the checkpoint route — and, for each, wall time against time actually
+spent executing.
+
+**The GIL-blocked fraction was named as the number that decides this: if
+it were small, the pipeline is bound by something this document cannot
+move and the whole thing should be closed the way phase 4 was.** It was
+measured. It is small. What the same measurement turned up instead is
+that the largest per-frame Python cost is not where this document assumed
+it was, so the ordering below survives while the reasoning under it does
+not. Both are recorded next.
+
+## Step 1 — where the time actually goes
+
+Instrument: `benchmarks/benchmark_reconstruction_stages.py`, which runs
+the real `_map_pending_ranges` and wraps each stage's existing call site
+from outside — no instrumentation compiled into the pipeline, no
+arithmetic touched. On-core time is `QueryThreadCycleTime`, not
+`time.thread_time`: the latter is `GetThreadTimes` on Windows and
+quantises to the ~15.6 ms scheduler tick, which is larger than most of
+these stages.
+
+Reference job as it stands on disk (1000^3 grid), `--depth 0`, 234
+frames, the scheduler's own choice of four frames per call and three
+concurrent calls of eight threads. **Milliseconds per frame, summed over
+every thread** — so a column may exceed the 111.5 ms/frame the run took.
+
+| stage | wall | on-core | blocked |
+|---|---|---|---|
+| load (`get_raw_img`) | 100.2 | 63.4 | 36.8 |
+| correct (`correct_frame`) | 132.8 | 89.3 | 43.5 |
+| **`_map_frame_group`** | **257.9** | **114.3** | **143.5** |
+| — per-tile assembly | 88.9 | 43.0 | 45.9 |
+| — of which `np.stack` | 70.3 | 27.7 | 42.6 |
+| — the native call | 103.0 | 22.4 | 80.6 |
+| — cross-tile merge | 21.6 | 15.2 | 6.4 |
+| — checkpoint route | 44.4 | 33.8 | 10.6 |
+
+Whole run: **111.5 ms/frame, 6.40 of 24 cores, 713 ms of CPU per frame**,
+of which 267 ms is on Python threads and the remaining ~446 ms is the
+kernel's own native worker pool. (The native call's own on-core column is
+the *calling* thread's only; its 80.6 ms of "blocked" is a thread waiting
+on its native workers, not on the GIL.)
+
+The same measurement on the documented 2000^3 grid, where the kernel
+dominates: 280.0 ms/frame, 6.19 cores, 1734 ms of CPU per frame, and the
+route becomes the largest single term (262.1 wall / 232.5 on-core) as
+records rise 5.2x. Per-tile assembly is essentially unchanged in absolute
+terms (95.4 / 53.8 / 41.7), as it should be — it is a function of the
+detector, not of the grid.
+
+### The GIL is not the constraint
+
+A probe thread runs bursts of pure Python — no I/O, nothing that could
+release the GIL voluntarily — several times the interpreter's switch
+interval, and compares each burst's wall time against the cycles it
+executed. That is the blocked fraction a Python thread wanting to run
+actually experiences:
+
+| | |
+|---|---|
+| idle machine | **1%** |
+| during the run, coarse grid | **23%** |
+| during the run, fine grid | **22%** |
+
+So a Python thread gets the GIL roughly three times out of four, and the
+total GIL-held work is on the order of 25-30 ms per frame against 713 ms
+of CPU per frame. **The pipeline is not GIL-bound**, and the premise this
+document was written on — "loading and correcting hold the GIL, so adding
+workers does not help" — is wrong as stated.
+
+### Correction is already overlapped
+
+The prepare pool the grouped scheduler introduced does what phase 3
+claimed for it, and more completely than anyone checked. Load plus
+correct is 233 ms/frame of thread-wall spread over the 14-16 prepare
+workers the pool grows to: **about 13% occupancy**. The compute worker,
+by contrast, is busy 257.9 / 3 workers / 111.5 ms = **77% of the time**.
+The critical path is the compute worker, and correction is not on it.
+
+### What *is* expensive is doing the same arithmetic under load
+
+| stage | uncontended, one thread | in the pipeline | inflation |
+|---|---|---|---|
+| load | 32.1 ms/frame on-core | 63.4 | 2.0x |
+| correct | 38.5 | 89.3 | 2.3x |
+
+Same code, same frames, twice the cycles. Cycles are counted while the
+thread is on a core, so this is not GIL waiting — it is memory: 24
+threads sharing one L3 and one controller. That matters twice over. It
+says the machine's spare cores are worth less than their count suggests,
+and it says **removing memory traffic is worth more than removing
+instructions**.
+
+### What this retires, and what survives
+
+- **Closed: the GIL argument.** 23% is not nothing, but it is not what
+  holds the pipeline at 6.4 cores, and no amount of releasing it converts
+  into the 2.1x this document opened with.
+- **Closed: "the prepare pool cannot run in parallel".** It can, and does;
+  it is idle 87% of the time.
+- **Survives, for a different reason: A.** The per-tile assembly is 88.9
+  ms/frame of wall on the one thread that is 77% busy — 34% of that
+  thread's occupied time, and the largest thing on the critical path
+  after the native call itself. It is not a GIL problem; it is 105 MB per
+  frame of pointless copying on the busiest thread, and 210 MB/frame of
+  memory traffic on a machine that is measurably short of bandwidth.
+- **In doubt: B.** Its stated mechanism (release the GIL so the prepare
+  pool can run) is measured false. Its unstated one (cut memory traffic)
+  survives, but the pool it would speed up is 13% occupied. See B below,
+  which is reconsidered against this rather than against the plan.
+- **The 2.1x headroom is not reachable by this route.** Reaching the
+  51.5 ms/frame cold-read floor needs the CPU work to become nearly free,
+  and 713 ms of CPU per frame at 2x inflation is not nearly free.
 
 ## Candidate mechanisms
 
 In increasing order of invasiveness. Each carries its own predicted band,
 stated before implementing so it can be wrong on the record.
 
-### A. Drop the per-tile stack and copy
+### A. Drop the per-tile stack and copy — **done, 0.840x**
 
-`_map_frame_group` builds `np.ascontiguousarray(np.stack(...))` for
+`_map_frame_group` built `np.ascontiguousarray(np.stack(...))` for
 intensity, variance and mask, per tile, per group. Tiles partition the
-detector, so this copies every corrected frame exactly once — 105 MB per
-frame — in Python, under the GIL, purely to hand the kernel a contiguous
-`(frames, rows, columns)` buffer.
-
-`accumulate_group` could instead take the whole corrected frame plus the
-tile rectangle and read the sub-rectangle itself; it already walks bricks
-within a tile, so a row stride costs it nothing.
-
-**Predicted: 0.80-0.90x** at depth 0, from removing ~24 ms of the ~110.
-Findings open item 1 named this direction; it is the cheapest of the
-three and does not touch any arithmetic, so it should be bit-for-bit.
-
-### B. Fuse and release the correction
+detector, so this copied every corrected frame exactly once — 105 MB per
+frame — in Python, on the compute worker, purely to hand the kernel a
+contiguous `(frames, rows, columns)` buffer.
+
+`accumulate_group_tile` now takes the whole corrected frames plus the
+tile rectangle and reads the sub-rectangle in place. The brick loop
+already walked rows within a tile, so the change is one stride: a
+`GroupPixels` view carries one pointer per frame at the tile's first
+pixel and the distance between rows in the caller's own array, and
+`accumulate_group` builds the same view over its contiguous buffer. One
+loop, two ways in, and on the reference job's full-width bands the row
+stride is *identical* either way — the tile is the frame's whole width,
+so only the base offset differs.
+
+**Predicted 0.80-0.90x** by this document, and **0.72-0.90x** from the
+step-1 stage table, which put the assembly at 34% of the busy time of the
+one thread that is 77% busy. Measured, six interleaved pairs, 234 frames,
+depth 0, coarse grid:
+
+| pair | baseline | tile view | ratio |
+|---|---|---|---|
+| 1 | 106.5 | 92.2 | 0.866 |
+| 2 | 105.8 | 91.4 | 0.864 |
+| 3 | 108.4 | 91.0 | 0.840 |
+| 4 | 103.1 | 85.7 | 0.831 |
+| 5 | 103.5 | 87.0 | 0.841 |
+| 6 | 102.2 | 85.0 | 0.832 |
+
+**0.840x, and every tile-view run beat every baseline run** (85.0-92.2
+against 102.2-108.4). Foreign load 2.16-2.71 cores throughout, inside
+the machine's usual band.
+
+**Bit-for-bit, as required.** Identical routed record count, identical
+checkpoint rows, contributors and voxel fingerprint, and identical
+totals to the last bit (`weight`, `weighted_intensity` and
+`weighted_variance` compared by `repr`) over a 78-frame window; the
+kernel-level test pins the same thing directly, against the copied form,
+at depths 0-2, stationary and moving, for a tile in the middle of the
+frame as well as one at its edge.
+
+Re-measured with the stage instrument afterwards, the pipeline moved as
+the model said it would: 111.5 -> 91.4 ms/frame, cores in use 6.40 ->
+8.40, compute-worker occupancy 77% -> 71%, GIL contention 23% -> 19%.
+Python-thread CPU fell 267 -> 246 ms/frame. Note that *process* CPU per
+frame did not fall — it rose slightly, to 768 ms — which is the memory
+story again: at higher concurrency every remaining stage costs more
+cycles for the same work.
+
+One thing this did not need: a memory-model change. `_frame_parallelism`
+never charged for the per-tile buffer (it charges the whole detector's
+corrected arrays per in-flight frame, which is 6x larger), so removing it
+makes the existing estimate slightly more conservative rather than
+wrong.
+
+### B. Fuse the correction
 
 Correction is several full-detector NumPy passes — background subtract,
 static factor, exposure and monitor scaling, the finiteness check, mask
-combination. Each is memory-bound over ~50 MB, and NumPy holds the GIL
-across the small ones. Fusing them into a single pass in the existing C++
-extension would both cut memory traffic and release the GIL, letting the
-prepare pool actually run in parallel.
-
-**Predicted: 0.65-0.80x** on top of A. **This is a `phys` change** — it
-moves arithmetic that produces the corrected intensities and variances
-into another language and another association order. It needs the same
-treatment phase 3 got: voxels, contributor counts and totals pinned, and
-the last-bit movement stated. Do not fold it in with A.
+combination. Each is memory-bound over ~50 MB, and NumPy takes and drops
+the GIL between them.
+
+**The stated mechanism was wrong.** This document predicted 0.65-0.80x
+from "release the GIL, letting the prepare pool actually run in
+parallel". Step 1 measured the GIL at 23% and the prepare pool at 13%
+occupancy: the pool already runs in parallel, and there is no serialised
+correction to release.
+
+**The unstated mechanism survives, and was measured before building
+anything.** A ceiling probe — `--correction minimal` in the stage
+benchmark, which keeps the mask and the finiteness check and drops the
+scaling arithmetic entirely, so it routes the identical records and only
+carries wrong intensities — bounds from above what any fusion could buy:
+
+| pair | full | no scaling | ratio |
+|---|---|---|---|
+| 1 | 98.2 | 83.7 | 0.852 |
+| 2 | 91.0 | 83.6 | 0.918 |
+| 3 | 90.9 | 85.1 | 0.937 |
+| 4 | 85.9 | 83.9 | 0.976 |
+
+**0.928x, clean separation.** The prediction stated before running that
+probe was 0.95-1.00x — *that is also wrong*, in the other direction. Not
+doing ~700 MB/frame of memory traffic is worth about 7%, on a pipeline
+whose prepare pool is idle 87% of the time. Which is the step-1 result
+again from a different angle: this machine is short of memory bandwidth,
+not of cores, so **work removed anywhere helps, including off the
+critical path.**
+
+That is what justified building B after its own premise had failed, and
+it is why what was built is narrower than what was proposed: one native
+pass for the arithmetic that costs, and not a native rewrite of a
+correction step that is mostly allocation.
+
+**What was built.** `apply_correction_factors` applies the per-pixel
+static factor, then each scalar factor in turn, then folds the finiteness
+check into the mask — one pass, one pixel at a time, with the GIL
+released for its whole duration. What stayed in NumPy is the part that is
+allocation rather than arithmetic (the raw copy, the variance
+initialisation, the background subtraction) and the repair plan, which is
+already native and has to run between the two.
+
+**It is bit-for-bit, which the plan assumed it could not be.** Correction
+is entirely element-wise: there is no reduction, so there is nothing to
+reassociate, and doing the same operations on the same values in the same
+order gives the same bits. Two details make that true rather than nearly
+true. `factor ** 2` is `pow` in Python and a multiplication in C++, and
+those are not guaranteed to agree in the last bit, so the caller passes
+the squared factor it already computed rather than letting the kernel
+recompute it. And the propagated-uncertainty branch scales the variance,
+*then* uses the intensity from before the factor was applied, *then*
+scales the intensity — an order an independent implementation would very
+plausibly get wrong, so the test exercises both branches. The end-to-end
+run agrees on voxels, contributors and all three totals by `repr`.
+
+**Measured, six interleaved pairs twice over**, because the first sweep
+ran into rising background load and the second into both that and a
+zero-record run:
+
+| sweep | ratios |
+|---|---|
+| 1 | 0.923, 0.924, 0.975, 1.152, 0.940, 0.780 |
+| 2 | 0.885, 0.939, 0.915, 0.912 (+2 discarded) |
+
+**Median 0.923 over the ten valid pairs, nine of them below 1.0** — but
+**no clean separation**, and by this project's own standard that is worth
+less than the number suggests. What makes it believable anyway is that an
+independent measurement predicted it before it existed: the ceiling probe
+separated cleanly at 0.928, and a fused pass that recovers essentially
+all of a 0.928 ceiling is what 0.923 says. Two measurements of different
+things agreeing is the evidence here, not either one alone.
+
+The prediction stated before implementing was 0.94-0.97x, reasoning that
+the fused pass still moves intensity and variance once and so recovers
+only part of the ceiling. It recovered all of it, and slightly more —
+most likely because the probe arm still made NumPy calls that take and
+drop the GIL, where the fused pass releases it once.
+
+The second sweep also cut correction's own cost, visible in the stage
+instrument: 89.3 -> 50.8 ms/frame of on-core CPU, a 43% cut in the stage
+itself, which is the mechanism doing what it was built to do even where
+the wall time is noisy.
 
 ### C. Extend the prepare pool to the per-frame path
 
@@ -116,7 +341,43 @@ Python behind its own native call. With B done, this is mostly
 bookkeeping.
 
 **Predicted: little on the reference job** (which groups), and up to the
-same 0.87x grouping bought, on jobs that do not.
+same 0.87x grouping bought, on jobs that do not. Sharpened from the stage
+table before implementing: **0.85-0.95x on the per-frame arm**, since
+that path spends 68.8 ms/frame of compute-worker wall on correction, 17%
+of the occupied time of a thread that is 81% busy.
+
+**Built, measured, and reverted.** `--group 1`, six interleaved pairs,
+234 frames, depth 0:
+
+| pair | correction in worker | correction in pool | ratio |
+|---|---|---|---|
+| 1 | 76.6 | 72.4 | 0.946 |
+| 2 | 69.5 | 73.7 | 1.060 |
+| 3 | 69.8 | 72.3 | 1.036 |
+| 4 | 69.5 | 71.9 | 1.035 |
+| 5 | 69.3 | 72.5 | 1.046 |
+| 6 | 69.7 | 71.1 | 1.020 |
+
+**1.035x — a regression**, in five pairs of six; the one pair that
+favoured it is the first, against that arm's slowest run. Outside the
+predicted band and on the wrong side of 1.
+
+**Why, and it is this document's GIL mistake in miniature.** The
+prediction assumed correction was *serialised* on the per-frame path
+because it sits on the compute worker. It was not: that path runs six
+compute workers, so correction was already spread across six threads.
+Hoisting moved it into a reader pool that had shrunk to **one** worker —
+the pool sizes itself on a blocked-fraction signal, compute was never
+starved, so it had every reason to shrink — and it made each queued frame
+hold its corrected arrays (105 MB) instead of its raw image (50 MB),
+buying fewer frames in flight for the same budget.
+
+The grouped scheduler's prepare pool pays because a group is *one* native
+call, so its correction really was queued behind it. The per-frame path
+had no such queue to remove. **The code is not kept**; what is kept is
+the measurement, and the rule it illustrates: on this pipeline, "move the
+Python work off the compute thread" is only worth anything where the
+compute thread is a bottleneck of one.
 
 ## Ordering
 
@@ -128,6 +389,41 @@ Do not do all three and measure once. Phase 4 is the standing argument:
 a saving of known size need not convert, and finding that out one change
 at a time is what makes it interpretable.
 
+*This held, and it earned its keep.* Measuring one change at a time is
+what made B's failed premise visible without taking A's win down with it,
+and it is the only reason C's regression is attributable to C rather than
+to a bundle.
+
+## Where this leaves the pipeline
+
+| | before | after A | after A and B |
+|---|---|---|---|
+| ms/frame, depth 0, coarse grid | 111.5 | 91.4 | ~84 |
+| cores in use of 24 | 6.40 | 8.40 | 8.2 |
+| GIL contention | 23% | 19% | 17% |
+| correction, on-core ms/frame | 89.3 | 93.7 | 50.8 |
+
+Absolute values across those columns come from different runs and drift
+with the page cache — the ratios are the measured quantities (0.840 and
+0.923, each over six interleaved pairs) and the table is the shape, not
+the evidence.
+
+**What is left, and what is not.** The document opened claiming 2.1x of
+headroom bounded by a 51.5 ms/frame cold-read floor. A and B together
+take about 23% of the wall, and the remaining gap to that floor is not
+reachable by this route: at 8 cores of 24 with the GIL at 17%, what
+limits the pipeline is that 700-odd ms of CPU per frame costs twice its
+uncontended cycles. The next real lever would be *less work*, not more
+overlap — the checkpoint route is now the largest Python-side term on the
+compute worker (36 ms/frame of on-core time, and 232 on the fine grid),
+and nothing in this document touched it.
+
+**The one general result worth carrying forward:** on this machine,
+removing memory traffic pays wherever it happens, including on threads
+that are idle 87% of the time, while moving work between threads pays
+only where it was genuinely serialised. Every measurement in this
+document is an instance of that, including the two that came out wrong.
+
 ## Risks
 
 **The ceiling is low and the noise is high.** 2.1x is the best case, and
@@ -136,12 +432,30 @@ cache state — one arm spread 176.5 to 340.6 ms/frame. Six interleaved
 pairs minimum, and the foreign-load guard on every run. It is entirely
 possible for a real 15% win to be invisible here.
 
+*Both of these happened.* The noise was the binding difficulty
+throughout: A separated cleanly, B did not and rests on agreeing with an
+independently measured ceiling, and one sweep had to be repeated for
+background load and another partly discarded for a zero-record run.
+
 **B changes reconstructed values.** Not in which voxels are reached, but
 in the last bits of the corrected intensities that feed them. That is
 acceptable on the same grounds the phase 3 change was — the feature has
 never been released — but it must be said, and it must not be discovered
 by a user.
 
+*It does not.* Scoping B to the element-wise arithmetic made it
+bit-for-bit, verified by test on both variance branches and end to end on
+a real 78-frame window. No reconstructed value moves, so nothing needs
+saying to a user beyond that mapping got faster. The `phys` scope stays
+on the commit anyway: the arithmetic that produces intensities and
+variances moved into another language, and that is what the scope is for.
+
 **I/O may already be the wall.** If step 1 of the measurement shows load
 sitting at the cold-read floor with correction overlapped, there is
 nothing here worth building, and saying so is the correct outcome.
+
+*It was not the wall, and neither was the GIL.* Step 1 found correction
+fully overlapped, as this risk anticipated, but load nowhere near the
+cold-read floor and the machine short of memory bandwidth instead. The
+document was right that the answer might not be what it assumed, and
+wrong about which way.
diff --git a/doc/design/reciprocal_space_performance_findings.md b/doc/design/reciprocal_space_performance_findings.md
index d9726a0..2d9482b 100644
--- a/doc/design/reciprocal_space_performance_findings.md
+++ b/doc/design/reciprocal_space_performance_findings.md
@@ -214,6 +214,19 @@ wrong. This item is now a plan of its own —
 `reciprocal_space_mapping_serial_fraction.md` — and further work on it
 belongs there rather than here.
 
+**Measured, and the diagnosis in this item is wrong.** The serial-fraction
+document's step 1 put a probe on the GIL itself: a Python thread wanting
+to run waits **23%** of its wall time during a depth-0 run (1% idle), on
+both grids. The pipeline is not GIL-bound, the prepare pool that was
+supposed to be blocked is idle 87% of the time, and "loading and
+correcting hold the GIL, so adding workers does not help" is not what
+holds this pipeline at 6.4 of 24 cores. What does is memory: the same
+correction costs 2.3x the cycles inside the pipeline that it costs alone
+on one thread. The useful consequence is that **work removed anywhere
+helps, including off the critical path** — which is not what a serial
+fraction predicts. Two changes came out of it (0.84x and 0.92x); see that
+document.
+
 ### 2. `bounded_block_size` still clamps memory with a depth-blind constant
 
 The Python side now caps the work block against the arena the kernel will
@@ -354,34 +367,145 @@ ms/frame against ~250, i.e. a 5x "speed-up". Always compare a run's
 checkpoint fingerprint against its partner's and discard mismatches; a
 timing-only harness will silently take these as wins.
 
+*It happened again during the serial-fraction work, once in about 50
+runs, and `benchmark_reconstruction_ab.py`'s fingerprint check caught it
+automatically — which is what that check is for.* Two details worth
+adding to the description above. It was on the **unchanged** arm of the
+comparison, so whatever it is has nothing to do with the changes being
+measured. And it was **46.4 ms/frame against 71-100 for its neighbours**,
+under half rather than a fifth: the "5x faster" signature is not
+reliable, because how fast a real run is depends on the page cache. The
+fingerprint is the reliable signal; the clock is not.
+
 `_map_pending_ranges` now warns, on both schedulers and through the
 progress channel as well as `warnings`, when a whole run routes nothing.
-That makes the next occurrence visible; it does not explain it. What the
-investigation ruled out, each by direct measurement rather than
-inspection:
-
-- **Not the static correction factor.** `1 / solidAngleArray` and the
-  polarization array are bit-identical across 12 fresh job loads, with no
-  zeros and no non-finite entries. A bad one would poison a whole run,
-  which fits the symptom, but it does not vary.
-- **Not the angle bounds or the grid.** Bit-identical across 20 fresh
-  loads. This was the best remaining candidate — garbage bounds would put
-  every sample outside the grid, which is silent, fast and record-free.
+That makes the next occurrence visible; it does not explain it.
+
+**The cause is still unknown, and the search was stopped deliberately
+rather than exhausted.** Everything below was measured, not reasoned
+about. What it adds up to is a reframing worth having before anyone
+starts again: *every static input the run is built from is
+deterministic*, so this is not a bad value read at setup — it is a race
+or a load-dependent condition during execution.
+
+Bit-identical across repeated fresh job loads, so none of them can be
+the poison:
+
+| input | repeats |
+|---|---|
+| solid angle and polarization (`static_factor`) | 12 |
+| exposure angle bounds, grid minimum/step/shape | 20 |
+| UB, U, wavevector K (`det(U)` = 1.000000000) | 8 |
+| detector corner rays (all unit norm) | 6 |
+| static mask (0.0857 masked) | 12 |
+
+Also eliminated:
+
 - **Not a swallowed exception.** Both schedulers re-raise the first
   recorded exception, and the failing runs exited 0 with no traceback, so
   correction did not raise.
 - **Not the native memory precheck.** It throws; it has no path that
   returns empty.
-- **Not reproducible on demand.** 30 consecutive runs of a 39-frame
-  window returned the identical record count every time.
-
-What remains is that the raw frames themselves, or the mask derived from
-them, occasionally come back degenerate for an entire run — consistent
-with the `RuntimeWarning` seen at `intensity *= static_factor` on one
-occurrence, which implies a non-finite *raw* image rather than a bad
-factor. Both observed failures came late in long, memory-heavy sequences.
-Unproven. A separate one-off hang of the same pipeline under `pytest`,
-not reproduced, may or may not be related.
+- **Not the pixel repair plan**, despite being the best structural
+  candidate: a shared native object called concurrently by every prepare
+  thread, whose return value *replaces* the mask outright. Its
+  `apply_inplace` is `const`, holds no `mutable` members, and keeps all
+  scratch local to the call. Thread-safe.
+- **Not reproducible on demand.** 108 attempts across four
+  configurations — subprocess and in-process, coarse grid and fine,
+  grouped scheduler and per-frame, depths 0 and 1 — returned the
+  identical record count and the identical 0.0857 masked fraction every
+  time. Against an observed rate of ~2 in 40, that is not merely bad
+  luck: whatever triggers it is not uniformly random per run.
+
+**A second, milder failure of the same family was caught in 2026-08, and
+it is the sharpest evidence yet.** One run in twelve of an interleaved
+sweep reported a total weighted intensity of **2,240,706.86 against
+1,839,128.67** — 22% high — and a total weighted variance of **236.23
+against 136.96**, 72% high, while the other eleven agreed with each other
+to one or two units in the last place. In the same run:
+
+| quantity | anomalous run | the other eleven |
+|---|---|---|
+| voxel fingerprint | identical | identical |
+| contributors | 1,329,409,714 | 1,329,409,714 |
+| total weight | 1,329,409,714.0 | 1,329,409,714.0 |
+| rows written | 47,226,553 | 47,226,553 (three others) |
+| **weighted intensity** | **2,240,706.86** | 1,839,128.665149 |
+| **weighted variance** | **236.23** | 136.960292 |
+
+Read what that excludes. Identical voxels and identical contributor
+counts mean the geometry, the angle bounds, the grid *and the mask* were
+all right: the same detector samples reached the same voxels. Only the
+*values* they carried were wrong. So this is not a geometry failure at
+all — it is the pixel data or the factors applied to it, which is the
+same half of the pipeline the zero-record failure points at, reached from
+the opposite direction.
+
+And the two totals moved by **different** factors, 1.22 against 1.72.
+A wrong image would move both alike, since the variance is the clipped
+image itself. A wrong *scalar factor* moves the intensity by `k` and the
+variance by `k**2`, which is the shape actually observed. The per-frame
+exposure and monitor factors are the only per-frame scalars in the
+pipeline, and they are read from arrays captured once at pipeline
+construction — so on the face of it they cannot vary, which is exactly
+what makes this worth writing down rather than explaining away.
+
+Two honesty notes. The run was on the **unchanged** arm of the
+comparison, so it is not caused by whatever was being measured. But it
+was on a build carrying the fused native correction pass
+(`apply_correction_factors`), which is verified bit-for-bit against the
+NumPy form by test and over a 78-frame real window — verified, not
+exonerated. Anyone hunting this should not assume that pass is innocent
+merely because its unit test passes.
+
+**The instrument to use next time is one bit.** Only two things can
+produce an empty result: every pixel masked, or every coordinate outside
+the grid. So record the masked fraction when it happens. Normal is
+0.0857; ~1.0 means the raw data or the correction went non-finite, and
+0.0857-with-no-records means the pixels were fine and the geometry
+applied during mapping is at fault. That single number halves the search
+space, and the guard above is what will tell you a run is worth looking
+at.
+
+*Add a second: the per-frame factor actually applied.* The 22% run above
+says the failure mode is not always all-or-nothing, and that a run can
+look completely healthy on every count anyone currently checks —
+records, voxels, contributors, masked fraction — while carrying wrong
+values. Logging the exposure and monitor factor per frame, or simply the
+per-frame corrected sum, would separate "the image was wrong" from "the
+factor was wrong" the first time it happens again. Total weighted
+intensity per checkpoint is worth comparing between runs for the same
+reason; `benchmark_reconstruction_ab.py` now does exactly that, which is
+how this one was found.
+
+Both observed failures came late in long, memory-heavy sequences with
+desktop applications resident, which the reproduction loops could not
+recreate. A separate one-off hang of the same pipeline under `pytest`,
+also not reproduced, may or may not be related.
+
+**The hang happened twice more in 2026-08, and this time it was caught
+alive.** Both were the pipeline benchmark on unchanged code. The state
+of the hung process is specific enough to be worth writing down, because
+it rules out most of where one would look first:
+
+| | |
+|---|---|
+| checkpoints | **all three written**, full size, on disk |
+| threads | 58, still alive |
+| resident memory | 883 MB |
+| CPU consumed before the hang | 188 s |
+| CPU while hung | **0.00 cores**, indefinitely |
+
+So mapping had finished and every record had been flushed; the process
+then stopped with its pools still up and nothing running. That points at
+the coordinator's exit condition or a pool join, not at mapping,
+correction or the kernel — and it is a *different* failure from the
+zero-record one, which completes and exits 0. `benchmark_reconstruction_ab.py`
+now kills and retries a run that exceeds a per-run timeout, so a hang
+costs a run rather than a sweep; it still deserves a stack dump the next
+time one is caught alive (`py-spy dump --pid`), which is the one thing
+not done here.
 
 ## Calibration constants and what they rest on
 
diff --git a/doc/source/release_notes.rst b/doc/source/release_notes.rst
index 7429e7a..4cea1f6 100644
--- a/doc/source/release_notes.rst
+++ b/doc/source/release_notes.rst
@@ -30,6 +30,8 @@ Scientific correctness and performance fixes:
 - Mapping several images per call uses a scheduler of its own, which runs a few concurrent native calls with the thread budget split between them and hoists whole-frame correction into the prefetch pool, so one call's GIL-held Python work overlaps the next call's native work instead of queueing behind it. Measured 0.87x the mapping wall time of the per-image pipeline at four images per call on a 3651-frame rotation scan.
 - The number of images per call is now measured per job rather than configured. Grouping only helps while consecutive images land within about one voxel of each other, so the job's own geometry is probed at three points in the scan to decide whether it does, and the group size is then the largest the memory budget can afford without giving up concurrency. Scans whose angles do not advance monotonically, such as interlaced ones, map one image per call as before. On a 0.1 deg/frame rotation scan this chooses four images per call and maps at 0.88x the previous wall time.
 - Corrected the detector band height used for reciprocal-space mapping, which was still derived from the worst-case adaptive leaf count rather than from the record ceiling the memory prechecks use. At the ``very_high`` and ``maximum`` accuracy settings this over-estimated a pixel's cost by four orders of magnitude and split the detector into extremely thin bands — one row per band on a Pilatus 6M at ``maximum``, giving 2527 native calls per image instead of six. Band height no longer depends on the accuracy setting. Measured throughput at ``balanced`` accuracy is unchanged; the fix matters at the two highest settings.
+- Reciprocal-space mapping no longer copies each detector tile out of the corrected images before mapping it. Detector tiles partition the image, so gathering every tile into its own buffer copied each corrected image exactly once — about 105 MB per image on a Pilatus 6M — purely to give the native call a shape it did not need. It now takes the whole images plus the rectangle to map and reads that rectangle in place. Measured 0.84x the mapping wall time at ``Center only`` accuracy on a 3651-frame rotation scan, with every reconstructed value identical bit for bit.
+- Image correction now applies the per-pixel solid-angle and polarization factor, the exposure and monitor normalizations, and the non-finite check that masks bad pixels in a single native pass, instead of eight or nine full-image NumPy passes over the same arrays. The arithmetic, its order and its uncertainty propagation are unchanged, and the results are identical bit for bit. Measured 0.92x the mapping wall time at ``Center only`` accuracy on the same scan; the saving is memory traffic, so it is largest where the mapping itself is cheap.
 - The reciprocal-space reconstruction documentation now says which footprint accuracy setting to choose, and why. Each setting's effect on a reconstructed intensity was measured against that voxel's own counting error: ``Balanced`` is sufficient for a full rotation scan, while ``High``, ``Very high`` and ``Maximum`` shift intensities by 0.02 sigma or less and are not distinguishable from ``Balanced`` by the data, at roughly 4x, 15x and 100x the runtime. ``Center only`` is genuinely lossy and also misses about 4% of the voxels the other settings reach. No behaviour or default has changed.
 
 A ***critical bug*** was fixed that affects bulk CTR calculations: