diff --git a/README.md b/README.md index 201d874..61ea9c0 100644 --- a/README.md +++ b/README.md @@ -68,7 +68,7 @@ Exit code 2 means it couldn't run at all (bad file, no model access, no AWS cred - Dates, times, UUIDs or user-specific text before the cache checkpoint. - A cached prefix shorter than the model's minimum (512 to 4,096 tokens depending on the model). - Tools listed in a different order, or a tool definition that changed. Tools come first, so this throws away the whole cache. -- More than 21 new content blocks between the old checkpoint and the new one, which is common after many parallel tool calls. Bedrock only looks back that far: measured live, 21 added blocks still hit and 22 always missed (`scripts/live_lookback_boundary.py`). +- More than 21 lookback positions between the old checkpoint and the new one, common in long chains of single tool calls or with many separate text blocks. Each text block is one position, a group of tool calls or tool results side by side is one, and thinking blocks don't count. Measured live on October 30 2026: 21 positions still hit and 22 always missed (`scripts/live_lookback_positions.py`). On October 7 2026 Bedrock still counted every tool call and result, so many parallel tool calls lost the cache; by October 28 they no longer did. - Model IDs or application inference profile ARNs that your library may not recognize, so it quietly stops sending cache markers. Legacy models (Sonnet 4, Opus 4 and 4.1, 3.5 Haiku) are checked against Anthropic's documented minimums, since AWS's caching table doesn't list them. - A switch between the Converse and InvokeModel APIs, which builds a different prompt. - More than 4 checkpoints, an unknown cache lifetime (TTL), or a 1-hour checkpoint after a 5-minute one. @@ -166,7 +166,7 @@ Only model calls (generations) are counted. Agent and chain spans can carry the ## How it's tested -The checks follow AWS's documented prompt-caching rules. The core behaviour was verified against live Amazon Bedrock: Converse and InvokeModel, streaming, both cache lifetimes (5 minutes and 1 hour), model minimums, prompt changes and the exact lookback limit (21 added blocks hit, 22 miss). Every check added in 0.3 was measured live first, and `scripts/live_v03_checks.py` re-runs those cases against Bedrock and compares CacheCanary's verdict with what Bedrock did (15 of 15 agreed in October 2026). Every check also has unit tests. The log reader is tested against real Bedrock invocation logs (redacted copies are in [tests/fixtures](https://github.com/Haarris/cachecanary/tree/main/tests/fixtures)). The Langfuse reader is tested against records that Strands, LiteLLM and LangChain wrote into a real Langfuse 4.64 server, and against a production Langfuse Cloud project, where its numbers matched a separate recount for every model. +The checks follow AWS's documented prompt-caching rules. The core behaviour was verified against live Amazon Bedrock: Converse and InvokeModel, streaming, both cache lifetimes (5 minutes and 1 hour), model minimums, prompt changes and the exact lookback limit (21 positions hit, 22 miss, re-measured October 30 2026 on Sonnet 4.6 and 5.5 after Bedrock changed how it counts tool calls). Every check added in 0.3 was measured live first, and `scripts/live_v03_checks.py` re-runs those cases against Bedrock and compares CacheCanary's verdict with what Bedrock did (15 of 15 agreed in October 2026). Every check also has unit tests. The log reader is tested against real Bedrock invocation logs (redacted copies are in [tests/fixtures](https://github.com/Haarris/cachecanary/tree/main/tests/fixtures)). The Langfuse reader is tested against records that Strands, LiteLLM and LangChain wrote into a real Langfuse 4.64 server, and against a production Langfuse Cloud project, where its numbers matched a separate recount for every model. To run the live checks yourself, use an AWS Region without production traffic: `python scripts/live_test.py --region `. diff --git a/scripts/capture_langchain_requests.py b/scripts/capture_langchain_requests.py index 7a9c6df..06334ce 100644 --- a/scripts/capture_langchain_requests.py +++ b/scripts/capture_langchain_requests.py @@ -52,7 +52,8 @@ def run(script, model, prompts, middleware, system=SYSTEM): def save(name, body): (out / name).write_text(json.dumps(body, indent=2) + "\n") - # Finding 2: ChatBedrock (InvokeModel) with the caching middleware, 11 parallel tool calls. + # Finding 2: ChatBedrock (InvokeModel) with the caching middleware, 11 parallel tool calls. On Oct 7 2026 this lost the cache; Bedrock now + # counts a group of tool calls as one lookback position, so cachecanary diff reports it as kept. chat_bedrock = ChatBedrock(model=MODEL, region_name="us-west-2", endpoint_url=fake.url, model_kwargs={"max_tokens": 50}) bodies = run(["tools:11", "text"], chat_bedrock, ["Check my orders."], [BedrockPromptCachingMiddleware()]) save("invoke-turn1.json", bodies[0]) diff --git a/scripts/capture_litellm_requests.py b/scripts/capture_litellm_requests.py index 95cf143..ea66c09 100644 --- a/scripts/capture_litellm_requests.py +++ b/scripts/capture_litellm_requests.py @@ -83,7 +83,8 @@ def capture(path: Path, **kwargs) -> None: out.mkdir(parents=True, exist_ok=True) last_message = [{"location": "message", "role": "system"}, {"location": "message", "index": -1}] - # Finding 2: an agent step with 12 parallel tool calls. + # Finding 2: an agent step with 12 parallel tool calls. On Oct 7 2026 this lost the cache; Bedrock now + # counts a group of tool calls as one lookback position, so cachecanary diff reports it as kept. turn1 = [{"role": "system", "content": SYSTEM}, {"role": "user", "content": "Check orders 1 to 12 and tell me which shipped."}] capture(out / "agent-turn1.json", model=SONNET_46, tools=TOOLS, messages=turn1, diff --git a/scripts/capture_strands_requests.py b/scripts/capture_strands_requests.py index 6f8c687..69ce1da 100644 --- a/scripts/capture_strands_requests.py +++ b/scripts/capture_strands_requests.py @@ -49,7 +49,8 @@ def save(name, body): (out / name).write_text(json.dumps(body, indent=2) + "\n") auto = CacheConfig(strategy="auto") - # Finding 2: one agent step with 11 parallel tool calls. + # Finding 2: one agent step with 11 parallel tool calls. On Oct 7 2026 this lost the cache; Bedrock now + # counts a group of tool calls as one lookback position, so cachecanary diff reports it as kept. bodies = run(["tools:11", "text"], ["Check my orders."], auto) save("agent-turn1.json", bodies[0]) save("agent-turn2.json", bodies[1]) diff --git a/scripts/live_langchain_checks.py b/scripts/live_langchain_checks.py index 8d8e62a..01e3e9a 100644 --- a/scripts/live_langchain_checks.py +++ b/scripts/live_langchain_checks.py @@ -19,7 +19,6 @@ import argparse import json -import os import sys import time import uuid @@ -128,16 +127,17 @@ def twice(case, middleware, **model_kwargs): checks += [looked_up[1]["read"] > 1000, named[1]["read"] > 1000] # Agent loops: LangChain's real loop on the stand-in, then the exact requests replayed to Bedrock. - os.environ.setdefault("AWS_ACCESS_KEY_ID", "unused-by-the-stand-in") fake = Fake() def capture(script, *, invoke_route=False, middleware=None, manual_point=False): system, history = tag_prompts() fake.reset(script) if invoke_route: - model = ChatBedrock(model=MODEL, region_name=args.region, endpoint_url=fake.url, model_kwargs={"max_tokens": 50}) + model = ChatBedrock(model=MODEL, region_name=args.region, endpoint_url=fake.url, model_kwargs={"max_tokens": 50}, + aws_access_key_id="unused-by-the-stand-in", aws_secret_access_key="unused-by-the-stand-in") else: - model = ChatBedrockConverse(model=MODEL, region_name=args.region, endpoint_url=fake.url, max_tokens=50) + model = ChatBedrockConverse(model=MODEL, region_name=args.region, endpoint_url=fake.url, max_tokens=50, + aws_access_key_id="unused-by-the-stand-in", aws_secret_access_key="unused-by-the-stand-in") if manual_point: system = SystemMessage(content=[{"type": "text", "text": system}, ChatBedrockConverse.create_cache_point()]) agent = create_agent(model=model, tools=[lookup_order], system_prompt=system, @@ -172,14 +172,15 @@ def kept_with_retry(case, script, **kwargs): return kept(rows) return False + # Since Oct 2026 (by Oct 28) Bedrock counts a group of tool calls (and their results) as one lookback + # position, so every route keeps the whole conversation after a wide step. checks.append(kept_with_retry("ChatBedrockConverse, 21 parallel calls", ["tools:21", "text"])) - checks.append(not kept(replay("ChatBedrockConverse, 22 parallel calls", capture(["tools:22", "text"])))) + checks.append(kept_with_retry("ChatBedrockConverse, 40 parallel calls", ["tools:40", "text"])) checks.append(kept_with_retry("ChatBedrock, 10 parallel calls", ["tools:10", "text"], invoke_route=True)) - invoke_11 = replay("ChatBedrock, 11 parallel calls", capture(["tools:11", "text"], invoke_route=True)) - checks.append(invoke_11[1]["read"] == 0) # nothing read: no separate system prompt marker on this route + checks.append(kept_with_retry("ChatBedrock, 40 parallel calls", ["tools:40", "text"], invoke_route=True)) checks.append(kept_with_retry("middleware only, 11 parallel calls", ["tools:11", "text"])) - checks.append(not kept(replay("manual system cache point + middleware, 11 parallel calls", - capture(["tools:11", "text"], manual_point=True)))) + checks.append(kept_with_retry("manual system cache point + middleware, 40 parallel calls", ["tools:40", "text"], + manual_point=True)) # Context editing: stock clearing vs clearing in batches vs none, 13 calls with long tool results. tool_output["repeat"] = 40 diff --git a/scripts/live_litellm_checks.py b/scripts/live_litellm_checks.py index 2325ad2..5e38613 100644 --- a/scripts/live_litellm_checks.py +++ b/scripts/live_litellm_checks.py @@ -25,28 +25,10 @@ MODEL = "bedrock/converse/us.anthropic.claude-sonnet-4-6" TOOLS = [{"type": "function", "function": {"name": "lookup_order", "description": "Look up an order.", "parameters": {"type": "object", "properties": {"id": {"type": "string"}}}}}] +# The setup on the page (tests/test_site.py checks they match). SYSTEM_AND_LAST = [{"location": "message", "role": "system"}, {"location": "message", "index": -1}] -# Same helper as on the page (tests/test_site.py checks they match). -def cache_points(messages): - """Bedrock cache points for this call (Bedrock allows 4).""" - # The system prompt and the end of the conversation. - points = [{"location": "message", "role": "system"}, - {"location": "message", "index": -1}] - users = [i for i, m in enumerate(messages) if m["role"] == "user"] - rounds = [i for i, m in enumerate(messages) - if m["role"] == "assistant" and m.get("tool_calls")] - # The newest user message. - if users: - points.append({"location": "message", "index": users[-1]}) - # The first tool result of the newest round of tool calls. - if rounds and (not users or rounds[-1] > users[-1]) \ - and rounds[-1] + 1 < len(messages): - points.append({"location": "message", "index": rounds[-1] + 1}) - return points - - def lifetime_check() -> int: """One call asking for a 1-hour cache; prints the lifetime Bedrock reports it used.""" model, extra = sys.argv[2], json.loads(sys.argv[3]) @@ -121,27 +103,27 @@ def tool_round(r, n): checks.append((label, call(f"{label}, turn 10", chat + [{"role": "user", "content": "Question 9?"}], points))) checks_ok = [checks[0][1]["uncached"] > 5000, checks[1][1]["uncached"] < 20] - # Finding 2: 4 vs 12 parallel tool calls with points on the system prompt and the last message. + # Finding 2: 4 vs 12 parallel tool calls with points on the system prompt and the last message. Since late + # Oct 2026 Bedrock counts a group of tool calls (and their results) as one lookback position, so both keep it. start = fresh() - call("agent start", start, SYSTEM_AND_LAST, TOOLS) + first = call("agent start", start, SYSTEM_AND_LAST, TOOLS) twelve = call("12 parallel calls", start + tool_round(1, 12), SYSTEM_AND_LAST, TOOLS) four = call("4 parallel calls", start + tool_round(1, 4), SYSTEM_AND_LAST, TOOLS) - checks_ok += [twelve["read"] < four["read"] / 2, four["write"] < 1000] + checks_ok += [twelve["read"] >= first["prompt"] - 5, four["read"] >= first["prompt"] - 5, four["write"] < 1000] - # The helper through a 7-step session, then a 21-call round. + # The page's setup through an agent session with wide rounds: every step reads the whole previous request. steps = [fresh()] for r, n in ((1, 12), (2, 1), (3, 20)): steps.append(steps[-1] + tool_round(r, n)) steps.append(steps[-1] + [{"role": "assistant", "content": "All checked."}, {"role": "user", "content": "Refund order 3-4."}]) steps.append(steps[-1] + tool_round(4, 3)) + steps.append(steps[-1] + tool_round(5, 40)) previous = None for i, messages in enumerate(steps): - row = call(f"helper step {i}", messages, cache_points(messages), TOOLS) + row = call(f"session step {i}", messages, SYSTEM_AND_LAST, TOOLS) if previous is not None: checks_ok.append(row["read"] >= previous["prompt"] - 5) # read almost the whole previous request previous = row - wide = call("helper, 21 parallel calls", steps[-1] + tool_round(5, 21), cache_points(steps[-1] + tool_round(5, 21)), TOOLS) - checks_ok.append(wide["read"] < previous["prompt"] - 5) # Finding 4: the 1-hour lifetime. The price list is read at import, so each case runs in its own process. def lifetime(case, model, extra=None, built_in_prices=False): diff --git a/scripts/live_lookback_boundary.py b/scripts/live_lookback_boundary.py index 9670478..faa9ea6 100644 --- a/scripts/live_lookback_boundary.py +++ b/scripts/live_lookback_boundary.py @@ -1,4 +1,8 @@ -"""Measure where Bedrock's ~20-block cache lookback actually ends. +"""Measure where Bedrock's ~20-block cache lookback actually ends (the first measurement, early Oct 2026). + +Historical: when this ran, Bedrock counted every block, so 22+ blocks of parallel tool calls missed. +By Oct 28 2026 a run of tool calls (or tool results) counts as one position and these shapes all +hit. scripts/live_lookback_positions.py is the current check. Request A: tools + a short uncached system prompt + one long user turn ending in a cache point. Request B: the same history with that cache point removed, then an assistant turn with N parallel diff --git a/scripts/live_lookback_positions.py b/scripts/live_lookback_positions.py new file mode 100644 index 0000000..dd3afff --- /dev/null +++ b/scripts/live_lookback_positions.py @@ -0,0 +1,196 @@ +"""Check CacheCanary's lookback counting against live Bedrock. + +Request A caches one long user turn. Request B is the same turn plus some added blocks, with a +checkpoint on B's last block. B reads A's entry only if the added blocks fit in Bedrock's lookback. +For each shape the script asks diff.lookback_positions how many positions were added, predicts hit +(<= MAX_LOOKBACK_POSITIONS) or miss, sends both requests and reports whether Bedrock agreed. + +Measured Oct 30 2026 in us-west-2: every shape below matched on Sonnet 4.6 (Converse and InvokeModel, two +runs each, 52/52) and on Sonnet 5.5 (one run each, 24/24; with adaptive thinking the model made no +reasoning block, so the thinking shape was skipped there). The thinking shape uses a real reasoning block. Roughly +$1.50 for the default run (Sonnet 4.6, both APIs, two runs). + + python scripts/live_lookback_positions.py --region us-west-2 + python scripts/live_lookback_positions.py --region us-west-2 --model us.anthropic.claude-sonnet-5-5 --repeats 1 +""" + +import argparse +import copy +import json +import sys +import time +import uuid + +from cachecanary import diff, probe +from cachecanary.models import MAX_LOOKBACK_POSITIONS +from cachecanary.request import normalize + +SONNET = "us.anthropic.claude-sonnet-4-6" + + +def _text(n): + return [("assistant", ["text"]), ("user", ["text"] * n)] + + +def _parallel(n, tail=0, lead=0): + return [("assistant", ["text"] * lead + ["tu"] * n), ("user", ["tr"] * n + ["text"] * tail)] + + +def _sequential(rounds): + return [turn for _ in range(rounds) for turn in (("assistant", ["tu"]), ("user", ["tr"]))] + + +def _mixed(m): + return [("assistant", ["text", "tu"] * m), ("user", ["tr"] * m)] + + +SHAPES = [ + ("20 text blocks", _text(20)), ("21 text blocks", _text(21)), + ("10 sequential calls", _sequential(10)), ("11 sequential calls", _sequential(11)), + ("11 parallel calls", _parallel(11)), ("40 parallel calls", _parallel(40)), + ("5 parallel + 19 text", _parallel(5, tail=19)), ("5 parallel + 20 text", _parallel(5, tail=20)), + ("text before each of 9 calls", _mixed(9)), ("text before each of 10 calls", _mixed(10)), + ("18 text, 10 parallel", _parallel(10, lead=18)), ("19 text, 10 parallel", _parallel(10, lead=19)), +] + + +def first_turn(salt: str) -> str: + return f"[run {salt}] Background for this ticket: " + " ".join( + f"Rule {i}: answer politely and cite section {i}." for i in range(180)) + + +def converse_pair(salt, shape): + tool = {"toolSpec": {"name": "lookup_order", "description": "Look up an order.", + "inputSchema": {"json": {"type": "object", "properties": {"id": {"type": "string"}}}}}} + cp = {"cachePoint": {"type": "default"}} + + def block(kind, i): + if kind == "text": + return {"text": f"note {i}"} + if kind == "tu": + return {"toolUse": {"toolUseId": f"tu{i}", "name": "lookup_order", "input": {"id": str(i)}}} + return {"toolResult": {"toolUseId": f"tu{i}", "content": [{"text": f"order {i}: shipped"}]}} + turn = first_turn(salt) + a = {"toolConfig": {"tools": [tool]}, "system": [{"text": "You are a support agent."}], + "messages": [{"role": "user", "content": [{"text": turn}, cp]}], "inferenceConfig": {"maxTokens": 5}} + b = copy.deepcopy(a) + msgs = [{"role": "user", "content": [{"text": turn}]}] + texts, calls, pending = 0, 0, [] + for role, kinds in shape: + content = [] + for k in kinds: + if k == "text": + content.append(block("text", texts)) + texts += 1 + elif k == "tu": + content.append(block("tu", calls)) + pending.append(calls) + calls += 1 + else: # results answer the previous assistant turn's calls, in order + content.append(block("tr", pending.pop(0))) + msgs.append({"role": role, "content": content}) + msgs[-1]["content"].append(cp) + b["messages"] = msgs + return a, b + + +def invoke_pair(salt, shape): + a, b = converse_pair(salt, shape) + eph = {"type": "ephemeral"} + + def conv(item): + if "text" in item: + return {"type": "text", "text": item["text"]} + if "toolUse" in item: + u = item["toolUse"] + return {"type": "tool_use", "id": f"toolu_{u['toolUseId']}", "name": u["name"], "input": u["input"]} + r = item["toolResult"] + return {"type": "tool_result", "tool_use_id": f"toolu_{r['toolUseId']}", "content": r["content"][0]["text"]} + + def to_invoke(req): + msgs = [{"role": m["role"], "content": [conv(c) for c in m["content"] if "cachePoint" not in c]} + for m in req["messages"]] + msgs[-1]["content"][-1]["cache_control"] = eph + tool = req["toolConfig"]["tools"][0]["toolSpec"] + return {"anthropic_version": "bedrock-2023-05-31", "max_tokens": 5, + "tools": [{"name": tool["name"], "description": tool["description"], "input_schema": tool["inputSchema"]["json"]}], + "system": [{"type": "text", "text": req["system"][0]["text"]}], "messages": msgs} + return to_invoke(a), to_invoke(b) + + +def thinking_pair(client, model, salt, tail): + """A real reasoning block: ask the model (thinking on) to call the tool five times at once.""" + tool = {"toolSpec": {"name": "lookup_order", "description": "Look up an order by id.", + "inputSchema": {"json": {"type": "object", "properties": {"id": {"type": "string"}}, "required": ["id"]}}}} + cp = {"cachePoint": {"type": "default"}} + turn = first_turn(salt) + " Now look up orders 1, 2, 3, 4 and 5. Call the tool for all five at once." + a = {"toolConfig": {"tools": [tool]}, "system": [{"text": "You are a support agent."}], + "messages": [{"role": "user", "content": [{"text": turn}, cp]}], "inferenceConfig": {"maxTokens": 2000}, + "additionalModelRequestFields": {"thinking": {"type": "enabled", "budget_tokens": 1024}}} + try: + reply = client.converse(modelId=model, **a)["output"]["message"]["content"] + except Exception as exc: # Claude 5.x only takes adaptive thinking + if "adaptive" not in str(exc): + raise + a["additionalModelRequestFields"] = {"thinking": {"type": "adaptive"}} + reply = client.converse(modelId=model, **a)["output"]["message"]["content"] + asst = [c for c in reply if "text" not in c] # reasoning + tool calls only + if not any("toolUse" in c for c in asst): + return a, None + results = [{"toolResult": {"toolUseId": c["toolUse"]["toolUseId"], "content": [{"text": "shipped"}]}} + for c in asst if "toolUse" in c] + b = copy.deepcopy(a) + b["messages"] = [{"role": "user", "content": [{"text": turn}]}, {"role": "assistant", "content": asst}, + {"role": "user", "content": results + [{"text": f"extra {j}"} for j in range(tail)] + [cp]}] + return a, b + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__.split("\n")[0]) + parser.add_argument("--region", required=True) + parser.add_argument("--model", default=SONNET) + parser.add_argument("--repeats", type=int, default=2) + parser.add_argument("--apis", default="converse,invoke") + parser.add_argument("--pause", type=float, default=2.0) + args = parser.parse_args() + client = probe.make_client(args.region) + rows, wrong = [], 0 + + def check(name, api, a, b): + nonlocal wrong + na, nb = normalize(a, args.model), normalize(b, args.model) + pos = diff.lookback_positions(nb.blocks, na.checkpoint_indexes[-1], nb.checkpoint_indexes[-1]) + predict_hit = pos <= MAX_LOOKBACK_POSITIONS + probe._call(client, a, args.model, False) + time.sleep(args.pause) + u = probe._call(client, b, args.model, False) + hit = u.cache_read > 2000 # A's long first turn is ~2,900 tokens + ok = hit == predict_hit + wrong += not ok + rows.append({"shape": name, "api": api, "model": args.model, "positions": pos, + "blocks_added": nb.checkpoint_indexes[-1] - na.checkpoint_indexes[-1], + "b_read": u.cache_read, "b_write": u.cache_write, "hit": hit, "prediction_ok": ok}) + print(f"{api:8} {name:30} positions={pos:3} predict={'hit ' if predict_hit else 'miss'} " + f"read={u.cache_read:5} -> {'hit ' if hit else 'miss'} {'OK' if ok else 'WRONG'}", flush=True) + + for name, shape in SHAPES: + for api in args.apis.split(","): + for _ in range(args.repeats): + a, b = (converse_pair if api == "converse" else invoke_pair)(uuid.uuid4().hex[:10], shape) + check(name, api, a, b) + for tail in (19, 20): + for _ in range(args.repeats): + a, b = thinking_pair(client, args.model, uuid.uuid4().hex[:10], tail) + if b is None or not any("reasoningContent" in c for m in b["messages"] for c in m["content"]): + print("thinking: the model made no tool call or no reasoning block, skipped") + continue + check(f"thinking, 5 parallel, {tail} text", "converse", a, b) + + with open("live_lookback_positions.json", "w") as fh: + json.dump({"region": args.region, "model": args.model, "when": time.strftime("%Y-%m-%dT%H:%M:%S%z"), "rows": rows}, fh, indent=2) + print(f"\n{len(rows) - wrong}/{len(rows)} matched CacheCanary's prediction. Raw data: live_lookback_positions.json") + return 0 if not wrong else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/live_strands_checks.py b/scripts/live_strands_checks.py index b06d215..2baeecd 100644 --- a/scripts/live_strands_checks.py +++ b/scripts/live_strands_checks.py @@ -19,7 +19,6 @@ import argparse import json -import os import sys import time import uuid @@ -29,47 +28,11 @@ from _fake_bedrock_stream import Fake # noqa: E402 from strands.agent.conversation_manager import SlidingWindowConversationManager # noqa: E402 -from strands.hooks import BeforeModelCallEvent, HookProvider, HookRegistry # noqa: E402 MODEL = "us.anthropic.claude-sonnet-4-6" -# Same hook and conversation manager as on the page (tests/test_site.py checks they match). -class BedrockCachePoints(HookProvider): - """Place cache points in the messages before every model call.""" - - def register_hooks(self, registry: HookRegistry, **kwargs) -> None: - registry.add_callback(BeforeModelCallEvent, self.place) - - def place(self, event: BeforeModelCallEvent) -> None: - messages = event.agent.messages - for message in messages: # drop the points from the previous call - message["content"] = [b for b in message["content"] - if "cachePoint" not in b] - typed = [i for i, m in enumerate(messages) if m["role"] == "user" - and not any("toolResult" in b for b in m["content"])] - if typed: # the newest message a person typed - newest = messages[typed[-1]]["content"] - self._add(newest, len(newest)) - last = messages[-1]["content"] if messages else [] - if messages and (not typed or typed[-1] != len(messages) - 1): - if any("toolResult" in b for b in last) and len(last) > 1: - self._add(last, 1) # after the first result of the round - self._add(last, len(last)) # the end of the conversation - - @staticmethod - def _add(content, index): - # Bedrock rejects a cache point right after a non-PDF document, - # so step back past those. - def other_document(block): - return block.get("document", {}).get("format", "pdf") != "pdf" - - while index > 0 and other_document(content[index - 1]): - index -= 1 - if index > 0 and "cachePoint" not in content[index - 1]: - content.insert(index, {"cachePoint": {"type": "default"}}) - - +# Same conversation manager as on the page (tests/test_site.py checks they match). class TrimInChunks(SlidingWindowConversationManager): """Trim old messages in bulk, so the cached start rarely changes.""" @@ -173,17 +136,18 @@ def twice(case, **model_kwargs): checks += [arn_auto[1]["read"] == 0 and arn_auto[1]["write"] == 0, arn_anthropic[1]["read"] > 1000] # Agent loops: Strands' real loop on the stand-in, then the exact requests replayed to Bedrock. - os.environ.setdefault("AWS_ACCESS_KEY_ID", "unused-by-the-stand-in") fake = Fake() - def capture(script, prompts, *, hook=False, manager=None): + def capture(script, prompts, *, manager=None): system, history = tag_prompts() fake.reset(script) - model = BedrockModel(model_id=MODEL, endpoint_url=fake.url, region_name=args.region, - **({} if hook else {"cache_config": CacheConfig(strategy="auto")})) + # The stand-in needs no real credentials; a full fake pair keeps them out of the live checks. + stand_in = boto3.Session(aws_access_key_id="unused-by-the-stand-in", + aws_secret_access_key="unused-by-the-stand-in", region_name=args.region) + model = BedrockModel(model_id=MODEL, endpoint_url=fake.url, boto_session=stand_in, + cache_config=CacheConfig(strategy="auto")) agent = Agent(model=model, tools=[lookup_order], callback_handler=None, conversation_manager=manager, - system_prompt=[{"text": system}, {"cachePoint": {"type": "default"}}] if hook else system, - hooks=[BedrockCachePoints()] if hook else []) + system_prompt=system) for i, prompt in enumerate(prompts): agent((history + " " if i == 0 else "") + prompt) return [request["body"] for request in fake.requests] @@ -205,13 +169,14 @@ def replay(case, bodies, only=None): ten = replay("auto, 10 parallel calls", capture(["tools:10", "text"], ["Check my orders."])) eleven = replay("auto, 11 parallel calls", capture(["tools:11", "text"], ["Check my orders."])) - checks += [ten[1]["read"] >= ten[0]["prompt"] - 5, eleven[1]["read"] < eleven[0]["prompt"] / 2] + # Since Oct 2026 (by Oct 28) Bedrock counts a group of tool calls (and their results) as one lookback + # position, so auto mode keeps the whole conversation even after a wide step. + checks += [ten[1]["read"] >= ten[0]["prompt"] - 5, eleven[1]["read"] >= eleven[0]["prompt"] - 5] - session = replay("hook session", capture(["tools:12", "tools:1", "tools:20", "text", "tools:3", "tools:21", "text"], - ["Check my orders.", "Now refund order 3-4."], hook=True)) - for previous, row in zip(session[:5], session[1:6]): # every step up to the 3-call round reads it all + session = replay("auto session", capture(["tools:12", "tools:1", "tools:20", "text", "tools:3", "tools:40", "text"], + ["Check my orders.", "Now refund order 3-4."])) + for previous, row in zip(session[:6], session[1:7]): # every step reads the whole previous request checks.append(row["read"] >= previous["prompt"] - 5) - checks.append(session[6]["read"] < session[5]["prompt"] - 5) # the 21-call round loses the round before chat = [f"Question {i}: summarise order batch {i}." for i in range(26)] default = replay("default window, long chat", capture(["text:40"] * 26, chat), only=set(range(19, 26))) @@ -220,10 +185,10 @@ def replay(case, bodies, only=None): checks += [all(r["read"] < r["prompt"] / 4 for r in default[2:]), # after the trim, every turn rewrites all(r["read"] >= p["prompt"] - 5 for p, r in zip(chunked[2:], chunked[3:]))] - # A fully live agent with the hook: the model chooses its own tool calls. + # A fully live agent in auto mode: the model chooses its own tool calls. system, _ = tag_prompts() - agent = Agent(model=live_model(model_id=MODEL), tools=[lookup_order], hooks=[BedrockCachePoints()], - callback_handler=None, system_prompt=[{"text": system}, {"cachePoint": {"type": "default"}}]) + agent = Agent(model=live_model(model_id=MODEL, cache_config=CacheConfig(strategy="auto")), tools=[lookup_order], + callback_handler=None, system_prompt=system) before = totals(agent) reads = [] for prompt in ("Look up orders 1, 2, 3, 4, 5 and 6 at the same time (call the tool for each in parallel), " @@ -233,7 +198,7 @@ def replay(case, bodies, only=None): delta = {k: now[k] - before[k] for k in now} before = now calls = [sum("toolUse" in b for b in m["content"]) for m in agent.messages if m["role"] == "assistant"] - show({"case": "live agent with the hook", "prompt": prompt[:28], **delta, "tool_calls_per_turn": calls}) + show({"case": "live agent, auto mode", "prompt": prompt[:28], **delta, "tool_calls_per_turn": calls}) reads.append(delta["read"]) checks.append(reads[1] > reads[0] > 0) diff --git a/scripts/live_test.py b/scripts/live_test.py index ec2a19b..6fd2d18 100644 --- a/scripts/live_test.py +++ b/scripts/live_test.py @@ -61,28 +61,32 @@ def with_tools(req: dict, tools: list) -> dict: def tool_history(system_text: str, first_turn_text: str, n_calls: int, intermediate_cp: bool) -> tuple[dict, dict]: - """Reproduces the 'rolling checkpoint' pattern from pydantic-ai #9404. + """Reproduces the 'rolling checkpoint' pattern from pydantic-ai #9404, in the shape that still + breaks it on Bedrock. Request A: system (checkpoint) + a long first user turn with a checkpoint at its end. - Request B: the same history WITHOUT the old turn checkpoint (it rolls to the end), plus one - assistant turn with n parallel tool calls and their results. 2n new blocks sit between A's - conversation checkpoint and B's, so with n=12 Bedrock's ~20-block lookback cannot reach A's - entry and only the system cache is read. intermediate_cp adds a checkpoint after the assistant - turn (within 20 blocks of A's), which should recover the conversation cache. + Request B: the same history WITHOUT the old turn checkpoint (it rolls to the end), plus n rounds of + one tool call and its result. Each call and each result is one lookback position, so with n=11 + (22 positions) Bedrock cannot reach A's entry and only the system cache is read. (Parallel calls no + longer break it: by Oct 28 2026 a group of calls side by side counts as one position.) + intermediate_cp adds a checkpoint after the fifth result (within reach of A's), which should + recover the conversation cache. """ cp = {"cachePoint": {"type": "default"}} system = [{"text": system_text}, cp] a = with_tools({"system": system, "messages": [{"role": "user", "content": [{"text": first_turn_text}, cp]}], "inferenceConfig": {"maxTokens": 5}}, [TOOL_A]) - calls = [{"toolUse": {"toolUseId": f"tu{i}", "name": "lookup_order", "input": {"id": str(i)}}} for i in range(n_calls)] - results = [{"toolResult": {"toolUseId": f"tu{i}", "content": [{"text": f"order {i}: shipped"}]}} for i in range(n_calls)] b = copy.deepcopy(a) - b["messages"] = [ - {"role": "user", "content": [{"text": first_turn_text}]}, - {"role": "assistant", "content": calls + ([cp] if intermediate_cp else [])}, - {"role": "user", "content": results + [cp]}, - ] + msgs = [{"role": "user", "content": [{"text": first_turn_text}]}] + for i in range(n_calls): + msgs.append({"role": "assistant", "content": [ + {"toolUse": {"toolUseId": f"tu{i}", "name": "lookup_order", "input": {"id": str(i)}}}]}) + msgs.append({"role": "user", "content": [ + {"toolResult": {"toolUseId": f"tu{i}", "content": [{"text": f"order {i}: shipped"}]}}] + + ([cp] if intermediate_cp and i == 4 else [])}) + msgs[-1]["content"].append(cp) + b["messages"] = msgs return a, b @@ -163,14 +167,14 @@ def pair(name, a, b, model, expect_hit, stream=False, expect_reason=None): except Exception as exc: record("model switched", "miss", False, f"ERROR {probe._translate(exc)}") - # 4. 20-block lookback: 12 parallel tool calls = 24 new blocks between conversation checkpoints. + # 4. Lookback: 11 rounds of one tool call = 22 new positions between conversation checkpoints. # Both variants still read the SYSTEM cache, so judge by how many tokens were read: without a # middle checkpoint the long first turn (~1.5k tokens) must NOT come from cache; with one it must. reads = {} for variant, mid in (("no middle checkpoint", False), ("middle checkpoint", True)): sys_text = policy(220, salt + ("k" if not mid else "l")) turn = "Background for this ticket: " + policy(130, salt + ("m" if not mid else "n")) - a, b2 = tool_history(sys_text, turn, 12, intermediate_cp=mid) + a, b2 = tool_history(sys_text, turn, 11, intermediate_cp=mid) try: call(client, a, SONNET) time.sleep(args.pause) @@ -183,7 +187,7 @@ def pair(name, a, b, model, expect_hit, stream=False, expect_reason=None): if "no middle checkpoint" in reads and "middle checkpoint" in reads: no_mid, mid = reads["no middle checkpoint"], reads["middle checkpoint"] recovered = mid - no_mid > 1000 # the first turn is ~1.5k tokens - record("lookback >20 blocks (rolling checkpoint)", "partial miss", + record("lookback >21 positions (rolling checkpoint)", "partial miss", recovered and "lookback-exceeded" in reads["no middle checkpoint diff"], f"read without middle cp={no_mid}, with middle cp={mid}", f"diff(no mid)={reads['no middle checkpoint diff']} diff(mid)={reads['middle checkpoint diff']}") @@ -197,7 +201,7 @@ def pair(name, a, b, model, expect_hit, stream=False, expect_reason=None): json.dump({"region": args.region, "run": salt, "results": results}, fh, indent=2) failed = [r for r in rows if r[2] != "PASS"] print(f"\n{len(rows) - len(failed)}/{len(rows)} scenarios matched expectations. Raw data: live_results.json") - print("Note: on 'lookback >20 blocks' the 2nd call may still hit the SYSTEM cache (read>0) while missing the " + print("Note: on 'lookback >21 positions' the 2nd call may still hit the SYSTEM cache (read>0) while missing the " "conversation cache; compare read tokens with the 'middle checkpoint' row to see the difference.") return 0 if not failed else 1 diff --git a/site/check.js b/site/check.js index 0520ddf..e50e108 100644 --- a/site/check.js +++ b/site/check.js @@ -178,13 +178,13 @@ var CAUSES = { healthy: ["Keep it that way: check saved requests in CI with cachecanary lint, so a library upgrade can't quietly break it."], leaking: [ - "Agents that run many tools at once: Bedrock only looks back about 20 blocks for the cached part.", + "Agents that make many tool calls one after another, or add lots of separate text blocks: Bedrock only looks back about 20 steps for the cached part (tool calls made at the same time count as one).", "Calls more than 5 minutes apart, so the cache expires in between.", "Something in the cached part changes on some requests, like a date, a user name or the order of your tools." ], poor: [ "Something in the cached part changes on most requests, like a date, a request ID or the order of your tools.", - "Agents that run many tools at once: Bedrock only looks back about 20 blocks for the cached part.", + "Agents that make many tool calls one after another, or add lots of separate text blocks: Bedrock only looks back about 20 steps for the cached part (tool calls made at the same time count as one).", "Calls more than 5 minutes apart, so the cache expires in between." ], "never-read": [ diff --git a/site/index.html b/site/index.html index f7277b9..602f2cd 100644 --- a/site/index.html +++ b/site/index.html @@ -31,7 +31,7 @@ "url": "https://cachecanary.com/", "applicationCategory": "DeveloperApplication", "operatingSystem": "macOS, Linux, Windows", - "softwareVersion": "0.4.0", + "softwareVersion": "0.4.1", "downloadUrl": "https://pypi.org/project/cachecanary/", "license": "https://www.apache.org/licenses/LICENSE-2.0", "offers": { @@ -177,7 +177,7 @@

Why caching breaks

  • A small edit put today's date, a request ID or the user's name into the system prompt.
  • Your tools are listed in a different order on each request.
  • Thinking or effort settings changed between two calls. On Bedrock that throws away the whole cache, system prompt included.
  • -
  • An agent ran a dozen tools at once. Bedrock only searches back about 20 pieces of the conversation (messages, tool calls and tool results) for the cached part, so it can lose track of it.
  • +
  • An agent made a long chain of tool calls one after another, or added lots of separate text blocks between two cache points. Bedrock only searches back about 20 steps for the cached part, so it can lose track of it. (Tool calls made at the same time count as one step.)
  • I wrote these up in more detail, with the bug reports behind each one and the lookback limit I measured: Five ways Claude prompt caching quietly breaks on Amazon Bedrock.

    Anthropic added cache diagnostics to its own API, so you can see why a request missed. They don't cover Bedrock. That's the gap CacheCanary fills.

    @@ -185,9 +185,9 @@

    Why caching breaks

    Guides for your framework

    For each framework I captured exactly what it sends to Bedrock, checked it with CacheCanary and tested it on live Bedrock. Each of those guides shows where the cache is lost and a setup that keeps it. The Langfuse guide shows how to read your real hit rate from your traces.

    diff --git a/site/langchain/index.html b/site/langchain/index.html index 2e6f8f5..0b3b617 100644 --- a/site/langchain/index.html +++ b/site/langchain/index.html @@ -4,9 +4,9 @@ LangChain prompt caching on Amazon Bedrock: what actually gets cached - + - + @@ -27,10 +27,10 @@ "@context": "https://schema.org", "@type": "TechArticle", "headline": "Prompt caching with LangChain on Amazon Bedrock", - "description": "I captured the exact Bedrock requests LangChain (langchain-aws 1.8.1) sends from create_agent and tested them on live Bedrock. ChatBedrock, context editing and your own cache points lose the cache. Here is a setup that keeps it.", + "description": "I captured the exact Bedrock requests LangChain (langchain-aws 1.8.1) sends from create_agent and tested them on live Bedrock. Caching is off until you add the middleware, and context editing loses the cache. Here is a setup that keeps it.", "image": "https://cachecanary.com/og-langchain.png?v=1", "datePublished": "2026-10-07", - "dateModified": "2026-10-07", + "dateModified": "2026-10-30", "author": { "@type": "Person", "name": "Haris Farooq", @@ -103,9 +103,9 @@

    Prompt caching with LangChain on Amazon Bedrock

    -

    LangChain talks to Bedrock through the langchain-aws package, and its agents add caching with a middleware. I captured the exact requests langchain-aws 1.8.1 and LangChain 1.4.3 send while create_agent runs its real agent loop, checked them with CacheCanary, and ran every case on live Bedrock (Claude Sonnet 4.6, us-west-2, October 2026).

    +

    LangChain talks to Bedrock through the langchain-aws package, and its agents add caching with a middleware. I captured the exact requests langchain-aws 1.8.1 and LangChain 1.4.3 send while create_agent runs its real agent loop, checked them with CacheCanary, and ran every case on live Bedrock (Claude Sonnet 4.6, us-west-2, October 2026, re-checked October 30).

    -

    The short version: caching is off until you add the middleware, and with ChatBedrockConverse it then works well, even with many tool calls at once. Three things quietly throw it away: using ChatBedrock instead, LangChain's own context editing, and adding a cache point of your own.

    +

    The short version: caching is off until you add the middleware, and it then works well, even with many tool calls at once. The thing that quietly throws it away is LangChain's own context editing.

    The setup I'd use

    from langchain.agents import create_agent
    @@ -118,7 +118,7 @@ 

    The setup I'd use

    system_prompt=SYSTEM_PROMPT, middleware=[BedrockPromptCachingMiddleware()], )
    -

    Don't add cache points of your own on top (finding 4), and if you clear old tool results, clear them in batches (finding 3).

    +

    You don't need cache points of your own on top, and if you clear old tool results, clear them in batches (finding 3).

    What I found

    @@ -135,20 +135,21 @@

    1. Caching is off until you add the middleware

    Outside an agent, pass the same setting per call: model.invoke(messages, cache_control={"type": "ephemeral"}). With the middleware, ChatBedrockConverse puts a cache point after the tool list, after the system prompt, at the end of the newest message and at the end of the message before it.

    -

    2. Use ChatBedrockConverse, not ChatBedrock

    -

    Bedrock only looks back 21 blocks from a cache point to find the previous cache entry, and every tool call and every tool result is a block. With ChatBedrockConverse, the point on the message before the newest one sits right after the model's tool calls, close enough to find the previous entry, so the cache holds for up to 21 tool calls at once. ChatBedrock (the InvokeModel API) gets only one point, at the end of the newest message, and sends the system prompt as plain text with no point of its own. So at 11 tool calls the previous entry is out of reach, and there is nothing earlier to fall back on: the whole prompt, system prompt included, is written to the cache again. On live Bedrock, with a long customer history in the first message:

    +

    2. Many tool calls at once: fine since October 2026

    +

    Bedrock only looks back about 20 positions from a cache point to find the previous cache entry. When I first tested this on October 7 2026, it counted every tool call and every tool result as its own position. Back then ChatBedrock (the InvokeModel API, one cache point at the end of the newest message) lost the whole cache at 11 tool calls at once, ChatBedrockConverse at 22, and with a cache point of your own on the system message it lost it at 11 again, because the middleware then skips its point on the message before the newest one. By October 28 Bedrock counted a group of tool calls side by side as one position, and the same for their results, as Anthropic documents for its own API. The same requests on live Bedrock, with a long customer history in the first message:

    - + - - - - + + + + +
    Next stepRead from cacheWritten again
    Next stepRead from cacheOn October 7
    ChatBedrock, 10 tool calls at once6,9301,783
    ChatBedrock, 11 tool calls at once08,890
    ChatBedrockConverse, 21 tool calls at once6,9303,708
    ChatBedrockConverse, 22 tool calls at once2,222 (system prompt only)8,587
    ChatBedrock, 10 tool calls at once6,9286,930
    ChatBedrock, 40 tool calls at once6,9300 at 11 calls
    ChatBedrockConverse, 21 tool calls at once6,9286,930
    ChatBedrockConverse, 40 tool calls at once6,9342,222 at 22 calls (system prompt only)
    Your own system cache point + middleware, 40 tool calls at once6,9282,226 at 11 calls (system prompt only)
    -

    Writing to the cache costs 1.25 times the normal input price, and a cache read a tenth of it, so each of those misses costs 12.5 times more than it should for the part written again.

    +

    So either model class works. I'd still use ChatBedrockConverse: ChatBedrock sends the system prompt as plain text with no cache point of its own, so if something does change in the conversation, the system prompt is written again too. A step can still lose the cache if it adds more than about 20 other blocks at once, like 22 or more separate text blocks; cachecanary diff counts positions the way Bedrock does now.

    3. Context editing rewrites the conversation on every call

    ContextEditingMiddleware with ClearToolUsesEdit replaces old tool results with "[cleared]" once the conversation passes a token count (100,000 by default), keeping the newest 3. With its default settings, it does this again on every call without saving it, so each new tool result pushes one more old result out of the newest 3. An earlier part of the history changes on every call, and the cache points that come after it can't find the previous entry. Once it starts, every call writes most of the conversation to the cache again. (Setting clear_at_least stops the rewrites, because the same oldest results get cleared every time, but then it never frees more than that many tokens.)

    @@ -191,19 +192,6 @@

    3. Context editing rewrites the conversation on every call

    Cost in full-price input tokens: reads count a tenth, writes 1.25.

    So the stock edit, meant to save tokens, cost about 4 times as much per step as not clearing at all, while clearing in batches cost a little less. Clearing in batches rewrote the history on 2 calls out of 31 in my offline run, against every call for the stock edit, and the newest tool results are never cleared. Clearing in batches keeps up to keep + chunk - 1 recent results instead of exactly keep, so the model sees a few more. It also works with exclude_tools and clear_tool_inputs.

    -

    4. A cache point of your own turns off LangChain's extra point

    -

    Older guides tell you to add ChatBedrockConverse.create_cache_point() to the system message. If you keep that and also use the middleware, LangChain skips the point on the message before the newest one, because it only adds it when the request has no cache points of yours. The limit drops from 21 tool calls at once back to 10. On live Bedrock, 11 tool calls at once:

    -
    - - - - - - -
    SetupRead from cacheWritten again
    Middleware only6,9341,958
    Your own system cache point + middleware2,226 (system prompt only)6,666
    -
    -

    The middleware already caches the system prompt, so remove your own cache points when you use it.

    -

    Smaller things

    What LangChain gets right

    -

    The point on the message before the newest one is what lets ChatBedrockConverse handle 21 tool calls at once. The middleware doesn't save cache points into your conversation history or checkpoints, so restored threads stay clean. Structured output with response_format adds its tool to every call, so it doesn't break the cache. It worked with extended thinking turned on, and a cache point in a tool result's content is moved to where Bedrock accepts it.

    +

    With ChatBedrockConverse, the middleware gives the tools and the system prompt cache points of their own, so a change later in the conversation never costs you those. The middleware doesn't save cache points into your conversation history or checkpoints, so restored threads stay clean. Structured output with response_format adds its tool to every call, so it doesn't break the cache. It worked with extended thinking turned on, and a cache point in a tool result's content is moved to where Bedrock accepts it.

    Check your own agent

    Save the exact requests your model sends. This works for ChatBedrockConverse and ChatBedrock, streaming or not; I checked each saved file is identical to what reached Bedrock.

    @@ -239,19 +227,16 @@

    Check your own agent

    model = ChatBedrockConverse(model="us.anthropic.claude-sonnet-4-6") save_requests(model)

    Run a conversation, then compare two requests in a row with cachecanary diff. These are real outputs on requests LangChain built:

    -
    $ cachecanary diff invoke-turn1.json invoke-turn2.json --model us.anthropic.claude-sonnet-4-6
    -lookback-exceeded: 22 blocks were added between the previous checkpoint and the next one. Bedrock only finds an earlier cache entry up to 21 blocks back (21 added still hits, 22 misses), so this call writes the cache again. Add a checkpoint in between (common after many parallel tool calls). (messages[2].content[10])
    -
    -$ cachecanary diff edit-call8.json edit-call9.json --model us.anthropic.claude-sonnet-4-6
    +
    $ cachecanary diff edit-call8.json edit-call9.json --model us.anthropic.claude-sonnet-4-6
     history-changed: Earlier conversation history was edited (not just appended). (messages[12].content[0])
     
     $ cachecanary diff date-day1.json date-day2.json --model us.anthropic.claude-sonnet-4-6
     system-changed: The system prompt changed inside the cached prefix (look for dates, IDs or per-user text). (system[0])
    -

    In plain words: the first pair is ChatBedrock with 11 tool calls at once (finding 2). The second is the stock context editing clearing one more tool result (finding 3). The third is a date in a dynamic system prompt.

    +

    In plain words: the first pair is the stock context editing clearing one more tool result (finding 3). The second is a date in a dynamic system prompt.

    Keep a few saved requests in your repo and run cachecanary lint on them in CI, and a LangChain upgrade that changes your cache points shows up in a pull request instead of on the bill. Setup is on the home page.

    How I tested this

    -

    langchain-aws 1.8.1, LangChain 1.4.3 and LangGraph 1.2.14 in a clean Python environment. To capture requests, I pointed LangChain at a small local server standing in for Bedrock that speaks the Converse and InvokeModel formats, streaming or not, so create_agent ran its real agent loop and every request body was saved. The settings cases ran fully live on Bedrock (Claude Sonnet 4.6, us-west-2, October 7 2026). A live model won't call exactly 21 tools on request, so for the agent-loop cases I sent the exact requests LangChain built, in order, to live Bedrock; caching depends only on the request. Every case uses made-up prompts with a fresh random tag, so old cache entries couldn't help. One run read nothing at all from the cache, system prompt included, and four reruns of the same case kept it. That pattern fits cross-Region inference: with a us. model ID, two calls can land in different Regions, and AWS notes this can mean more cache writes. The capture and the live checks are scripts in the repo: capture and live checks. Token counts move by a few tokens from run to run because each run uses a new random tag. LangChain changes quickly, so check your own version with the snippet above.

    +

    langchain-aws 1.8.1, LangChain 1.4.3 and LangGraph 1.2.14 in a clean Python environment. To capture requests, I pointed LangChain at a small local server standing in for Bedrock that speaks the Converse and InvokeModel formats, streaming or not, so create_agent ran its real agent loop and every request body was saved. The settings cases ran fully live on Bedrock (Claude Sonnet 4.6, us-west-2, October 7 2026), and I re-ran every case on October 30 2026 after Bedrock changed how it counts tool calls. A live model won't call exactly 21 tools on request, so for the agent-loop cases I sent the exact requests LangChain built, in order, to live Bedrock; caching depends only on the request. Every case uses made-up prompts with a fresh random tag, so old cache entries couldn't help. One run read nothing at all from the cache, system prompt included, and four reruns of the same case kept it. That pattern fits cross-Region inference: with a us. model ID, two calls can land in different Regions, and AWS notes this can mean more cache writes. The capture and the live checks are scripts in the repo: capture and live checks. Token counts move by a few tokens from run to run because each run uses a new random tag. LangChain changes quickly, so check your own version with the snippet above.

    Same tests for other frameworks: LiteLLM and Strands Agents. More on why caching breaks on Bedrock: Five ways Claude prompt caching quietly breaks on Amazon Bedrock.

    Want a quick number for your own app? Paste three totals from CloudWatch into the cache checker. It runs in your browser, nothing to install.

    Using CacheCanary? I'd love to hear how it's going, good or bad, and whether you'd want the dashboard. Email me at hello@cachecanary.com. I read every message.

    diff --git a/site/litellm/index.html b/site/litellm/index.html index 8f84ff6..9c5e016 100644 --- a/site/litellm/index.html +++ b/site/litellm/index.html @@ -6,7 +6,7 @@ LiteLLM prompt caching on Amazon Bedrock: what actually gets cached - + @@ -30,7 +30,7 @@ "description": "I captured the exact Bedrock requests LiteLLM 1.104.0 sends and tested them on live Bedrock. Where cache_control_injection_points lose the cache, and a setup that keeps it.", "image": "https://cachecanary.com/og-litellm.png?v=2", "datePublished": "2026-10-07", - "dateModified": "2026-10-07", + "dateModified": "2026-10-30", "author": { "@type": "Person", "name": "Haris Farooq", @@ -103,56 +103,46 @@

    Prompt caching with LiteLLM on Amazon Bedrock

    -

    LiteLLM can add Bedrock cache points for you. I wanted to know what it really sends, so I captured the exact requests LiteLLM 1.104.0 builds for Bedrock, checked them with CacheCanary, and ran the important cases on live Bedrock (Claude Sonnet 4.6, us-west-2, October 2026).

    +

    LiteLLM can add Bedrock cache points for you. I wanted to know what it really sends, so I captured the exact requests LiteLLM 1.104.0 builds for Bedrock, checked them with CacheCanary, and ran the important cases on live Bedrock (Claude Sonnet 4.6, us-west-2, October 2026, re-checked October 30).

    The short version: LiteLLM puts cache points in valid places. The money is lost in where you ask it to put them, and in a few settings it drops without telling you.

    The setup I'd use

    -

    Bedrock lets you use 4 cache points per request. This helper spends them on the four places that matter: the system prompt, your newest user message, the start of the newest round of tool results, and the end of the conversation.

    -
    def cache_points(messages):
    -    """Bedrock cache points for this call (Bedrock allows 4)."""
    -    # The system prompt and the end of the conversation.
    -    points = [{"location": "message", "role": "system"},
    -              {"location": "message", "index": -1}]
    -    users = [i for i, m in enumerate(messages) if m["role"] == "user"]
    -    rounds = [i for i, m in enumerate(messages)
    -              if m["role"] == "assistant" and m.get("tool_calls")]
    -    # The newest user message.
    -    if users:
    -        points.append({"location": "message", "index": users[-1]})
    -    # The first tool result of the newest round of tool calls.
    -    if rounds and (not users or rounds[-1] > users[-1]) \
    -            and rounds[-1] + 1 < len(messages):
    -        points.append({"location": "message", "index": rounds[-1] + 1})
    -    return points
    -
    -
    -response = litellm.completion(
    +

    Two cache points are enough: one on the system prompt and one on the last message, so the conversation point moves forward on every call.

    +
    response = litellm.completion(
         model="bedrock/converse/us.anthropic.claude-sonnet-4-6",
         messages=messages,
         tools=tools,
    -    cache_control_injection_points=cache_points(messages),
    +    cache_control_injection_points=[
    +        {"location": "message", "role": "system"},
    +        {"location": "message", "index": -1},
    +    ],
     )
    -

    Work out the points again on every call, because the indexes move as the conversation grows. I ran a 7-step agent session with it on live Bedrock: each step read almost the whole previous prompt from the cache.

    +

    I ran a 7-step agent session with it on live Bedrock, with rounds of 12, 20 and 40 parallel tool calls: each step read almost the whole previous prompt from the cache.

    - - - - - + + + + + +
    StepRead from cacheSize of the previous request
    12 parallel tool calls6,9266,929
    1 tool call8,8678,868
    20 parallel tool calls9,0309,031
    A follow-up question from the user12,24012,241
    3 parallel tool calls12,25612,259
    12 parallel tool calls6,9206,923
    1 tool call8,8618,862
    20 parallel tool calls9,0249,025
    A follow-up question from the user12,23412,235
    3 parallel tool calls12,25012,253
    40 parallel tool calls12,76012,761

    Tokens. The last few tokens of each request are always new, so the cache can't hold them.

    +

    On October 7 2026 wide tool rounds needed two more points, and this page had a helper for them. Bedrock no longer needs it (finding 2).

    If you use the LiteLLM Proxy, the same points go in the request body. With the OpenAI client:

    response = client.chat.completions.create(
         model="claude-sonnet",
         messages=messages,
         tools=tools,
    -    extra_body={"cache_control_injection_points": cache_points(messages)},
    +    extra_body={"cache_control_injection_points": [
    +        {"location": "message", "role": "system"},
    +        {"location": "message", "index": -1},
    +    ]},
     )

    What I found

    @@ -171,21 +161,21 @@

    1. role: "user" caches your oldest messages, not your newest

    Use index: -1 (the last message) instead of role: "user".

    -

    2. Many parallel tool calls lose the conversation

    -

    With points on the system prompt and the last message, the conversation point moves to the newest tool result on each step. That works until one step adds a lot of blocks. Bedrock only looks back 21 blocks from a cache point for the previous cache entry, and every tool call and every tool result is a block. 12 parallel calls add 24 blocks, so the previous entry is out of reach and the conversation is written to the cache again. On live Bedrock, with a long customer history in the first user message:

    +

    2. Many parallel tool calls: fine since October 2026

    +

    With points on the system prompt and the last message, the conversation point moves to the newest tool result on each step. Bedrock only looks back about 20 positions from a cache point for the previous cache entry. When I first tested this on October 7 2026, it counted every tool call and every tool result as its own position, so 12 parallel calls put the previous entry out of reach and the conversation was written to the cache again, at 1.25 times the normal input price instead of a tenth of it. By October 28 Bedrock counted a group of tool calls side by side as one position, and the same for their results, as Anthropic documents for its own API. The same requests on live Bedrock, with a long customer history in the first user message:

    - + - - + +
    Next stepRead from cacheWritten again
    Next stepRead from cacheWritten
    4 parallel tool calls6,920669
    12 parallel tool calls2,214 (system prompt only)6,647
    4 parallel tool calls6,918669
    12 parallel tool calls6,918 (on October 7: 2,214, the system prompt only)1,941
    -

    The part that was written again costs 1.25 times the normal input price instead of a tenth of it, so 12.5 times more than a cache read. In a long agent session that happens on every wide step. The helper above fixes it with the point on the first result of the newest round: that point is close enough to find the previous entry. It holds for up to 20 parallel calls in one round. At 21, the round before it is written again, but the point on the newest user message still holds everything up to there (in my run: 12,256 of the previous 12,767 tokens read from the cache).

    +

    A step can still lose the cache if it adds more than about 20 other blocks at once, like 22 or more separate text blocks; cachecanary diff counts positions the way Bedrock does now.

    3. A cache point on an assistant tool-call message is dropped

    -

    The obvious place for an extra point is the assistant message that holds the tool calls. When that message has only tool calls and no text, LiteLLM has nothing to attach the point to and sends the request without it. No warning. That's why the helper uses the first tool result instead, which LiteLLM turns into a valid Bedrock cache point.

    +

    If you add a point on the assistant message that holds the tool calls, and that message has only tool calls and no text, LiteLLM has nothing to attach the point to and sends the request without it. No warning. A point on a tool result does work: LiteLLM turns it into a valid Bedrock cache point after the result.

    4. The 1-hour cache lifetime can be dropped without a warning

    You can ask for a 1-hour cache with "ttl": "1h" in cache_control, or in an injection point's control. LiteLLM only sends it for models whose entry in its price list says they support it. Otherwise it sends a normal 5-minute cache point. I found two cases where that happens to a model that does support 1 hour:

    @@ -232,20 +222,17 @@

    Check your own app

    litellm.callbacks = [SaveBedrockRequests()]

    Run one conversation, then compare two requests in a row with cachecanary diff. These are real outputs on requests LiteLLM built:

    -
    $ cachecanary diff agent-turn1.json agent-turn2.json --model us.anthropic.claude-sonnet-4-6
    -lookback-exceeded: 24 blocks were added between the previous checkpoint and the next one. Bedrock only finds an earlier cache entry up to 21 blocks back (21 added still hits, 22 misses), so this call writes the cache again. Add a checkpoint in between (common after many parallel tool calls). (messages[2].content[11])
    -
    -$ cachecanary diff chat-turn6.json chat-turn7.json --model us.anthropic.claude-sonnet-4-6
    +
    $ cachecanary diff chat-turn6.json chat-turn7.json --model us.anthropic.claude-sonnet-4-6
     prefix-identical: The cached prefix is identical. If B still missed, the entry likely expired (TTL elapsed between calls) or cross-region inference routed to a Region without the entry.
     checkpoint-not-moved: The cached part was read, but the last cache point didn't move while 8 new blocks were added after it. Those are paid at full price on every call. Put the cache point at the end of the newest message. (messages[4].content[0])
     
     $ cachecanary diff with-download.json without-download.json --model us.anthropic.claude-sonnet-5-5
     ttl-changed: Checkpoint TTL changed (1h -> 5m); entries are keyed by TTL.
    -

    In plain words: the first pair is the 12-tool-call step from finding 2. The second is the role: "user" chat from finding 1: the cache works, but the newest messages are never added to it. The third is the same Sonnet 5.5 request with and without LiteLLM's price list download (finding 4).

    +

    In plain words: the first pair is the role: "user" chat from finding 1: the cache works, but the newest messages are never added to it. The second is the same Sonnet 5.5 request with and without LiteLLM's price list download (finding 4).

    Keep a few saved requests in your repo and run cachecanary lint on them in CI, and a LiteLLM upgrade that changes your cache points shows up in a pull request instead of on the bill. Setup is on the home page.

    How I tested this

    -

    LiteLLM 1.104.0 in a clean Python environment. To capture requests, I pointed LiteLLM at a small local server standing in for Bedrock that saved every request body. For the live numbers I sent the same requests to Bedrock (Claude Sonnet 4.6, us-west-2, October 7 2026) with made-up prompts and a fresh random tag in each run, so old cache entries couldn't help. The capture and the live checks are scripts in the repo, so you can run them yourself: capture and live checks. I also ran LiteLLM's own proxy server against live Bedrock, called with the OpenAI client as shown above, and got the same numbers as the library. Token counts move by a few tokens from run to run because each run uses a new random tag. On October 12 I captured the same requests again with LiteLLM 1.105.0: findings 1 to 3 and the profile ARN case are unchanged, and the Sonnet 5.5 case is fixed. LiteLLM changes quickly, so check your own version with the callback above.

    +

    LiteLLM 1.104.0 in a clean Python environment. To capture requests, I pointed LiteLLM at a small local server standing in for Bedrock that saved every request body. For the live numbers I sent the same requests to Bedrock (Claude Sonnet 4.6, us-west-2, October 7 2026, and again on October 30 2026 after Bedrock changed how it counts tool calls) with made-up prompts and a fresh random tag in each run, so old cache entries couldn't help. The capture and the live checks are scripts in the repo, so you can run them yourself: capture and live checks. I also ran LiteLLM's own proxy server against live Bedrock, called with the OpenAI client as shown above, and got the same numbers as the library. Token counts move by a few tokens from run to run because each run uses a new random tag. On October 12 I captured the same requests again with LiteLLM 1.105.0: findings 1 to 3 and the profile ARN case are unchanged, and the Sonnet 5.5 case is fixed. LiteLLM changes quickly, so check your own version with the callback above.

    Using Strands Agents or LangChain instead? Same tests: Strands Agents and LangChain.

    More on why caching breaks on Bedrock, with the bug reports behind each case: Five ways Claude prompt caching quietly breaks on Amazon Bedrock.

    Want a quick number for your own app? Paste three totals from CloudWatch into the cache checker. It runs in your browser, nothing to install.

    diff --git a/site/llms.txt b/site/llms.txt index bdf6796..5123dae 100644 --- a/site/llms.txt +++ b/site/llms.txt @@ -8,7 +8,7 @@ What it checks, each verified against live Amazon Bedrock (Claude Sonnet 4.6, us - Dynamic text (dates, times, IDs) before a cache point, which makes every request a cache miss. - A cached prefix shorter than the model's minimum (512 to 4,096 tokens depending on the model), which caches nothing without an error. -- More than 21 content blocks added between two cache points: Bedrock finds an earlier cache entry only up to 21 blocks back (21 added still hits, 22 always misses). Every tool call and tool result counts as a block. +- More than 21 lookback positions added between two cache points: Bedrock finds an earlier cache entry only up to 21 positions back (21 still hits, 22 always misses; re-measured October 30 2026). Each text block is one position, a group of tool calls or tool results side by side is one, and thinking blocks don't count. On October 7 2026 Bedrock still counted every tool call and result, so many parallel tool calls lost the cache; by October 28 they no longer did. - A cache point with nothing before it in its own message, system or tools list: Bedrock rejects the request ("There is nothing available to cache"). - A Converse cachePoint inside a toolResult's content: boto3 refuses it, and over plain HTTP Bedrock accepts the request but caches nothing. - Thinking or effort settings changed between requests: on Bedrock this throws away the whole cache, system prompt included. No effort setting behaves like "high". @@ -19,9 +19,9 @@ What it checks, each verified against live Amazon Bedrock (Claude Sonnet 4.6, us - [README](https://github.com/Haarris/cachecanary#readme): install, commands, every check, GitHub Action usage, limits - [Website](https://cachecanary.com/): overview, how prompt caching works, examples with real output -- [LiteLLM guide](https://cachecanary.com/litellm/): what LiteLLM 1.104.0 sends to Bedrock, where cache_control_injection_points lose the cache (role "user", parallel tool calls, dropped 1-hour lifetime), and a tested setup that keeps it -- [Strands Agents guide](https://cachecanary.com/strands/): what Strands Agents 1.58.1 sends to Bedrock: caching off by default, more than 10 parallel tool calls losing the conversation, the default 40-message window losing the cache every turn, inference profile ARNs needing strategy "anthropic", and lifetime settings Bedrock rejects, with a tested hook and conversation manager -- [LangChain guide](https://cachecanary.com/langchain/): what langchain-aws 1.8.1 and LangChain 1.4.3 send to Bedrock from create_agent: caching off until you add BedrockPromptCachingMiddleware, ChatBedrock losing the whole cache after 10 parallel tool calls (ChatBedrockConverse holds 21), ClearToolUsesEdit context editing rewriting the conversation on every call, and your own cache point turning off LangChain's extra one, with a tested fix for each +- [LiteLLM guide](https://cachecanary.com/litellm/): what LiteLLM 1.104.0 sends to Bedrock, where cache_control_injection_points lose the cache (role "user", a point on an assistant tool-call message, dropped 1-hour lifetime), and a tested setup that keeps it +- [Strands Agents guide](https://cachecanary.com/strands/): what Strands Agents 1.58.1 sends to Bedrock: caching off by default, the default 40-message window losing the cache every turn, inference profile ARNs needing strategy "anthropic", and lifetime settings Bedrock rejects, with a tested conversation manager; many parallel tool calls keep the cache since Bedrock's October 2026 change +- [LangChain guide](https://cachecanary.com/langchain/): what langchain-aws 1.8.1 and LangChain 1.4.3 send to Bedrock from create_agent: caching off until you add BedrockPromptCachingMiddleware, ClearToolUsesEdit context editing rewriting the conversation on every call, with a tested fix; many parallel tool calls keep the cache with ChatBedrockConverse and ChatBedrock since Bedrock's October 2026 change - [Langfuse guide](https://cachecanary.com/langfuse/): reading the real Bedrock cache hit rate from Langfuse traces, tested with Strands 1.61.1 and 1.53.0, LiteLLM 1.106.0 (langfuse callback and langfuse_otel) and langchain-aws 1.9.0 against Langfuse 4.64: Strands before 1.54 makes the cache look perfect (input 0 on cache reads, writes under cache_write_input_tokens which Langfuse does not recognise) and LiteLLM's langfuse callback counts cache writes twice; CacheCanary rebuilds the uncached input from the total and counts only generations - [Cache checker](https://cachecanary.com/check/): in-browser calculator, paste the CloudWatch sums of InputTokenCount (excludes cached tokens), CacheReadInputTokenCount and CacheWriteInputTokenCount for one model to get the hit rate, input cost at Bedrock list price and dollars lost to misses; same maths and prices as the CLI, sends nothing - [PyPI](https://pypi.org/project/cachecanary/): package and release history @@ -34,5 +34,5 @@ What it checks, each verified against live Amazon Bedrock (Claude Sonnet 4.6, us ## Optional -- [Live lookback measurement script](https://github.com/Haarris/cachecanary/blob/main/scripts/live_lookback_boundary.py) +- [Live lookback check (positions, October 30 2026)](https://github.com/Haarris/cachecanary/blob/main/scripts/live_lookback_positions.py) - [Live agreement check for the 0.3 rules](https://github.com/Haarris/cachecanary/blob/main/scripts/live_v03_checks.py) diff --git a/site/sitemap.xml b/site/sitemap.xml index 0225768..8e21de7 100644 --- a/site/sitemap.xml +++ b/site/sitemap.xml @@ -2,19 +2,19 @@ https://cachecanary.com/ - 2026-10-20 + 2026-10-30 https://cachecanary.com/litellm/ - 2026-10-07 + 2026-10-30 https://cachecanary.com/strands/ - 2026-10-07 + 2026-10-30 https://cachecanary.com/langchain/ - 2026-10-07 + 2026-10-30 https://cachecanary.com/langfuse/ @@ -22,6 +22,6 @@ https://cachecanary.com/check/ - 2026-10-20 + 2026-10-30 diff --git a/site/strands/index.html b/site/strands/index.html index db0d325..5cf448c 100644 --- a/site/strands/index.html +++ b/site/strands/index.html @@ -4,9 +4,9 @@ Strands Agents prompt caching on Amazon Bedrock: what actually gets cached - + - + @@ -27,10 +27,10 @@ "@context": "https://schema.org", "@type": "TechArticle", "headline": "Prompt caching with Strands Agents on Amazon Bedrock", - "description": "I captured the exact Bedrock requests Strands Agents 1.58.1 sends and tested them on live Bedrock. Caching is off by default, and wide tool calls and long chats lose the cache. Here is a setup that keeps it.", + "description": "I captured the exact Bedrock requests Strands Agents 1.58.1 sends and tested them on live Bedrock. Caching is off by default, and long chats lose the cache. Here is a setup that keeps it.", "image": "https://cachecanary.com/og-strands.png?v=1", "datePublished": "2026-10-07", - "dateModified": "2026-10-07", + "dateModified": "2026-10-30", "author": { "@type": "Person", "name": "Haris Farooq", @@ -103,9 +103,9 @@

    Prompt caching with Strands Agents on Amazon Bedrock

    -

    Strands Agents is AWS's open-source agent SDK, and Bedrock is its default model provider. I captured the exact requests Strands 1.58.1 sends to Bedrock while it runs its real agent loop, checked them with CacheCanary, and ran every case on live Bedrock (Claude Sonnet 4.6, us-west-2, October 2026).

    +

    Strands Agents is AWS's open-source agent SDK, and Bedrock is its default model provider. I captured the exact requests Strands 1.58.1 sends to Bedrock while it runs its real agent loop, checked them with CacheCanary, and ran every case on live Bedrock (Claude Sonnet 4.6, us-west-2, October 2026, re-checked October 30).

    -

    The short version: caching is off unless you turn it on. Once it's on, it works for simple calls, but two things that agents do all the time quietly throw the cache away: many tool calls at once, and long conversations.

    +

    The short version: caching is off unless you turn it on. Once it's on, it works well, including steps with many tool calls at once, but long conversations quietly throw the cache away.

    The setup I'd use

    For most agents, turn caching on and swap the conversation manager for one that trims old messages in bulk (finding 3 explains why):

    @@ -147,7 +147,6 @@

    The setup I'd use

    tools=tools, conversation_manager=TrimInChunks(), )
    -

    If your agent sometimes calls more than 10 tools at once, add the hook from finding 2 as well.

    What I found

    @@ -164,79 +163,33 @@

    1. Caching is off by default

    With strategy="auto", Strands puts one cache point after the system prompt and one at the end of the newest user message. After a round of tool calls, the newest user message is the one holding the tool results.

    -

    2. More than 10 tool calls at once lose the conversation

    -

    Bedrock only looks back 21 blocks from a cache point to find the previous cache entry, and every tool call and every tool result is a block. Auto mode keeps a single cache point in the conversation and moves it to the newest message, so a step that runs 11 tools at once adds 22 blocks and the previous entry is out of reach. The whole conversation is written to the cache again, at 1.25 times the normal price instead of a tenth of it. On live Bedrock, with a long customer history in the first message:

    +

    2. Many tool calls at once: fine since October 2026

    +

    Bedrock only looks back about 20 positions from a cache point to find the previous cache entry. When I first tested this on October 7 2026, it counted every tool call and every tool result as its own position, so a step that ran 11 tools at once lost the cached conversation and wrote it again at 1.25 times the normal price (issue #3348). By October 28 Bedrock counted a group of tool calls side by side as one position, and the same for their results, as Anthropic documents for its own API. The same requests on live Bedrock, with a long customer history in the first message:

    - + - +
    Next stepRead from cacheWritten again
    Next stepRead from cacheWritten
    10 tool calls at once6,9331,783
    11 tool calls at once2,221 (system prompt only)6,662
    11 tool calls at once6,929 (on October 7: 2,221, the system prompt only)1,958
    -

    This is issue #3348, still open. Auto mode removes any cache point you add to earlier messages, so the fix is to leave cache_config unset and place the points yourself with a hook. This one uses the 4 points Bedrock allows on the system prompt, your newest message, the first result of the newest round of tool calls, and the end of the conversation:

    -
    from strands.hooks import BeforeModelCallEvent, HookProvider, HookRegistry
    -
    -
    -class BedrockCachePoints(HookProvider):
    -    """Place cache points in the messages before every model call."""
    -
    -    def register_hooks(self, registry: HookRegistry, **kwargs) -> None:
    -        registry.add_callback(BeforeModelCallEvent, self.place)
    -
    -    def place(self, event: BeforeModelCallEvent) -> None:
    -        messages = event.agent.messages
    -        for message in messages:  # drop the points from the previous call
    -            message["content"] = [b for b in message["content"]
    -                                  if "cachePoint" not in b]
    -        typed = [i for i, m in enumerate(messages) if m["role"] == "user"
    -                 and not any("toolResult" in b for b in m["content"])]
    -        if typed:  # the newest message a person typed
    -            newest = messages[typed[-1]]["content"]
    -            self._add(newest, len(newest))
    -        last = messages[-1]["content"] if messages else []
    -        if messages and (not typed or typed[-1] != len(messages) - 1):
    -            if any("toolResult" in b for b in last) and len(last) > 1:
    -                self._add(last, 1)  # after the first result of the round
    -            self._add(last, len(last))  # the end of the conversation
    -
    -    @staticmethod
    -    def _add(content, index):
    -        # Bedrock rejects a cache point right after a non-PDF document,
    -        # so step back past those.
    -        def other_document(block):
    -            return block.get("document", {}).get("format", "pdf") != "pdf"
    -
    -        while index > 0 and other_document(content[index - 1]):
    -            index -= 1
    -        if index > 0 and "cachePoint" not in content[index - 1]:
    -            content.insert(index, {"cachePoint": {"type": "default"}})
    -
    -
    -agent = Agent(
    -    # no cache_config: the hook places the message cache points
    -    model=BedrockModel(model_id="us.anthropic.claude-sonnet-4-6"),
    -    system_prompt=[{"text": SYSTEM_PROMPT}, {"cachePoint": {"type": "default"}}],
    -    tools=tools,
    -    hooks=[BedrockCachePoints()],
    -    conversation_manager=TrimInChunks(),
    -)
    -

    On live Bedrock, through Strands' own agent loop, each step read almost the whole previous request from the cache:

    +

    Through Strands' own agent loop in auto mode, each step read almost the whole previous request from the cache:

    - - - - - + + + + + +
    StepRead from cacheSize of the previous request
    12 tool calls at once6,9336,936
    1 tool call9,0669,067
    20 tool calls at once9,2459,246
    A follow-up question from the user12,77512,776
    3 tool calls at once12,79112,794
    12 tool calls at once6,9356,938
    1 tool call9,0689,069
    20 tool calls at once9,2479,248
    A follow-up question from the user12,77712,778
    3 tool calls at once12,79312,796
    40 tool calls at once13,35113,352

    Tokens. The last few tokens of each request are always new, so the cache can't hold them.

    -

    It holds for up to 20 tool calls at once. At 21, the round before it is written again, but the point on your newest message still holds everything up to there (12,791 of 13,350 tokens read). I also ran it with a live model choosing its own tools: it called 6 at once, then 2, and the second question read 7,108 tokens from the cache and wrote 485. It works with Strands' session managers (they don't save the cache points, so a restored session starts clean) and with extended thinking turned on.

    +

    So auto mode is enough, and the hook I used to recommend here isn't needed any more. With a live model choosing its own tools, it called 6 at once, then 2, and the second question read 7,115 tokens from the cache and wrote 496. A step can still lose the cache if it adds more than about 20 other blocks at once, like 22 or more separate text blocks; cachecanary diff counts positions the way Bedrock does now.

    3. Long conversations lose the cache on every turn

    By default Strands keeps the last 40 messages. Once a conversation is longer, it drops the oldest messages at the end of every call to the agent, so the start of the conversation changes every turn and nothing after the system prompt can be read from the cache. On live Bedrock, a chat with full-length answers, from the 22nd question on:

    @@ -249,7 +202,7 @@

    3. Long conversations lose the cache on every turn

    -

    TrimInChunks (in the setup above) lets the history grow to 40 messages and then trims it back to 20 in one go, so the start of the conversation changes about once every 10 turns instead of every turn. Over 45 questions, the default manager changed it on 24 calls and TrimInChunks on 3. It keeps 20 to 40 messages instead of always 40, so the model sees a little less history; raise both numbers if you need more. Without tools or with them, it works the same, and it works with the hook from finding 2 and with per_turn and pin_first. The last method matters if you save sessions: Strands checks the manager's class name when it restores one, and without it, a session saved with the default manager fails to load ("Invalid conversation manager state").

    +

    TrimInChunks (in the setup above) lets the history grow to 40 messages and then trims it back to 20 in one go, so the start of the conversation changes about once every 10 turns instead of every turn. Over 45 questions, the default manager changed it on 24 calls and TrimInChunks on 3. It keeps 20 to 40 messages instead of always 40, so the model sees a little less history; raise both numbers if you need more. Without tools or with them, it works the same, and it works with per_turn and pin_first. The last method matters if you save sessions: Strands checks the manager's class name when it restores one, and without it, a session saved with the default manager fails to load ("Invalid conversation manager state").

    4. Application inference profiles need strategy="anthropic"

    Auto mode only caches when the model ID contains "claude" or "anthropic". An application inference profile ARN contains neither, so Strands logs a warning ("cache_config is enabled but this model does not support automatic caching") and sends no cache points. The maintainers chose this on purpose. If the profile points to a Claude model, say so:

    @@ -296,19 +249,16 @@

    Check your own agent

    ) save_requests(model)

    Run a conversation, then compare two requests in a row with cachecanary diff, or check one with cachecanary lint. These are real outputs on requests Strands built:

    -
    $ cachecanary diff agent-turn1.json agent-turn2.json --model us.anthropic.claude-sonnet-4-6
    -lookback-exceeded: 22 blocks were added between the previous checkpoint and the next one. Bedrock only finds an earlier cache entry up to 21 blocks back (21 added still hits, 22 misses), so this call writes the cache again. Add a checkpoint in between (common after many parallel tool calls). (messages[2].content[10])
    -
    -$ cachecanary diff chat-turn20.json chat-turn21.json --model us.anthropic.claude-sonnet-4-6
    +
    $ cachecanary diff chat-turn20.json chat-turn21.json --model us.anthropic.claude-sonnet-4-6
     history-changed: Earlier conversation history was edited (not just appended). (messages[0].content[0])
     
     $ cachecanary lint ttl-order.json --model us.anthropic.claude-sonnet-4-6
     [error] ttl-order: A 1h checkpoint appears after a 5m checkpoint. Longer TTLs must come first.
    -

    In plain words: the first pair is the 11-tool-call step from finding 2. The second is the default conversation manager trimming the first messages (finding 3). The third is the lifetime setting Bedrock rejects (finding 5).

    +

    In plain words: the first pair is the default conversation manager trimming the first messages (finding 3). The second is the lifetime setting Bedrock rejects (finding 5).

    Keep a few saved requests in your repo and run cachecanary lint on them in CI, and a Strands upgrade that changes your cache points shows up in a pull request instead of on the bill. Setup is on the home page.

    How I tested this

    -

    Strands Agents 1.58.1 in a clean Python environment. To capture requests, I pointed Strands at a small local server standing in for Bedrock that speaks its streaming format, so Strands ran its real agent loop (tool calls, streaming, conversation trimming) and every request body was saved. The settings cases ran fully live on Bedrock (Claude Sonnet 4.6, us-west-2, October 7 2026). A live model won't call exactly 11 tools on request, so for the agent-loop cases I sent the exact requests Strands built, in order, to live Bedrock; caching depends only on the request. Every case uses made-up prompts with a fresh random tag, so old cache entries couldn't help. The capture and the live checks are scripts in the repo, so you can run them yourself: capture and live checks. Token counts move by a few tokens from run to run because each run uses a new random tag. Strands changes quickly, so check your own version with the snippet above.

    +

    Strands Agents 1.58.1 in a clean Python environment. To capture requests, I pointed Strands at a small local server standing in for Bedrock that speaks its streaming format, so Strands ran its real agent loop (tool calls, streaming, conversation trimming) and every request body was saved. The settings cases ran fully live on Bedrock (Claude Sonnet 4.6, us-west-2, October 7 2026), and I re-ran every case on October 30 2026 after Bedrock changed how it counts tool calls. A live model won't call exactly 11 tools on request, so for the agent-loop cases I sent the exact requests Strands built, in order, to live Bedrock; caching depends only on the request. Every case uses made-up prompts with a fresh random tag, so old cache entries couldn't help. The capture and the live checks are scripts in the repo, so you can run them yourself: capture and live checks. Token counts move by a few tokens from run to run because each run uses a new random tag. Strands changes quickly, so check your own version with the snippet above.

    Same tests for other frameworks: LiteLLM and LangChain. More on why caching breaks on Bedrock: Five ways Claude prompt caching quietly breaks on Amazon Bedrock.

    Want a quick number for your own app? Paste three totals from CloudWatch into the cache checker. It runs in your browser, nothing to install.

    Using CacheCanary? I'd love to hear how it's going, good or bad, and whether you'd want the dashboard. Email me at hello@cachecanary.com. I read every message.

    diff --git a/src/cachecanary/__init__.py b/src/cachecanary/__init__.py index 1934f55..9ec0d26 100644 --- a/src/cachecanary/__init__.py +++ b/src/cachecanary/__init__.py @@ -1,3 +1,3 @@ """CacheCanary: know the moment your Claude prompt cache breaks on Amazon Bedrock.""" -__version__ = "0.4.0" +__version__ = "0.4.1" diff --git a/src/cachecanary/diff.py b/src/cachecanary/diff.py index 613ce4b..e70f1f1 100644 --- a/src/cachecanary/diff.py +++ b/src/cachecanary/diff.py @@ -7,8 +7,8 @@ import json from dataclasses import dataclass -from cachecanary.models import MAX_BLOCKS_ADDED -from cachecanary.request import NormalizedRequest +from cachecanary.models import MAX_LOOKBACK_POSITIONS +from cachecanary.request import Block, NormalizedRequest @dataclass @@ -87,14 +87,17 @@ def explain(a: NormalizedRequest, b: NormalizedRequest) -> list[MissReason]: # B reads A's entry only if one of B's checkpoints sits at A's last checkpoint or within reach after it. later = [i for i in b.checkpoint_indexes if i >= a_last] - if later and later[0] - a_last > MAX_BLOCKS_ADDED: - added = later[0] - a_last + added = lookback_positions(b.blocks, a_last, later[0]) if later else 0 + if added > MAX_LOOKBACK_POSITIONS: reasons.append(MissReason( "lookback-exceeded", - f"{added} blocks were added between the previous checkpoint and the next one. Bedrock only finds an " - f"earlier cache entry up to {MAX_BLOCKS_ADDED} blocks back ({MAX_BLOCKS_ADDED} added still hits, " - f"{MAX_BLOCKS_ADDED + 1} misses), so this call writes the cache again. Add a checkpoint in between " - "(common after many parallel tool calls).", + f"{added} lookback positions were added between the previous checkpoint and the next one " + f"({later[0] - a_last} blocks). Bedrock only finds an earlier cache entry up to " + f"{MAX_LOOKBACK_POSITIONS} positions back ({MAX_LOOKBACK_POSITIONS} added still hits, " + f"{MAX_LOOKBACK_POSITIONS + 1} misses), so this call writes the cache again. Each text block is one " + "position, a group of tool calls or tool results side by side is one (the block with the cache point " + "always counts on its own), and thinking blocks don't count. Add a checkpoint in between (common in long chains of single tool calls or many separate " + "text blocks).", b.blocks[later[0]].location, )) @@ -123,6 +126,31 @@ def explain(a: NormalizedRequest, b: NormalizedRequest) -> list[MissReason]: STUCK_MIN_ADDED = 2 +_RUN_KINDS = ("tool_use", "tool_result") + + +def lookback_positions(blocks: list[Block], start: int, end: int) -> int: + """Positions Bedrock counts from the checkpoint at blocks[start] back-to-front to blocks[end]. + + Counts blocks start+1 .. end. Measured on live Bedrock (Oct 30 2026): a run of consecutive tool-use + blocks in one message is one position, and so is a run of tool results; thinking blocks are not + counted; every other block is one. The checkpoint block itself is always its own position, so a run + of tool results that ends on the checkpoint counts as two. + """ + positions = 0 + prev: Block | None = None + for i in range(start + 1, end + 1): + block = blocks[i] + if block.kind == "reasoning" and i != end: + continue + same_run = (prev is not None and block.kind in _RUN_KINDS and block.kind == prev.kind + and block.message is not None and block.message == prev.message) + if not same_run or i == end: + positions += 1 + prev = block + return positions + + def _thinking_label(key: str | None) -> str: if key is None: return "off" diff --git a/src/cachecanary/models.py b/src/cachecanary/models.py index ee17748..aa9b0cf 100644 --- a/src/cachecanary/models.py +++ b/src/cachecanary/models.py @@ -46,12 +46,14 @@ class CacheLimits: "claude-3-5-haiku": CacheLimits(2048, 4, None, legacy=True), } -# Bedrock's automatic prefix check only looks back ~20 content blocks from a checkpoint. -LOOKBACK_BLOCKS = 20 -# Measured on live Bedrock (Oct 2026, Sonnet 4.6, scripts/live_lookback_boundary.py, two runs per size): -# a checkpoint still finds an earlier cache entry when at most 21 blocks were added after it (20 in -# between plus the checkpoint's own block); 22 or more always missed. pydantic-ai#9404 saw the same. -MAX_BLOCKS_ADDED = LOOKBACK_BLOCKS + 1 +# How far back a checkpoint finds an earlier cache entry, in lookback positions (see +# diff.lookback_positions for how blocks turn into positions). Measured on live Bedrock, us-west-2, +# Oct 30 2026 (scripts/live_lookback_positions.py): Sonnet 4.6 two runs per shape (Converse and +# InvokeModel, plus real thinking turns), Sonnet 5.5 one run (no thinking shape). 21 positions added +# still hit, 22 always missed. On Oct 7 2026 Bedrock still counted every block, so many parallel tool +# calls lost the cache; by Oct 28 it counted a run of tool calls (or tool results) once, as Anthropic's +# own API documents: https://platform.claude.com/docs/en/build-with-claude/prompt-caching +MAX_LOOKBACK_POSITIONS = 21 def lookup(model_id: str | None) -> tuple[str | None, CacheLimits | None]: diff --git a/src/cachecanary/request.py b/src/cachecanary/request.py index 3f07256..f7181cd 100644 --- a/src/cachecanary/request.py +++ b/src/cachecanary/request.py @@ -21,6 +21,10 @@ class Block: content: str # canonical JSON of the block without cache markers checkpoint: bool = False ttl: str | None = None # "5m" | "1h" when checkpoint + # What the block is, for counting Bedrock's lookback positions: "tool_use" | "tool_result" | + # "reasoning" | "other". message is the index of the message holding it (None outside messages). + kind: str = "other" + message: int | None = None @dataclass @@ -53,6 +57,26 @@ def checkpoint_markers(self) -> int: return len(self.checkpoint_indexes) + len(self.duplicate_checkpoints) + len(self.orphan_checkpoints) +# Converse content keys and InvokeModel block types that matter for lookback counting. +_CONVERSE_KINDS = {"toolUse": "tool_use", "toolResult": "tool_result", "reasoningContent": "reasoning"} +_INVOKE_KINDS = {"tool_use": "tool_use", "tool_result": "tool_result", + "thinking": "reasoning", "redacted_thinking": "reasoning"} + + +def _converse_kind(item) -> str: + if isinstance(item, dict): + for key, kind in _CONVERSE_KINDS.items(): + if key in item: + return kind + return "other" + + +def _invoke_kind(item) -> str: + if isinstance(item, dict): + return _INVOKE_KINDS.get(item.get("type"), "other") + return "other" + + def _canon(obj) -> str: return json.dumps(obj, sort_keys=True, ensure_ascii=False, separators=(",", ":")) @@ -143,7 +167,8 @@ def _normalize_converse(payload: dict, req: NormalizedRequest) -> None: if isinstance(part, dict) and "cachePoint" in part: req.nested_checkpoints.append(f"{loc}.toolResult.content[{k}]") body = item if isinstance(item, dict) else {"value": item} - req.blocks.append(Block("messages", loc, _canon({"role": msg.get("role"), **body}))) + req.blocks.append(Block("messages", loc, _canon({"role": msg.get("role"), **body}), + kind=_converse_kind(item), message=m)) extra = payload.get("additionalModelRequestFields") if isinstance(extra, dict): req.thinking = _thinking_key(extra.get("thinking")) @@ -189,6 +214,7 @@ def _normalize_invoke(payload: dict, req: NormalizedRequest) -> None: if inner and not block.checkpoint: block.checkpoint, block.ttl = True, _ttl(inner[-1]["cache_control"]) block.content = _canon({"role": msg.get("role"), "block": json.loads(block.content)}) + block.kind, block.message = _invoke_kind(item), m req.blocks.append(block) req.thinking = _thinking_key(payload.get("thinking")) req.effort = _effort(payload.get("output_config")) diff --git a/tests/test_cachecanary.py b/tests/test_cachecanary.py index 2b4c3c4..4394096 100644 --- a/tests/test_cachecanary.py +++ b/tests/test_cachecanary.py @@ -172,19 +172,26 @@ def test_model_change(): assert diff.explain(a, b)[0].code == "model-changed" -def test_lookback_exceeded_after_many_tool_blocks(): +def test_lookback_exceeded_after_a_long_chain_of_single_tool_calls(): base = [{"role": "user", "content": [{"text": "start"}, {"cachePoint": {"type": "default"}}]}] a_req = converse_request(messages=base) + b_msgs = [{"role": "user", "content": [{"text": "start"}]}] + for i in range(11): # 11 rounds of one call + one result = 22 positions + b_msgs.append({"role": "assistant", "content": [{"toolUse": {"toolUseId": f"t{i}", "name": "x", "input": {}}}]}) + b_msgs.append({"role": "user", "content": [{"toolResult": {"toolUseId": f"t{i}", "content": [{"text": "ok"}]}}]}) + b_msgs[-1]["content"].append({"cachePoint": {"type": "default"}}) + reasons = diff.explain(normalize(a_req), normalize(converse_request(messages=b_msgs))) + assert "lookback-exceeded" in {r.code for r in reasons} + + +def test_many_parallel_tool_calls_no_longer_exceed_the_lookback(): + base = [{"role": "user", "content": [{"text": "start"}, {"cachePoint": {"type": "default"}}]}] calls = [{"toolUse": {"toolUseId": f"t{i}", "name": "x", "input": {}}} for i in range(12)] results = [{"toolResult": {"toolUseId": f"t{i}", "content": [{"text": "ok"}]}} for i in range(12)] - b_msgs = [ - {"role": "user", "content": [{"text": "start"}]}, - {"role": "assistant", "content": calls}, - {"role": "user", "content": results + [{"cachePoint": {"type": "default"}}]}, - ] - b_req = converse_request(messages=b_msgs) - reasons = diff.explain(normalize(a_req), normalize(b_req)) - assert "lookback-exceeded" in {r.code for r in reasons} + b_msgs = [{"role": "user", "content": [{"text": "start"}]}, {"role": "assistant", "content": calls}, + {"role": "user", "content": results + [{"cachePoint": {"type": "default"}}]}] + reasons = diff.explain(normalize(converse_request(messages=base)), normalize(converse_request(messages=b_msgs))) + assert "lookback-exceeded" not in {r.code for r in reasons} def test_history_edit_detected(): diff --git a/tests/test_edge_cases.py b/tests/test_edge_cases.py index dc03c8c..febf8ef 100644 --- a/tests/test_edge_cases.py +++ b/tests/test_edge_cases.py @@ -409,41 +409,204 @@ def test_live_calibration_case_is_not_a_false_error(): assert "prefix-too-short" in haiku -# --- lookback boundary, measured on live Bedrock (scripts/live_lookback_boundary.py) ---------- - -def _lookback_pair(n_calls, with_text=False, b_extra_cp_after=None): - """A caches one user turn; B adds an assistant turn with n tool calls and the results. - b_extra_cp_after puts an extra checkpoint after that many tool calls in the assistant turn.""" - a = {"messages": [{"role": "user", "content": [{"text": "start " * 50}, CP]}]} - calls = [{"toolUse": {"toolUseId": f"t{i}", "name": "x", "input": {}}} for i in range(n_calls)] - if b_extra_cp_after is not None: - calls.insert(b_extra_cp_after, CP) - results = [{"toolResult": {"toolUseId": f"t{i}", "content": [{"text": "ok"}]}} for i in range(n_calls)] - b = {"messages": [{"role": "user", "content": [{"text": "start " * 50}]}, - {"role": "assistant", "content": ([{"text": "checking"}] if with_text else []) + calls}, - {"role": "user", "content": results + [CP]}]} - return normalize(a, "us.anthropic.claude-sonnet-4-6"), normalize(b, "us.anthropic.claude-sonnet-4-6") - - -@pytest.mark.parametrize("n_calls,with_text,added,miss", [ - (9, True, 19, False), (10, False, 20, False), (10, True, 21, False), # live: hit - (11, False, 22, True), (11, True, 23, True), (12, False, 24, True), # live: miss -]) -def test_lookback_boundary_matches_live_bedrock(n_calls, with_text, added, miss): - a, b = _lookback_pair(n_calls, with_text) - assert b.checkpoint_indexes[-1] - a.checkpoint_indexes[-1] == added +# --- lookback boundary, measured on live Bedrock Oct 30 2026 (scripts/live_lookback_positions.py) ---- +# Shapes are lists of (role, [block kinds]); "tu"/"tr"/"text"/"think". Expected hit/miss is what live +# Bedrock did (Sonnet 4.6, two runs each; most shapes also on InvokeModel and on Sonnet 5.5, one run). + +def _render_converse(kind, i): + if kind == "text": + return {"text": f"note {i}"} + if kind == "tu": + return {"toolUse": {"toolUseId": f"t{i}", "name": "x", "input": {}}} + if kind == "tr": + return {"toolResult": {"toolUseId": f"t{i}", "content": [{"text": "ok"}]}} + return {"reasoningContent": {"reasoningText": {"text": "thinking", "signature": "sig"}}} + + +def _render_invoke(kind, i): + if kind == "text": + return {"type": "text", "text": f"note {i}"} + if kind == "tu": + return {"type": "tool_use", "id": f"toolu_{i}", "name": "x", "input": {}} + if kind == "tr": + return {"type": "tool_result", "tool_use_id": f"toolu_{i}", "content": "ok"} + return {"type": "thinking", "thinking": "thinking", "signature": "sig"} + + +def _lookback_shape_pair(shape, api="converse"): + """A caches one user turn; B is the same turn plus `shape`, with a checkpoint on B's last block.""" + model = "us.anthropic.claude-sonnet-4-6" + if api == "converse": + a = {"messages": [{"role": "user", "content": [{"text": "start " * 50}, CP]}]} + msgs = [{"role": "user", "content": [{"text": "start " * 50}]}] + for role, kinds in shape: + msgs.append({"role": role, "content": [_render_converse(k, i) for i, k in enumerate(kinds)]}) + msgs[-1]["content"].append(CP) + else: + eph = {"type": "ephemeral"} + first = {"type": "text", "text": "start " * 50} + a = {"anthropic_version": "bedrock-2023-05-31", "messages": [{"role": "user", "content": [{**first, "cache_control": eph}]}]} + msgs = [{"role": "user", "content": [first]}] + for role, kinds in shape: + msgs.append({"role": role, "content": [_render_invoke(k, i) for i, k in enumerate(kinds)]}) + msgs[-1]["content"][-1]["cache_control"] = eph + b = {"anthropic_version": "bedrock-2023-05-31", "messages": msgs} + return normalize(a, model), normalize(b, model) + return normalize(a, model), normalize({"messages": msgs}, model) + + +def _text(n): + return [("assistant", ["text"]), ("user", ["text"] * n)] + + +def _parallel(n, tail=0, lead=0): + return [("assistant", ["text"] * lead + ["tu"] * n), ("user", ["tr"] * n + ["text"] * tail)] + + +def _sequential(rounds): + return [turn for _ in range(rounds) for turn in (("assistant", ["tu"]), ("user", ["tr"]))] + + +def _mixed(m): + return [("assistant", ["text", "tu"] * m), ("user", ["tr"] * m)] + + +LIVE_LOOKBACK = [ + # name, shape, positions, live result (Sonnet 4.6 Converse, two runs; most also on InvokeModel and Sonnet 5.5) + ("20 text blocks", _text(20), 21, "hit"), + ("21 text blocks", _text(21), 22, "miss"), + ("23 text blocks", _text(23), 24, "miss"), + ("10 sequential calls", _sequential(10), 20, "hit"), + ("11 sequential calls", _sequential(11), 22, "miss"), + ("11 parallel calls", _parallel(11), 3, "hit"), + ("40 parallel calls", _parallel(40), 3, "hit"), + ("5 parallel + 19 text", _parallel(5, tail=19), 21, "hit"), + ("5 parallel + 20 text", _parallel(5, tail=20), 22, "miss"), + ("text, 11 parallel, 18 text", _parallel(11, tail=18, lead=1), 21, "hit"), + ("text, 11 parallel, 19 text", _parallel(11, tail=19, lead=1), 22, "miss"), + ("text before each of 9 calls", _mixed(9), 20, "hit"), + ("text before each of 10 calls", _mixed(10), 22, "miss"), # results run ends on the checkpoint: 2 + ("18 text, 10 parallel", _parallel(10, lead=18), 21, "hit"), + ("19 text, 10 parallel", _parallel(10, lead=19), 22, "miss"), + ("18 text, 2 parallel", _parallel(2, lead=18), 21, "hit"), + ("19 text, 2 parallel", _parallel(2, lead=19), 22, "miss"), + ("thinking, 5 parallel, 19 text", [("assistant", ["think"] + ["tu"] * 5), ("user", ["tr"] * 5 + ["text"] * 19)], 21, "hit"), + ("thinking, 5 parallel, 20 text", [("assistant", ["think"] + ["tu"] * 5), ("user", ["tr"] * 5 + ["text"] * 20)], 22, "miss"), +] + + +@pytest.mark.parametrize("api", ["converse", "invoke"]) +@pytest.mark.parametrize("name,shape,positions,live", LIVE_LOOKBACK, ids=[c[0] for c in LIVE_LOOKBACK]) +def test_lookback_matches_live_bedrock(api, name, shape, positions, live): + a, b = _lookback_shape_pair(shape, api) + assert diff.lookback_positions(b.blocks, a.checkpoint_indexes[-1], b.checkpoint_indexes[-1]) == positions reasons = {r.code: r for r in diff.explain(a, b)} - assert ("lookback-exceeded" in reasons) is miss - if miss: - assert reasons["lookback-exceeded"].message.startswith(f"{added} blocks were added") + assert ("lookback-exceeded" in reasons) is (live == "miss") + if live == "miss": + assert reasons["lookback-exceeded"].message.startswith(f"{positions} lookback positions were added") + + +def test_lookback_message_names_blocks_and_the_counting_rule(): + a, b = _lookback_shape_pair(_sequential(11)) + msg = {r.code: r for r in diff.explain(a, b)}["lookback-exceeded"].message + assert "(22 blocks)" in msg and "21 added still hits, 22 misses" in msg + assert "thinking blocks don't count" in msg and "parallel tool calls" not in msg def test_extra_checkpoint_within_reach_avoids_the_miss(): - a, b = _lookback_pair(15, b_extra_cp_after=5) # a Converse cachePoint marks the block before it: 5 after A's + # 30 text blocks would be 31 positions; a checkpoint after the 10th is 11 positions from A's. + shape = [("assistant", ["text"]), ("user", ["text"] * 30)] + a, b = _lookback_shape_pair(shape) + raw = {"messages": [{"role": "user", "content": [{"text": "start " * 50}]}, {"role": "assistant", "content": [{"text": "note 0"}]}, + {"role": "user", "content": [{"text": f"note {i}"} for i in range(10)] + [CP] + + [{"text": f"note {i}"} for i in range(10, 30)] + [CP]}]} + b = normalize(raw, "us.anthropic.claude-sonnet-4-6") assert "lookback-exceeded" not in {r.code for r in diff.explain(a, b)} def test_extra_checkpoint_out_of_reach_does_not_help(): - a, b = _lookback_pair(30, b_extra_cp_after=25) # extra point 25 blocks after A's: also too far + a, _ = _lookback_shape_pair(_text(1)) + raw = {"messages": [{"role": "user", "content": [{"text": "start " * 50}]}, {"role": "assistant", "content": [{"text": "note 0"}]}, + {"role": "user", "content": [{"text": f"note {i}"} for i in range(24)] + [CP] + + [{"text": f"note {i}"} for i in range(24, 30)] + [CP]}]} + b = normalize(raw, "us.anthropic.claude-sonnet-4-6") reasons = {r.code: r for r in diff.explain(a, b)} - assert reasons["lookback-exceeded"].message.startswith("25 blocks were added") + assert reasons["lookback-exceeded"].message.startswith("25 lookback positions were added") + + +def _positions(messages, model="us.anthropic.claude-sonnet-4-6"): + blocks = normalize({"messages": messages}, model).blocks + cps = [i for i, b in enumerate(blocks) if b.checkpoint] + return diff.lookback_positions(blocks, cps[0], cps[-1]) + + +def test_tool_runs_split_by_text_inside_one_message(): + # tu run (1) + text (1) + tu (1) + tr run (1) + text checkpoint (1) + assert _positions([ + {"role": "user", "content": [{"text": "x"}, CP]}, + {"role": "assistant", "content": [_render_converse("tu", 0), _render_converse("tu", 1), {"text": "more"}, _render_converse("tu", 2)]}, + {"role": "user", "content": [_render_converse("tr", 0), _render_converse("tr", 1), _render_converse("tr", 2), {"text": "end"}, CP]}, + ]) == 5 + + +def test_tool_result_runs_do_not_merge_across_messages(): + # [tr, tr] then [tr, tr + checkpoint]: run (1) + rest of the second run (1) + checkpoint (1) = 3, not 2 + assert _positions([ + {"role": "user", "content": [{"text": "x"}, CP]}, + {"role": "user", "content": [_render_converse("tr", 0), _render_converse("tr", 1)]}, + {"role": "user", "content": [_render_converse("tr", 2), _render_converse("tr", 3), CP]}, + ]) == 3 + + +def test_lookback_edge_cases_follow_the_rule(): + tu, tr, think = (lambda i: _render_converse("tu", i)), (lambda i: _render_converse("tr", i)), _render_converse("think", 0) + start = {"role": "user", "content": [{"text": "x"}, CP]} + # A checkpoint ending a tool-use run counts on its own, like one ending a tool-result run. + assert _positions([start, {"role": "assistant", "content": [tu(0), tu(1), tu(2), CP]}]) == 2 + # A checkpoint on a thinking block still counts as its position. + assert _positions([start, {"role": "assistant", "content": [{"text": "a"}, think, CP]}]) == 2 + # Thinking between two tool calls doesn't break their run (not measured live; thinking is skipped). + assert _positions([start, {"role": "assistant", "content": [tu(0), think, tu(1)]}, + {"role": "user", "content": [tr(0), tr(1), {"text": "end"}, CP]}]) == 3 + # The previous checkpoint inside a run: the rest of that run starts a new position. + blocks = normalize({"messages": [{"role": "user", "content": [tr(0), CP, tr(1), tr(2), {"text": "e"}, CP]}]}, + "us.anthropic.claude-sonnet-4-6").blocks + assert diff.lookback_positions(blocks, 0, len(blocks) - 1) == 2 + # Nothing added. + assert diff.lookback_positions(blocks, 0, 0) == 0 + # InvokeModel: cache_control inside the last tool_result's content makes it the checkpoint; a run of 2 ends there. + inv = normalize({"anthropic_version": "bedrock-2023-05-31", "messages": [ + {"role": "user", "content": [{"type": "text", "text": "x", "cache_control": {"type": "ephemeral"}}]}, + {"role": "assistant", "content": [_render_invoke("tu", 0), _render_invoke("tu", 1)]}, + {"role": "user", "content": [_render_invoke("tr", 0), + {"type": "tool_result", "tool_use_id": "toolu_1", + "content": [{"type": "text", "text": "ok", "cache_control": {"type": "ephemeral"}}]}]}]}, + "us.anthropic.claude-sonnet-4-6") + assert inv.blocks[-1].checkpoint and inv.blocks[-1].kind == "tool_result" + assert diff.lookback_positions(inv.blocks, 0, len(inv.blocks) - 1) == 3 + + +@pytest.mark.parametrize("fixture", ["strands/agent-turn1.json strands/agent-turn2.json", + "litellm/agent-turn1.json litellm/agent-turn2.json", + "langchain/invoke-turn1.json langchain/invoke-turn2.json"]) +def test_real_framework_wide_steps_no_longer_exceed_the_lookback(fixture): + """Requests Strands, LiteLLM and LangChain really built for 11-12 parallel tool calls; these lost the cache + on Oct 7 2026 and kept it on live Bedrock on Oct 30.""" + import pathlib + root = pathlib.Path(__file__).parent / "fixtures" + a_path, b_path = fixture.split() + a = normalize(json.loads((root / a_path).read_text()), "us.anthropic.claude-sonnet-4-6") + b = normalize(json.loads((root / b_path).read_text()), "us.anthropic.claude-sonnet-4-6") + assert "lookback-exceeded" not in {r.code for r in diff.explain(a, b)} + + +def test_block_kinds_are_tagged_for_both_apis(): + conv = normalize({"messages": [{"role": "assistant", "content": [_render_converse(k, 0) for k in ("think", "text", "tu")]}, + {"role": "user", "content": [_render_converse("tr", 0)]}]}) + assert [b.kind for b in conv.blocks] == ["reasoning", "other", "tool_use", "tool_result"] + assert [b.message for b in conv.blocks] == [0, 0, 0, 1] + inv = normalize({"anthropic_version": "bedrock-2023-05-31", "messages": [ + {"role": "assistant", "content": [_render_invoke("think", 0), {"type": "redacted_thinking", "data": "x"}, _render_invoke("tu", 0)]}, + {"role": "user", "content": [_render_invoke("tr", 0), _render_invoke("text", 1)]}]}) + assert [b.kind for b in inv.blocks] == ["reasoning", "reasoning", "tool_use", "tool_result", "other"] + assert normalize({"system": [{"text": "s"}]}).blocks[0].kind == "other" diff --git a/tests/test_site.py b/tests/test_site.py index f0ee900..d4de50c 100644 --- a/tests/test_site.py +++ b/tests/test_site.py @@ -116,8 +116,8 @@ def test_llms_txt_follows_the_format_and_links_resolve_locally(): links = re.findall(r"\]\((https://[^)]+)\)", text) assert "https://cachecanary.com/" in links and "https://github.com/Haarris/cachecanary#readme" in links # Facts that must stay in step with the code. - from cachecanary.models import MAX_BLOCKS_ADDED - assert f"{MAX_BLOCKS_ADDED} added still hits, {MAX_BLOCKS_ADDED + 1} always misses" in text + from cachecanary.models import MAX_LOOKBACK_POSITIONS + assert f"{MAX_LOOKBACK_POSITIONS} still hits, {MAX_LOOKBACK_POSITIONS + 1} always misses" in text def test_litellm_page_basics_and_links(): @@ -148,7 +148,7 @@ def test_litellm_page_outputs_match_real_output(capsys, monkeypatch): monkeypatch.chdir(SITE.parent / "tests" / "fixtures" / "litellm") page = html.unescape(re.sub(r"<[^>]+>", "", LITELLM)) cmds = re.findall(r"\$ (cachecanary diff [^\n]+)", page) - assert len(cmds) == 3 + assert len(cmds) == 2 for cmd in cmds: args = cmd.split()[1:] cli.main(args) @@ -158,52 +158,17 @@ def test_litellm_page_outputs_match_real_output(capsys, monkeypatch): assert line in page, f"{cmd}: {line}" -def _page_helper(): +def test_litellm_page_setup_is_what_the_live_script_ran(): + """The setup on the page is the two points the live checks used (system prompt + last message).""" + import ast import html - code = html.unescape(re.search(r'
    (def cache_points.*?)\n\n\nresponse', LITELLM, re.S).group(1))
    -    ns = {}
    -    exec(code, ns)
    -    return ns["cache_points"]
    -
    -
    -def _calls(n):
    -    return {"role": "assistant", "content": None,
    -            "tool_calls": [{"id": f"t{i}", "type": "function", "function": {"name": "f", "arguments": "{}"}} for i in range(n)]}
    -
    -
    -def _results(n):
    -    return [{"role": "tool", "tool_call_id": f"t{i}", "content": "ok"} for i in range(n)]
    -
    -
    -def test_litellm_page_helper_places_points_as_described():
    -    cache_points = _page_helper()
    -    system, user = {"role": "system", "content": "s"}, {"role": "user", "content": "u"}
    -    indexes = lambda pts: sorted(p["index"] for p in pts if "index" in p)
    -    # First call: system + end; the newest user message is the end, so LiteLLM merges the two.
    -    pts = cache_points([system, user])
    -    assert {"location": "message", "role": "system"} in pts and indexes(pts) == [-1, 1]
    -    # A tool round: newest user message, first tool result of the round, end of conversation.
    -    msgs = [system, user, _calls(12)] + _results(12)
    -    assert indexes(cache_points(msgs)) == [-1, 1, 3]
    -    # A second round: the round point moves to the newest round.
    -    msgs2 = msgs + [_calls(3)] + _results(3)
    -    assert indexes(cache_points(msgs2)) == [-1, 1, 16]
    -    # The user speaks after the tools: no round point (the newest round is older than the user message).
    -    msgs3 = msgs + [{"role": "assistant", "content": "done"}, user]
    -    assert indexes(cache_points(msgs3)) == [-1, 16]
    -    # Assistant tool calls with no results yet: no point past the end.
    -    assert indexes(cache_points([system, user, _calls(2)])) == [-1, 1]
    -    # No system message, no user message: still valid, never more than 4 points.
    -    assert indexes(cache_points([_calls(1)] + _results(1))) == [-1, 1]
    -    for m in (msgs, msgs2, msgs3):
    -        assert len(cache_points(m)) <= 4
    -
    -
    -def test_live_script_uses_the_page_helper():
    -    import html
    -    page = html.unescape(re.search(r'
    (def cache_points.*?)\n\n\nresponse', LITELLM, re.S).group(1))
    +    code = html.unescape(re.search(r"cache_control_injection_points=(\[.*?\]),\n\)", LITELLM, re.S).group(1))
         script = (SITE.parent / "scripts" / "live_litellm_checks.py").read_text()
    -    assert page in script
    +    in_script = re.search(r"^SYSTEM_AND_LAST = (\[.*\])$", script, re.M).group(1)
    +    assert ast.literal_eval(code) == ast.literal_eval(in_script) == [
    +        {"location": "message", "role": "system"}, {"location": "message", "index": -1}]
    +    # The old 4-point helper for wide tool rounds is gone from both (Bedrock no longer needs it, Oct 2026).
    +    assert "def cache_points" not in LITELLM and "def cache_points" not in script
     
     
     def test_social_preview_images():
    @@ -238,7 +203,7 @@ def test_strands_page_outputs_match_real_output(capsys, monkeypatch):
         monkeypatch.chdir(SITE.parent / "tests" / "fixtures" / "strands")
         page = html.unescape(re.sub(r"<[^>]+>", "", STRANDS))
         cmds = re.findall(r"\$ (cachecanary (?:diff|lint) [^\n]+)", page)
    -    assert len(cmds) == 3
    +    assert len(cmds) == 2
         for cmd in cmds:
             cli.main(cmd.split()[1:])
             out = capsys.readouterr().out.strip()
    @@ -287,39 +252,6 @@ def restore_from_session(self, state):  # Strands checks the class name like thi
                     sys.modules[k] = v
     
     
    -def test_strands_page_hook_places_points_as_described():
    -    import types
    -    hook = _with_fake_strands(_strands_code("from strands.hooks import", "\n\n\nagent = Agent("))["BedrockCachePoints"]()
    -    CP = {"cachePoint": {"type": "default"}}
    -
    -    def place(messages):
    -        hook.place(types.SimpleNamespace(agent=types.SimpleNamespace(messages=messages)))
    -        return [[i for i, b in enumerate(m["content"]) if "cachePoint" in b] for m in messages]
    -
    -    typed = lambda text: {"role": "user", "content": [{"text": text}]}
    -    calls = lambda n: {"role": "assistant", "content": [{"toolUse": {"toolUseId": f"t{i}"}} for i in range(n)]}
    -    results = lambda n: {"role": "user", "content": [{"toolResult": {"toolUseId": f"t{i}"}} for i in range(n)]}
    -    # First call: the typed message is the end, one point.
    -    assert place([typed("hi")]) == [[1]]
    -    # A tool round: the typed message, the first result, the end (3 message points + system = 4).
    -    assert place([typed("hi"), calls(12), results(12)]) == [[1], [], [1, 13]]
    -    # Old points are removed: the next round moves them, never more than 3 in messages.
    -    msgs = [typed("hi"), calls(12), results(12)]
    -    place(msgs)
    -    msgs += [calls(3), results(3)]
    -    assert place(msgs) == [[1], [], [], [], [1, 4]]
    -    # One tool result: a single point at the end.
    -    assert place([typed("hi"), calls(1), results(1)]) == [[1], [], [1]]
    -    # The person speaks after the tools: only their message carries a point.
    -    assert place([typed("hi"), calls(2), results(2), {"role": "assistant", "content": [{"text": "done"}]}, typed("more")]) == [[], [], [], [], [1]]
    -    # A non-PDF document at the end: the point goes before it; a PDF keeps the point after it.
    -    doc = lambda fmt: {"role": "user", "content": [{"text": "read"}, {"document": {"format": fmt}}]}
    -    assert place([doc("txt")]) == [[1]] and place([doc("pdf")]) == [[2]]
    -    # A message that is only a non-PDF document gets no point (Bedrock would reject it).
    -    assert place([{"role": "user", "content": [{"document": {"format": "csv"}}]}]) == [[]]
    -    assert place([]) == []
    -
    -
     def test_strands_page_trim_in_chunks():
         import types
         code = _strands_code("class TrimInChunks", "\n\n\nagent = Agent(")
    @@ -344,9 +276,10 @@ def test_strands_page_trim_in_chunks():
     
     def test_strands_live_script_uses_the_page_code():
         script = (SITE.parent / "scripts" / "live_strands_checks.py").read_text()
    -    hook = _strands_code("class BedrockCachePoints", "\n\n\nagent = Agent(")
         trim = _strands_code("class TrimInChunks", "\n\n\nagent = Agent(")
    -    assert hook in script and trim in script
    +    assert trim in script
    +    # The hook for wide tool fan-outs is gone: Bedrock no longer needs it (Oct 2026).
    +    assert "BedrockCachePoints" not in STRANDS and "BedrockCachePoints" not in script
     
     
     def test_icons_for_browsers_and_search_results():
    @@ -420,7 +353,7 @@ def test_langchain_page_outputs_match_real_output(capsys, monkeypatch):
         monkeypatch.chdir(SITE.parent / "tests" / "fixtures" / "langchain")
         page = html.unescape(re.sub(r"<[^>]+>", "", LANGCHAIN))
         cmds = re.findall(r"\$ (cachecanary (?:diff|lint) [^\n]+)", page)
    -    assert len(cmds) == 3
    +    assert len(cmds) == 2
         for cmd in cmds:
             cli.main(cmd.split()[1:])
             out = capsys.readouterr().out.strip()