Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
41 commits
Select commit Hold shift + click to select a range
52601c2
feat: keep raw file text around + let you pin a ref
FloareDor Oct 18, 2026
e29b568
feat: agent toolbox - search, grep, read, defs, refs
FloareDor Oct 18, 2026
fe9d873
feat: agent loop behind a provider adapter
FloareDor Oct 18, 2026
d20b217
feat: rag/agent mode switch + live tool trace
FloareDor Oct 18, 2026
625f11d
feat: eval set - 45 questions, half borrowed from SWE-QA
FloareDor Oct 18, 2026
b4bd85f
refactor: pull the rag answer path out of the chat page
FloareDor Oct 18, 2026
b0a1fce
feat: eval harness - two arms, a flash judge, flat records
FloareDor Oct 18, 2026
0ac8a58
feat: /eval page - index at the pinned ref, run both arms, export json
FloareDor Oct 18, 2026
42d1f03
feat: run the retrieval ablation on the eval questions too
FloareDor Oct 18, 2026
f8d7451
chore: headless smoke runner for /eval
FloareDor Oct 18, 2026
f490471
fix: make the eval survive rate limits and hung calls
FloareDor Oct 18, 2026
3140f00
feat: error bars on the ablation table
FloareDor Oct 18, 2026
5d4176c
feat: failure taxonomy + keep the agent trace in the record
FloareDor Oct 18, 2026
01306fb
fix: 2.5-flash is gone for new keys, move to 3.8 and count thinking t…
FloareDor Oct 19, 2026
30d2fb9
fix: agent arm was dead on gemini 3 - function role and thought signa…
FloareDor Oct 19, 2026
16a8dcb
chore: say which thinking level produced the numbers
FloareDor Oct 19, 2026
08b1f54
chore: full-eval runner, and make its progress log actually move
FloareDor Oct 19, 2026
bce8c53
feat: eval report - the deck tables, and what the run actually cost
FloareDor Oct 19, 2026
7fa92e3
fix: don't offer agent mode to people who can't run it
FloareDor Oct 19, 2026
13fff1d
chore: stop the benchmark lying about being a passing test
FloareDor Oct 19, 2026
70fd363
feat: report the source/type confound, because the headline table hid…
FloareDor Oct 19, 2026
c9af6d6
feat: show the repo behind each cell, and let exports merge
FloareDor Oct 19, 2026
d0d2eeb
fix: vector-only drops three components, not two
FloareDor Oct 19, 2026
44ed1d6
fix: the run timeout was deciding the result
FloareDor Oct 19, 2026
9a166ff
chore: cost and scoring should agree on which runs count
FloareDor Oct 19, 2026
cbd7c66
feat: find the questions where the arms consistently disagree
FloareDor Oct 19, 2026
e316e4c
feat: record an agent run for the demo slide
FloareDor Oct 19, 2026
3367590
feat: paired comparison, which is the test this design calls for
FloareDor Oct 19, 2026
e59812e
chore: report gaps in standard errors, not a significant/not flag
FloareDor Oct 19, 2026
bb36582
feat: print the ablation, pooled and per repo
FloareDor Oct 19, 2026
07c45e5
fix: stop the agent citing every file it walked past
FloareDor Oct 19, 2026
9be5bca
fix: don't spend an hour writing an export full of nothing
FloareDor Oct 19, 2026
124ac3a
feat: router analysis, computed from the eval that already ran
FloareDor Oct 19, 2026
4cdfb33
fix: before/after was comparing two differently censored samples
FloareDor Oct 19, 2026
fb271e7
chore: separate a judge that failed from a judge that rambled
FloareDor Oct 19, 2026
27bcf3f
chore: ignore recorded demos
FloareDor Oct 19, 2026
3e0b862
feat: isolate the graph hop, and split the ablation by question type
FloareDor Oct 20, 2026
9fe0e48
chore: let the ablation run and export on its own
FloareDor Oct 20, 2026
e90d4dc
fix: --ablation-only was missing its flag, and asked for a key it nev…
FloareDor Oct 20, 2026
efd2525
chore: don't show the auto mode button until it does something
FloareDor Oct 26, 2026
84a8cfb
Merge remote-tracking branch 'origin/main' into agent-mode
FloareDor Oct 26, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 11 additions & 1 deletion .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -43,4 +43,14 @@ next-env.d.ts

/agent-context
agent-context/
.context/
.context/
# SWE-QA splits + pinned source trees, fetched by scripts/download_sweqa.py
scripts/sweqa/

# local dev / smoke-run artifacts
dev.log
smoke.log
smoke-eval*.png

# Recorded agent demos - large media, regenerate with scripts/record-agent-demo.js
demo/
119 changes: 119 additions & 0 deletions scripts/build_eval_questions.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,119 @@
"""
Builds src/lib/eval/questions.json.

The SWE-QA half is lifted from the published benchmark (Apache-2.0) — question
and gold answer verbatim, with `goldLocations` parsed out of the prose the gold
answer already cites and then checked by hand against the pinned commit. The
`type` label and the authored half are ours.

Usage:
python scripts/download_sweqa.py # fetches the two splits
python scripts/build_eval_questions.py
"""

import json
import os
import re
import sys

ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
SWEQA_DIR = os.path.join(ROOT, "scripts", "sweqa")
OUT = os.path.join(ROOT, "src", "lib", "eval", "questions.json")

# Pinned by SWE-QA's own repo_commit.txt. Gold line numbers only hold at these.
REFS = {"flask": "85c5d93", "requests": "46e939b"}
REPOS = {"flask": "pallets/flask", "requests": "psf/requests"}

# Hand-picked: every one cites two or more distinct source files, so a single
# retrieval round can't cover it. Index into the published split, then the gold
# spans transcribed from that record's answer.
PICKS = {
"flask": [
(0, [("src/flask/json/provider.py", 144, 148), ("tests/test_json.py", 48, 54)]),
(1, [("src/flask/sansio/blueprints.py", 172, 172), ("src/flask/sansio/scaffold.py", 42, 42)]),
(5, [("src/flask/sansio/blueprints.py", 461, 475), ("src/flask/templating.py", 138, 150)]),
(11, [("src/flask/ctx.py", 117, 149), ("src/flask/app.py", 1313, 1314)]),
(19, [("src/flask/sessions.py", 52, 82), ("src/flask/ctx.py", 384, 389)]),
(20, [("src/flask/ctx.py", 29, 115), ("src/flask/globals.py", 24, 24)]),
(39, [("src/flask/app.py", 208, 208), ("src/flask/views.py", 56, 56)]),
(40, [("src/flask/sansio/app.py", 823, 846), ("src/flask/sansio/scaffold.py", 123, 126)]),
(45, [("src/flask/ctx.py", 380, 389), ("src/flask/app.py", 1321, 1322), ("src/flask/sessions.py", 141, 146)]),
],
"requests": [
(1, [("tests/test_utils.py", 381, 384), ("src/requests/utils.py", 479, 501)]),
(3, [("tests/test_requests.py", 82, 82), ("src/requests/__init__.py", 165, 176)]),
(9, [("src/requests/exceptions.py", 17, 24), ("src/requests/models.py", 1026, 1026), ("src/requests/adapters.py", 368, 368)]),
(11, [("src/requests/models.py", 483, 492), ("src/requests/utils.py", 1024, 1032), ("src/requests/_internal_utils.py", 25, 35), ("src/requests/structures.py", 13, 80)]),
(24, [("src/requests/models.py", 351, 377), ("src/requests/auth.py", 299, 299)]),
(28, [("src/requests/utils.py", 251, 255), ("src/requests/models.py", 185, 185)]),
(35, [("src/requests/sessions.py", 223, 232), ("src/requests/models.py", 548, 551)]),
(41, [("src/requests/_internal_utils.py", 25, 25), ("tests/test_utils.py", 712, 712), ("src/requests/utils.py", 27, 27)]),
(45, [("src/requests/sessions.py", 191, 191), ("src/requests/exceptions.py", 95, 95)]),
],
}

PROVENANCE = (
"30 questions from SWE-QA (Apache-2.0, swe-qa/SWE-QA-Benchmark) for flask and requests, "
"pinned to the benchmark's own commits; question and gold answer verbatim, gold line spans "
"transcribed from the gold answer's prose and verified against the pinned source. "
"Caveat: SWE-QA gold answers were drafted by a RAG+LLM pipeline before human validation, "
"so the judge may mildly favour RAG-shaped answers — that bias runs against the agent. "
"The remaining 15 gitask questions and all 18 lookups are authored; lookups deliberately "
"name a concrete symbol, which favours the RAG baseline."
)


def load_split(name):
path = os.path.join(SWEQA_DIR, f"{name}.jsonl")
if not os.path.exists(path):
sys.exit(f"missing {path} — run scripts/download_sweqa.py first")
with open(path, encoding="utf-8") as fh:
return [json.loads(line) for line in fh if line.strip()]


def build_sweqa():
out = []
for split, picks in PICKS.items():
rows = load_split(split)
for index, locations in picks:
row = rows[index]
out.append({
"id": f"sweqa-{split}-{index:02d}",
"repo": REPOS[split],
"ref": REFS[split],
"source": "swe-qa",
"type": "multihop",
"question": row["question"],
"goldAnswer": row["answer"],
"goldLocations": [
{"path": p, "startLine": a, "endLine": b} for p, a, b in locations
],
})
return out


def main():
with open(os.path.join(ROOT, "scripts", "eval_authored.json"), encoding="utf-8") as fh:
authored = json.load(fh)
questions = build_sweqa() + authored
payload = {
"name": "gitask agent eval v1",
"provenance": PROVENANCE,
"questions": questions,
}
with open(OUT, "w", encoding="utf-8", newline="\n") as fh:
json.dump(payload, fh, indent=1, ensure_ascii=False)
fh.write("\n")

by_repo = {}
by_type = {}
for q in questions:
by_repo[q["repo"]] = by_repo.get(q["repo"], 0) + 1
by_type[q["type"]] = by_type.get(q["type"], 0) + 1
print(f"wrote {len(questions)} questions -> {OUT}")
print(" by repo:", by_repo)
print(" by type:", by_type)


if __name__ == "__main__":
main()
14 changes: 14 additions & 0 deletions scripts/download_sweqa.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
"""Fetch the SWE-QA splits the eval set borrows from. Output is gitignored."""

import os
import urllib.request

ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
OUT = os.path.join(ROOT, "scripts", "sweqa")
BASE = "https://huggingface.co/datasets/swe-qa/SWE-QA-Benchmark/resolve/main"

os.makedirs(OUT, exist_ok=True)
for split in ("flask", "requests"):
dest = os.path.join(OUT, f"{split}.jsonl")
urllib.request.urlretrieve(f"{BASE}/{split}.jsonl", dest)
print(f"{split}: {os.path.getsize(dest)} bytes -> {dest}")
Loading
Loading