Add GPT-6.1 Terra Codex pipelines and GPT-6.1 Terra/Sol rows

README.md

@@ -57,14 +57,14 @@ In the official benchmark, documents that fail or exceed a 30-minute limit score
 | 2 | Reducto Deep Extract | Specialized APIs | <u>96.08</u> | 96.84 | **94.80** | 92.85 | 6.35¢ |
 | 3 | Pulse (Effort) | Specialized APIs | 96.07 | 96.83 | 94.70 | 93.24 | 10.50¢ |
 | 4 | LlamaExtract Agentic | LlamaExtract | 95.65 | 96.06 | <u>94.79</u> | **94.73** | 3.12¢ |
-| 5 | Claude Code (Sonnet 5.5 Evidence) | Coding Agents | 94.68 | 96.22 | 92.97 | 83.69 | 5.69¢ |
-| 6 | Codex (GPT-6 Sol Evidence) | Coding Agents | 94.57 | 95.99 | 92.42 | 87.25 | 9.80¢ |
-| 7 | Pulse (Non-effort) | Specialized APIs | 94.51 | 95.39 | 92.49 | 93.31 | 3.00¢ |
-| 8 | LlamaExtract Cost-Effective | LlamaExtract | 94.21 | 95.49 | 91.58 | 90.86 | 1.00¢ |
-| 9 | Codex (GPT-5.6 Sol Evidence) | Coding Agents | 93.77 | 96.04 | 90.19 | 82.73 | 21.65¢ |
-| 10 | Codex (GPT-5.5) | Coding Agents | 93.57 | 95.68 | 91.15 | 78.88 | 27.83¢ |
-
-Top 10 of 49 systems — full table in [leaderboard.csv](leaderboard.csv).
+| 5 | Codex (GPT-6.1 Sol Evidence) | Coding Agents | 95.31 | 97.01 | 92.96 | 85.46 | 7.13¢ |
+| 6 | Claude Code (Sonnet 5.5 Evidence) | Coding Agents | 94.68 | 96.22 | 92.97 | 83.69 | 5.69¢ |
+| 7 | Codex (GPT-6 Sol Evidence) | Coding Agents | 94.57 | 95.99 | 92.42 | 87.25 | 9.80¢ |
+| 8 | Pulse (Non-effort) | Specialized APIs | 94.51 | 95.39 | 92.49 | 93.31 | 3.00¢ |
+| 9 | Codex (GPT-6.1 Terra Evidence) | Coding Agents | 94.28 | 96.26 | 91.86 | 81.21 | 4.68¢ |
+| 10 | LlamaExtract Cost-Effective | LlamaExtract | 94.21 | 95.49 | 91.58 | 90.86 | 1.00¢ |
+
+Top 10 of 51 systems — full table in [leaderboard.csv](leaderboard.csv).
 <!-- LEADERBOARD:END -->
 
 <!-- GROUNDING:START -->
@@ -76,20 +76,20 @@ Top 10 of 49 systems — full table in [leaderboard.csv](leaderboard.csv).
     <tr><th align="right">Overall</th><th align="right">Short</th><th align="right">Medium</th><th align="right">Long</th><th align="right">Overall</th><th align="right">Short</th><th align="right">Medium</th><th align="right">Long</th></tr>
   </thead>
   <tbody>
-    <tr><td align="right">1</td><td>LlamaExtract Agentic Plus</td><td align="right"><strong>83.25</strong></td><td align="right"><strong>83.22</strong></td><td align="right"><u>83.44</u></td><td align="right"><strong>82.84</strong></td><td align="right"><strong>88.26</strong></td><td align="right"><strong>87.97</strong></td><td align="right"><u>88.48</u></td><td align="right"><u>90.61</u></td></tr>
-    <tr><td align="right">2</td><td>LlamaExtract Agentic</td><td align="right"><u>82.33</u></td><td align="right"><u>81.39</u></td><td align="right"><strong>84.94</strong></td><td align="right"><u>81.63</u></td><td align="right"><u>88.20</u></td><td align="right">86.95</td><td align="right"><strong>90.33</strong></td><td align="right"><strong>92.59</strong></td></tr>
-    <tr><td align="right">3</td><td>LlamaExtract Cost-Effective</td><td align="right">79.80</td><td align="right">80.99</td><td align="right">77.52</td><td align="right">76.16</td><td align="right">86.28</td><td align="right">86.97</td><td align="right">84.60</td><td align="right">86.41</td></tr>
-    <tr><td align="right">4</td><td>Codex (GPT-6 Sol Evidence)</td><td align="right">77.11</td><td align="right">77.61</td><td align="right">75.53</td><td align="right">78.31</td><td align="right">86.50</td><td align="right">87.20</td><td align="right">85.12</td><td align="right">84.82</td></tr>
-    <tr><td align="right">5</td><td>Claude Code (Opus 5.5 Evidence)</td><td align="right">71.55</td><td align="right">73.57</td><td align="right">65.54</td><td align="right">74.87</td><td align="right">76.95</td><td align="right">79.27</td><td align="right">71.36</td><td align="right">76.47</td></tr>
-    <tr><td align="right">6</td><td>Codex (GPT-6 Luna Evidence)</td><td align="right">65.70</td><td align="right">62.91</td><td align="right">71.11</td><td align="right">74.36</td><td align="right">83.93</td><td align="right">84.41</td><td align="right">83.47</td><td align="right">80.17</td></tr>
-    <tr><td align="right">7</td><td>Claude Code (Sonnet 5.5 Evidence)</td><td align="right">64.61</td><td align="right">64.59</td><td align="right">63.18</td><td align="right">71.22</td><td align="right">75.99</td><td align="right">77.93</td><td align="right">71.50</td><td align="right">74.98</td></tr>
-    <tr><td align="right">8</td><td>Reducto Extract (v4)</td><td align="right">54.97</td><td align="right">60.53</td><td align="right">49.35</td><td align="right">16.30</td><td align="right">75.28</td><td align="right"><u>87.47</u></td><td align="right">57.38</td><td align="right">18.59</td></tr>
-    <tr><td align="right">9</td><td>Codex (GPT-5.6 Sol Evidence)</td><td align="right">54.66</td><td align="right">50.87</td><td align="right">64.58</td><td align="right">69.01</td><td align="right">83.62</td><td align="right">85.12</td><td align="right">80.77</td><td align="right">78.84</td></tr>
-    <tr><td align="right">10</td><td>Codex (GPT-5.5 Evidence)</td><td align="right">52.57</td><td align="right">53.13</td><td align="right">48.01</td><td align="right">61.87</td><td align="right">80.64</td><td align="right">83.27</td><td align="right">73.87</td><td align="right">79.90</td></tr>
+    <tr><td align="right">1</td><td>Codex (GPT-6.1 Sol Evidence)</td><td align="right"><strong>87.61</strong></td><td align="right"><strong>89.03</strong></td><td align="right"><u>84.38</u></td><td align="right"><strong>85.70</strong></td><td align="right"><strong>89.37</strong></td><td align="right"><strong>89.63</strong></td><td align="right"><u>89.25</u></td><td align="right">86.80</td></tr>
+    <tr><td align="right">2</td><td>Codex (GPT-6.1 Terra Evidence)</td><td align="right"><u>85.69</u></td><td align="right"><u>88.13</u></td><td align="right">81.32</td><td align="right">77.21</td><td align="right">87.42</td><td align="right"><u>88.49</u></td><td align="right">86.10</td><td align="right">81.15</td></tr>
+    <tr><td align="right">3</td><td>LlamaExtract Agentic Plus</td><td align="right">83.25</td><td align="right">83.22</td><td align="right">83.44</td><td align="right"><u>82.84</u></td><td align="right"><u>88.26</u></td><td align="right">87.97</td><td align="right">88.48</td><td align="right"><u>90.61</u></td></tr>
+    <tr><td align="right">4</td><td>LlamaExtract Agentic</td><td align="right">82.33</td><td align="right">81.39</td><td align="right"><strong>84.94</strong></td><td align="right">81.63</td><td align="right">88.20</td><td align="right">86.95</td><td align="right"><strong>90.33</strong></td><td align="right"><strong>92.59</strong></td></tr>
+    <tr><td align="right">5</td><td>LlamaExtract Cost-Effective</td><td align="right">79.80</td><td align="right">80.99</td><td align="right">77.52</td><td align="right">76.16</td><td align="right">86.28</td><td align="right">86.97</td><td align="right">84.60</td><td align="right">86.41</td></tr>
+    <tr><td align="right">6</td><td>Codex (GPT-6 Sol Evidence)</td><td align="right">77.11</td><td align="right">77.61</td><td align="right">75.53</td><td align="right">78.31</td><td align="right">86.50</td><td align="right">87.20</td><td align="right">85.12</td><td align="right">84.82</td></tr>
+    <tr><td align="right">7</td><td>Claude Code (Opus 5.5 Evidence)</td><td align="right">71.55</td><td align="right">73.57</td><td align="right">65.54</td><td align="right">74.87</td><td align="right">76.95</td><td align="right">79.27</td><td align="right">71.36</td><td align="right">76.47</td></tr>
+    <tr><td align="right">8</td><td>Codex (GPT-6 Luna Evidence)</td><td align="right">65.70</td><td align="right">62.91</td><td align="right">71.11</td><td align="right">74.36</td><td align="right">83.93</td><td align="right">84.41</td><td align="right">83.47</td><td align="right">80.17</td></tr>
+    <tr><td align="right">9</td><td>Claude Code (Sonnet 5.5 Evidence)</td><td align="right">64.61</td><td align="right">64.59</td><td align="right">63.18</td><td align="right">71.22</td><td align="right">75.99</td><td align="right">77.93</td><td align="right">71.50</td><td align="right">74.98</td></tr>
+    <tr><td align="right">10</td><td>Reducto Extract (v4)</td><td align="right">54.97</td><td align="right">60.53</td><td align="right">49.35</td><td align="right">16.30</td><td align="right">75.28</td><td align="right">87.47</td><td align="right">57.38</td><td align="right">18.59</td></tr>
   </tbody>
 </table>
 
-Top 10 of 49 systems — full table in [leaderboard.csv](leaderboard.csv).
+Top 10 of 51 systems — full table in [leaderboard.csv](leaderboard.csv).
 <!-- GROUNDING:END -->
 
 ### ExtractBench Pro leaderboard

docs/pipelines.md

@@ -38,7 +38,7 @@ uv run extract-bench run <pipeline_name>
 | `codex_code_extract_gpt_5_5_low_evidence` / `codex_code_extract_gpt_5_6_sol_low_evidence` / `codex_code_extract_gpt_5_6_terra_low_evidence` | Codex CLI | low-effort extraction with page and box citations; unrestricted shell |
 | `codex_code_extract_gpt_5_6_luna_medium_evidence` | Codex CLI | medium-effort extraction with page and box citations; unrestricted shell, no cost cap |
 | `codex_code_extract_gpt_6_luna_medium_evidence` / `codex_code_extract_gpt_6_luna_high_evidence` / `codex_code_extract_gpt_6_luna_xhigh_evidence` / `codex_code_extract_gpt_6_sol_low_evidence` / `codex_code_extract_gpt_6_sol_medium_evidence` / `codex_code_extract_gpt_6_sol_high_evidence` | Codex CLI | GPT-6 Luna and Sol extraction with page and box citations at the effort in the name; unrestricted shell, no cost cap |
-| `codex_code_extract_gpt_6_astra_xhigh_evidence` / `codex_code_extract_gpt_6_1_sol_{low,medium,high,xhigh}_evidence` | Codex CLI | GPT-6 Astra and GPT-6.1 Sol extraction with page and box citations at the effort in the name; unrestricted shell, no cost cap |
+| `codex_code_extract_gpt_6_astra_xhigh_evidence` / `codex_code_extract_gpt_6_1_sol_{low,medium,high,xhigh}_evidence` / `codex_code_extract_gpt_6_1_terra_{low,medium}_evidence` | Codex CLI | GPT-6 Astra, GPT-6.1 Sol and GPT-6.1 Terra extraction with page and box citations at the effort in the name; unrestricted shell, no cost cap |
 | `codex_code_extract_gpt_6_luna_{low,medium,high,xhigh}_evidence_prompt_opt` | Codex CLI | GPT-6 Luna evidence extraction plus a short prompt addendum: always write the output files, the documents are authorized for extraction, no field needs web access |
 | `reducto_extract` / `reducto_extract_v4` / `reducto_deep_extract` | Reducto | `_v4` uses Reducto's v4 extract model |
 | `extend_extract` / `extend_extract_max` | Extend | citations enabled; max-context array strategy variant |

leaderboard.csv

@@ -1,8 +1,10 @@
 Provider,Category,Overall,Short,Medium,Long,Cost_Per_Page,Cost_Short,Cost_Medium,Cost_Long,P_Short,P_Medium,P_Long,R_Short,R_Medium,R_Long,Word_Grounding,Word_Grounding_Short,Word_Grounding_Medium,Word_Grounding_Long,Page_Grounding,Page_Grounding_Short,Page_Grounding_Medium,Page_Grounding_Long,Latency_S_Per_Doc_Short,Latency_S_Per_Doc_Medium,Latency_S_Per_Doc_Long
 LlamaExtract Agentic Plus,LlamaExtract,96.17,97.02,94.30,94.65,0.0750,0.0750,0.0750,0.0750,97.08,94.86,94.77,97.02,93.93,94.54,83.25,83.22,83.44,82.84,88.26,87.97,88.48,90.61,106.7,172.9,495.2
 Pulse (Effort),Specialized APIs,96.07,96.83,94.70,93.24,0.1050,0.1050,0.1050,0.1050,96.85,94.93,93.71,96.86,94.60,92.96,14.94,12.58,20.80,15.96,83.62,81.32,87.95,90.08,118.0,249.7,507.3
+Codex (GPT-6.1 Sol Evidence),Coding Agents,95.31,97.01,92.96,85.46,0.0713,0.0924,0.0287,0.0098,96.88,93.44,85.24,97.19,92.84,85.69,87.61,89.03,84.38,85.70,89.37,89.63,89.25,86.80,112.5,101.5,136.2
 Claude Code (Sonnet 5.5 Evidence),Coding Agents,94.68,96.22,92.97,83.69,0.0569,0.0735,0.0237,0.0091,96.68,94.47,83.36,95.93,91.83,84.05,64.61,64.59,63.18,71.22,75.99,77.93,71.50,74.98,82.5,87.7,187.0
 Codex (GPT-6 Sol Evidence),Coding Agents,94.57,95.99,92.42,87.25,0.0980,0.1279,0.0388,0.0122,96.33,93.49,91.28,95.80,91.72,86.70,77.11,77.61,75.53,78.31,86.50,87.20,85.12,84.82,104.2,89.9,101.4
+Codex (GPT-6.1 Terra Evidence),Coding Agents,94.28,96.26,91.86,81.21,0.0468,0.0612,0.0179,0.0052,97.23,92.18,95.51,95.86,91.71,81.37,85.69,88.13,81.32,77.21,87.42,88.49,86.10,81.15,86.0,77.3,86.5
 Codex (GPT-5.6 Sol Evidence),Coding Agents,93.77,96.04,90.19,82.73,0.2165,0.2813,0.0870,0.0300,96.28,91.37,86.24,95.84,89.21,82.42,54.66,50.87,64.58,69.01,83.62,85.12,80.77,78.84,95.2,106.6,197.2
 Codex (GPT-5.5),Coding Agents,93.57,95.68,91.15,78.88,0.2783,0.3579,0.1214,0.0434,96.01,93.64,94.03,95.51,90.16,78.58,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,70.3,82.6,122.0
 Codex (GPT-5.5 Evidence),Coding Agents,93.35,95.59,88.72,87.97,0.3363,0.4354,0.1406,0.0472,95.86,89.85,89.29,95.36,87.86,87.08,52.57,53.13,48.01,61.87,80.64,83.27,73.87,79.90,89.0,117.8,164.6

src/extract_bench/inference/pipelines/extract.py

@@ -375,6 +375,7 @@ def register_extract_pipelines(register_fn) -> None:  # type: ignore[no-untyped-
         ("gpt_6_luna", "gpt-6-luna", ("medium", "high", "xhigh")),
         ("gpt_6_sol", "gpt-6-sol", ("low", "medium", "high")),
         ("gpt_6_1_sol", "gpt-6.1-sol", ("low", "medium", "high", "xhigh")),
+        ("gpt_6_1_terra", "gpt-6.1-terra", ("low", "medium")),
     ):
         for _reasoning_effort in _gpt6_efforts:
             register_fn(

src/extract_bench/inference/providers/extract/codex_code_extract.py

@@ -80,11 +80,12 @@
 )
 
 # USD per million tokens (input, cached input, output); estimates use Codex CLI JSONL usage.
-# Cached input is input/10 across the table (input/20 for gpt-6.1-sol), matching
+# Cached input is input/10 across the table (input/20 for gpt-6.1-sol and gpt-6.1-terra), matching
 # OpenAI's cached-token discount.
 _OPENAI_CODEX_PRICING_PER_M: dict[str, tuple[float, float, float]] = {
     "gpt-6-astra": (10.00, 1.00, 50.00),
     "gpt-6.1-sol": (2.00, 0.10, 10.00),
+    "gpt-6.1-terra": (1.00, 0.05, 5.00),
     "gpt-6-sol": (2.00, 0.20, 10.00),
     "gpt-6-luna": (0.10, 0.01, 0.50),
     "gpt-5.6-sol": (4.00, 0.40, 20.00),
@@ -101,6 +102,7 @@
 _OPENAI_CODEX_CACHE_WRITE_PER_M: dict[str, float] = {
     "gpt-6-astra": 12.50,
     "gpt-6.1-sol": 2.50,
+    "gpt-6.1-terra": 1.25,
     "gpt-6-sol": 2.50,
     "gpt-6-luna": 0.125,
 }