From a3b6f030c1e6e6f218a061aa475211005eb29a99 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Wed, 7 Oct 2026 22:12:57 +0000 Subject: [PATCH 1/6] Rewrite Behaviors product copy as labels a judge applies. Agent- and user-visible wording now uses Labels. The MCP search filter field stays "behaviors", and the old behavior-monitoring docs link points at /documentation/monitoring. Co-authored-by: shunuen0 --- .claude-plugin/plugin.json | 2 +- .codex-plugin/plugin.json | 4 ++-- README.md | 2 +- skills/judgment-tracing/references/tracing.md | 14 +++++++------- skills/mcp-server-best-practices/SKILL.md | 4 ++-- 5 files changed, 13 insertions(+), 13 deletions(-) diff --git a/.claude-plugin/plugin.json b/.claude-plugin/plugin.json index 7fcd14a..78281cd 100644 --- a/.claude-plugin/plugin.json +++ b/.claude-plugin/plugin.json @@ -2,7 +2,7 @@ "$schema": "https://json.schemastore.org/claude-code-plugin-manifest.json", "name": "judgment", "displayName": "Judgment", - "description": "Access Judgment traces, sessions, behaviors, judges, prompts, datasets, and tests directly from your AI coding tool, with official skills for tracing, evaluation, and MCP usage best practices.", + "description": "Access Judgment traces, sessions, labels, judges, prompts, datasets, and tests directly from your AI coding tool, with official skills for tracing, evaluation, and MCP usage best practices.", "version": "1.0.1", "author": { "name": "Judgment Labs", diff --git a/.codex-plugin/plugin.json b/.codex-plugin/plugin.json index 5ccd054..ba8a759 100644 --- a/.codex-plugin/plugin.json +++ b/.codex-plugin/plugin.json @@ -1,7 +1,7 @@ { "name": "judgment", "version": "1.0.1", - "description": "Access Judgment traces, sessions, behaviors, judges, prompts, datasets, and tests directly from Codex, with official skills for tracing, evaluation, and MCP usage best practices.", + "description": "Access Judgment traces, sessions, labels, judges, prompts, datasets, and tests directly from Codex, with official skills for tracing, evaluation, and MCP usage best practices.", "author": { "name": "Judgment Labs", "email": "support@judgmentlabs.ai", @@ -24,7 +24,7 @@ "mcpServers": "./.mcp.json", "interface": { "displayName": "Judgment", - "shortDescription": "Query Judgment traces, sessions, behaviors, judges, and docs.", + "shortDescription": "Query Judgment traces, sessions, labels, judges, and docs.", "longDescription": "Adds the Judgment MCP server and the official Judgment skills so Codex can inspect live agent data and follow Judgment best practices for tracing, evaluations, code judges, datasets, and monitoring.", "developerName": "Judgment Labs", "category": "Productivity", diff --git a/README.md b/README.md index 8876b9b..072677b 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ The official [Judgment](https://judgmentlabs.ai) plugin for AI coding agents. One repo, installable in both Claude Code and OpenAI Codex, bundling: 1. **The Judgment MCP server** (`https://mcp.judgmentlabs.ai`): query traces, - sessions, behaviors, judges, prompts, datasets, tests, automations, and + sessions, labels, judges, prompts, datasets, tests, automations, and agent memory directly from your editor. 2. **The `judgment-tracing` skill**: best-practice guidance for instrumenting apps with tracing, writing evaluations, building code judges, and running diff --git a/skills/judgment-tracing/references/tracing.md b/skills/judgment-tracing/references/tracing.md index 431f55a..719e66e 100644 --- a/skills/judgment-tracing/references/tracing.md +++ b/skills/judgment-tracing/references/tracing.md @@ -150,7 +150,7 @@ when possible, only ask when unclear. | Customer, org, workspace, or tenant identifiers | Multi-tenant app | `customer_id`, `tenant_id`, or plan tier | | Multiple routes, tools, agents, or product features | Multi-feature app | `feature`, `route`, or agent name attribute | | A/B tests, model routing, prompt variants | Experimented app | experiment, prompt, or model-family tag | -| Feedback collection, ratings, thumbs up/down | Has user feedback | capture as scores or behavior signals | +| Feedback collection, ratings, thumbs up/down | Has user feedback | capture as a score, or as a label a judge applies | | Stateless HTTP handlers, workers, queues, serverless functions, or RPC calls | Distributed app | distributed tracing with `service.name` | | Environment-specific projects | Needs project routing | active tracers with project names | | Traced functions spread across modules | Cross-file instrumentation | one tracer init, observed functions | @@ -166,8 +166,8 @@ when possible, only ask when unclear. Determines whether project routing is needed - "Does one request cross a stateless HTTP, worker, queue, serverless, or RPC boundary?" -> Determines whether distributed tracing is needed -- "How do you know when a response is good vs bad?" -> Determines scoring or - behavior-monitoring follow-up +- "How do you know when a response is good vs bad?" -> Determines whether the + follow-up is a score or a label a judge applies **Additions and their value:** @@ -181,7 +181,7 @@ when possible, only ask when unclear. | `service.name` | Separates services in OpenTelemetry and distributed flows | https://docs.judgmentlabs.ai/documentation/performance/tracing#opentelemetry-integration | | Distributed trace propagation | Connects stateless downstream service spans to the original request | https://docs.judgmentlabs.ai/documentation/performance/tracing#distributed-tracing | | Agent linked traces | Links delegated agent/subsystem work back to the parent trace | https://docs.judgmentlabs.ai/documentation/performance/tracing#subagent-tracing | -| Behavior or score signals | Enables quality filtering and production monitoring | https://docs.judgmentlabs.ai/documentation/performance/agent-behavior-monitoring | +| Labels a judge applies | Filter traces by that label, or by a score | https://docs.judgmentlabs.ai/documentation/monitoring | These are NOT baseline best practices. Only add what's relevant based on inference or user input. @@ -211,9 +211,9 @@ After adding context, point users to relevant UI features: - Traces view: See individual requests, span hierarchy, latency, cost, inputs, outputs, and errors - Sessions view: See grouped conversations if `session_id` is added -- Behaviors: Filter traces or sessions by monitored behavior -- Automations and alerts: Route failures, behaviors, or thresholds into - follow-up workflows +- Labels: Filter traces or sessions by a label a judge applied +- Automations and alerts: Start a follow-up when a run fails, a judge + applies a label, or a threshold is crossed - Judgment Agent search: Ask the in-product agent to search traces, cite spans, and investigate failures diff --git a/skills/mcp-server-best-practices/SKILL.md b/skills/mcp-server-best-practices/SKILL.md index b110d83..ab25873 100644 --- a/skills/mcp-server-best-practices/SKILL.md +++ b/skills/mcp-server-best-practices/SKILL.md @@ -11,7 +11,7 @@ metadata: ## Default to MCP -**Always use MCP tools first** when the user asks about anything related to Judgment data — traces, behaviors, sessions, projects, automations, judges, prompts, datasets, tests, documentation, agent memory, agent threads, or organizations. Do not fall back to reading code or asking for IDs if an MCP tool can fetch the data directly. +**Always use MCP tools first** when the user asks about anything related to Judgment data — traces, labels, sessions, projects, automations, judges, prompts, datasets, tests, documentation, agent memory, agent threads, or organizations. Do not fall back to reading code or asking for IDs if an MCP tool can fetch the data directly. ## Using search_traces Effectively @@ -49,7 +49,7 @@ Each filter is an object with a `field` discriminator: // LLM cost (USD) { "field": "llm_cost", "op": ">", "value": 0.10 } -// Behaviors (any of the listed judge/value pairs) +// Labels a judge applied (the filter field is still "behaviors"; any of the listed judge/value pairs) { "field": "behaviors", "op": "any", "value": [{ "judge_name": "toxicity", "value": "toxic" }] } // Numeric score by name From c20db7e492e4f7baf6466e643df27cf9b7560ccb Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Wed, 7 Oct 2026 22:14:10 +0000 Subject: [PATCH 2/6] Align label wording with the internal plugin mirror. Noun lists stay plain "labels". The tracing skill and MCP filter comment now use the same sentences as judgment-plugin-internal. Co-authored-by: shunuen0 --- skills/judgment-tracing/references/tracing.md | 14 +++++++------- skills/mcp-server-best-practices/SKILL.md | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/skills/judgment-tracing/references/tracing.md b/skills/judgment-tracing/references/tracing.md index 719e66e..8db69fd 100644 --- a/skills/judgment-tracing/references/tracing.md +++ b/skills/judgment-tracing/references/tracing.md @@ -150,7 +150,7 @@ when possible, only ask when unclear. | Customer, org, workspace, or tenant identifiers | Multi-tenant app | `customer_id`, `tenant_id`, or plan tier | | Multiple routes, tools, agents, or product features | Multi-feature app | `feature`, `route`, or agent name attribute | | A/B tests, model routing, prompt variants | Experimented app | experiment, prompt, or model-family tag | -| Feedback collection, ratings, thumbs up/down | Has user feedback | capture as a score, or as a label a judge applies | +| Feedback collection, ratings, thumbs up/down | Has user feedback | capture as scores or labels | | Stateless HTTP handlers, workers, queues, serverless functions, or RPC calls | Distributed app | distributed tracing with `service.name` | | Environment-specific projects | Needs project routing | active tracers with project names | | Traced functions spread across modules | Cross-file instrumentation | one tracer init, observed functions | @@ -166,8 +166,8 @@ when possible, only ask when unclear. Determines whether project routing is needed - "Does one request cross a stateless HTTP, worker, queue, serverless, or RPC boundary?" -> Determines whether distributed tracing is needed -- "How do you know when a response is good vs bad?" -> Determines whether the - follow-up is a score or a label a judge applies +- "How do you know when a response is good vs bad?" -> Determines whether to + follow up with a score or a label **Additions and their value:** @@ -181,7 +181,7 @@ when possible, only ask when unclear. | `service.name` | Separates services in OpenTelemetry and distributed flows | https://docs.judgmentlabs.ai/documentation/performance/tracing#opentelemetry-integration | | Distributed trace propagation | Connects stateless downstream service spans to the original request | https://docs.judgmentlabs.ai/documentation/performance/tracing#distributed-tracing | | Agent linked traces | Links delegated agent/subsystem work back to the parent trace | https://docs.judgmentlabs.ai/documentation/performance/tracing#subagent-tracing | -| Labels a judge applies | Filter traces by that label, or by a score | https://docs.judgmentlabs.ai/documentation/monitoring | +| Labels or scores | Filter by label or score and track how often each label is applied | https://docs.judgmentlabs.ai/documentation/monitoring | These are NOT baseline best practices. Only add what's relevant based on inference or user input. @@ -211,9 +211,9 @@ After adding context, point users to relevant UI features: - Traces view: See individual requests, span hierarchy, latency, cost, inputs, outputs, and errors - Sessions view: See grouped conversations if `session_id` is added -- Labels: Filter traces or sessions by a label a judge applied -- Automations and alerts: Start a follow-up when a run fails, a judge - applies a label, or a threshold is crossed +- Labels: Filter traces or sessions by the labels judges applied +- Automations and alerts: Route failures, a "Judge applied label" condition, or + thresholds into follow-up workflows - Judgment Agent search: Ask the in-product agent to search traces, cite spans, and investigate failures diff --git a/skills/mcp-server-best-practices/SKILL.md b/skills/mcp-server-best-practices/SKILL.md index ab25873..669ebba 100644 --- a/skills/mcp-server-best-practices/SKILL.md +++ b/skills/mcp-server-best-practices/SKILL.md @@ -49,7 +49,7 @@ Each filter is an object with a `field` discriminator: // LLM cost (USD) { "field": "llm_cost", "op": ">", "value": 0.10 } -// Labels a judge applied (the filter field is still "behaviors"; any of the listed judge/value pairs) +// Labels (any of the listed judge/value pairs) { "field": "behaviors", "op": "any", "value": [{ "judge_name": "toxicity", "value": "toxic" }] } // Numeric score by name From 00361ab37587b01d970619246009950f0f6c81f7 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Wed, 7 Oct 2026 22:26:01 +0000 Subject: [PATCH 3/6] Point label tracking at the judges docs page and capitalize Code judges. User feedback is captured as scores only. Code judge headings in the tracing skill use sentence case, and the monitor-judge-results link is the one that ships with the docs deploy. Co-authored-by: shunuen0 --- .codex-plugin/plugin.json | 2 +- README.md | 2 +- skills/judgment-tracing/references/agent-testing.md | 2 +- skills/judgment-tracing/references/code-judges.md | 12 ++++++------ skills/judgment-tracing/references/tracing.md | 4 ++-- 5 files changed, 11 insertions(+), 11 deletions(-) diff --git a/.codex-plugin/plugin.json b/.codex-plugin/plugin.json index ba8a759..79d156a 100644 --- a/.codex-plugin/plugin.json +++ b/.codex-plugin/plugin.json @@ -25,7 +25,7 @@ "interface": { "displayName": "Judgment", "shortDescription": "Query Judgment traces, sessions, labels, judges, and docs.", - "longDescription": "Adds the Judgment MCP server and the official Judgment skills so Codex can inspect live agent data and follow Judgment best practices for tracing, evaluations, code judges, datasets, and monitoring.", + "longDescription": "Adds the Judgment MCP server and the official Judgment skills so Codex can inspect live agent data and follow Judgment best practices for tracing, evaluations, Code judges, datasets, and monitoring.", "developerName": "Judgment Labs", "category": "Productivity", "capabilities": [ diff --git a/README.md b/README.md index 072677b..5e9333a 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ One repo, installable in both Claude Code and OpenAI Codex, bundling: sessions, labels, judges, prompts, datasets, tests, automations, and agent memory directly from your editor. 2. **The `judgment-tracing` skill**: best-practice guidance for instrumenting - apps with tracing, writing evaluations, building code judges, and running + apps with tracing, writing evaluations, building Code judges, and running offline agent tests. 3. **The `mcp-server-best-practices` skill**: usage patterns for the Judgment MCP itself (search_traces batching, filter reference, fan-out queries). diff --git a/skills/judgment-tracing/references/agent-testing.md b/skills/judgment-tracing/references/agent-testing.md index 9398223..03d6d10 100644 --- a/skills/judgment-tracing/references/agent-testing.md +++ b/skills/judgment-tracing/references/agent-testing.md @@ -153,7 +153,7 @@ results = client.evaluation.create().run( The agent entrypoint should be traced as the root span, usually with `@Tracer.observe(span_type="agent")` or the framework/provider integration the application already uses. Hosted scorers can use `offline_trace_id` to inspect -the stored trace server-side. Custom code judges can also use the fields copied +the stored trace server-side. Code judges can also use the fields copied into each generated example. ## Implementation Notes diff --git a/skills/judgment-tracing/references/code-judges.md b/skills/judgment-tracing/references/code-judges.md index 84aa237..0d4cb53 100644 --- a/skills/judgment-tracing/references/code-judges.md +++ b/skills/judgment-tracing/references/code-judges.md @@ -1,8 +1,8 @@ -# Judgment Code Judges +# Judgment Code judges Use this reference when creating Python code judges. -## Current Support +## Current support Code judges are Python-first. Do not invent a TypeScript `Judge` subclass workflow unless the current docs say it exists. @@ -12,7 +12,7 @@ Fetch the current docs before implementation: curl -s https://docs.judgmentlabs.ai/documentation/evaluation/custom-scorers.md ``` -## When To Use A Code Judge +## When to use a Code judge Use a code judge when scoring needs: @@ -21,7 +21,7 @@ Use a code judge when scoring needs: - direct trace inspection - structured numeric, binary, or categorical outputs -## Minimal Pattern +## Minimal pattern ```python from judgeval.data import Example @@ -37,7 +37,7 @@ class ResolutionScorer(Judge[BinaryResponse]): ) ``` -## Trace-Aware Pattern +## Trace-aware pattern Check for trace data before reading spans: @@ -61,7 +61,7 @@ class ToolCallScorer(Judge[NumericResponse]): ) ``` -## Packaging Guidance +## Packaging guidance - Keep the scorer class in the entrypoint file. - Add dependencies to `requirements.txt`. diff --git a/skills/judgment-tracing/references/tracing.md b/skills/judgment-tracing/references/tracing.md index 8db69fd..9fcf09d 100644 --- a/skills/judgment-tracing/references/tracing.md +++ b/skills/judgment-tracing/references/tracing.md @@ -150,7 +150,7 @@ when possible, only ask when unclear. | Customer, org, workspace, or tenant identifiers | Multi-tenant app | `customer_id`, `tenant_id`, or plan tier | | Multiple routes, tools, agents, or product features | Multi-feature app | `feature`, `route`, or agent name attribute | | A/B tests, model routing, prompt variants | Experimented app | experiment, prompt, or model-family tag | -| Feedback collection, ratings, thumbs up/down | Has user feedback | capture as scores or labels | +| Feedback collection, ratings, thumbs up/down | Has user feedback | capture as scores | | Stateless HTTP handlers, workers, queues, serverless functions, or RPC calls | Distributed app | distributed tracing with `service.name` | | Environment-specific projects | Needs project routing | active tracers with project names | | Traced functions spread across modules | Cross-file instrumentation | one tracer init, observed functions | @@ -181,7 +181,7 @@ when possible, only ask when unclear. | `service.name` | Separates services in OpenTelemetry and distributed flows | https://docs.judgmentlabs.ai/documentation/performance/tracing#opentelemetry-integration | | Distributed trace propagation | Connects stateless downstream service spans to the original request | https://docs.judgmentlabs.ai/documentation/performance/tracing#distributed-tracing | | Agent linked traces | Links delegated agent/subsystem work back to the parent trace | https://docs.judgmentlabs.ai/documentation/performance/tracing#subagent-tracing | -| Labels or scores | Filter by label or score and track how often each label is applied | https://docs.judgmentlabs.ai/documentation/monitoring | +| Labels or scores | Filter by label or score and track how often each label is applied | https://docs.judgmentlabs.ai/documentation/judges/monitor-judge-results | These are NOT baseline best practices. Only add what's relevant based on inference or user input. From aa49c62fafb49ef510548cdf334a88e73b955189 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Wed, 7 Oct 2026 22:34:49 +0000 Subject: [PATCH 4/6] Capitalize Code judge in agent-visible prose. Rubric scoring in the tracing skill is described as Agent judges. Paths, identifiers, and the behaviors filter field are unchanged. Co-authored-by: shunuen0 --- .claude-plugin/marketplace.json | 2 +- skills/judgment-tracing/SKILL.md | 10 +++++----- skills/judgment-tracing/references/code-judges.md | 4 ++-- skills/judgment-tracing/references/evaluations.md | 4 ++-- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json index a26b938..eabe52e 100644 --- a/.claude-plugin/marketplace.json +++ b/.claude-plugin/marketplace.json @@ -12,7 +12,7 @@ { "name": "judgment", "source": "./", - "description": "Judgment MCP server plus official skills (tracing, evaluations, code judges, MCP usage best practices)." + "description": "Judgment MCP server plus official skills (tracing, evaluations, Code judges, MCP usage best practices)." } ] } diff --git a/skills/judgment-tracing/SKILL.md b/skills/judgment-tracing/SKILL.md index f1e60bf..eba65fa 100644 --- a/skills/judgment-tracing/SKILL.md +++ b/skills/judgment-tracing/SKILL.md @@ -1,6 +1,6 @@ --- name: judgment-tracing -description: Use Judgment for agent tracing, evaluations, code judges, datasets, and monitoring. Use when integrating Judgment or judgeval, adding tracing to agents/workflows, creating evaluations or scorers, debugging traces, or looking up Judgment SDK usage and docs. +description: Use Judgment for agent tracing, evaluations, Code judges, datasets, and monitoring. Use when integrating Judgment or judgeval, adding tracing to agents/workflows, creating evaluations or scorers, debugging traces, or looking up Judgment SDK usage and docs. allowed-tools: - WebFetch(domain:docs.judgmentlabs.ai) - Bash(curl *docs.judgmentlabs.ai/*) @@ -8,7 +8,7 @@ allowed-tools: # Judgment -This skill helps you use Judgment effectively across common agent development workflows: instrumenting applications, evaluating outputs, creating code judges, and looking up current Judgment docs. +This skill helps you use Judgment effectively across common agent development workflows: instrumenting applications, evaluating outputs, creating Code judges, and looking up current Judgment docs. ## Core Principles @@ -16,10 +16,10 @@ Follow these principles for all Judgment work: 1. **Docs first**: Fetch current Judgment docs and refer to reference files before implementing SDK patterns from memory. 2. **Instrument the real path**: Add tracing to the functions, tools, and LLM calls the app actually executes. -3. **Tracing first for integrations**: When the user asks to add Judgment to an app, start with [references/tracing.md](references/tracing.md) unless they explicitly ask for evaluations or code judges. +3. **Tracing first for integrations**: When the user asks to add Judgment to an app, start with [references/tracing.md](references/tracing.md) unless they explicitly ask for evaluations or Code judges. 4. **Stay on the tracer surface**: In most cases, use Judgment's tracer, wrappers, and documented integrations directly. Do not reach into underlying provider objects or create additional wrapper layers unless the current docs require it or a real instrumentation gap remains after using the supported integration. 5. **Start small**: For evaluations, begin with a focused example set and one scorer before expanding. -6. **Use the right scorer**: Use prompt/hosted scorers for rubric-based judgment and Python code judges for deterministic logic, custom dependencies, or trace inspection. +6. **Use the right scorer**: Use Agent judges for rubric-based judgment and Python Code judges for deterministic logic, custom dependencies, or trace inspection. 7. **Keep credentials out of chat**: Ask the user to set `JUDGMENT_API_KEY` and `JUDGMENT_ORG_ID` locally rather than pasting secrets. 8. **Use offline agent tests before production changes**: For model, prompt, tool, or agent config changes, pull a stable dataset, collect fresh `OfflineTracer` traces for each input, then evaluate the generated offline examples in one batch. If the production agent is already traced with Judgment, leave that tracing intact and swap only the test harness initialization to `client.offline_tracer(...)`. @@ -28,5 +28,5 @@ Follow these principles for all Judgment work: - Adding or auditing tracing: [references/tracing.md](references/tracing.md) - Creating evaluations and choosing scorers: [references/evaluations.md](references/evaluations.md) - Testing agent changes with OfflineTracer: [references/agent-testing.md](references/agent-testing.md) -- Creating Python code judges: [references/code-judges.md](references/code-judges.md) +- Creating Python Code judges: [references/code-judges.md](references/code-judges.md) - Using Judgment docs and SDK references: [references/docs.md](references/docs.md) diff --git a/skills/judgment-tracing/references/code-judges.md b/skills/judgment-tracing/references/code-judges.md index 0d4cb53..26abbf0 100644 --- a/skills/judgment-tracing/references/code-judges.md +++ b/skills/judgment-tracing/references/code-judges.md @@ -1,6 +1,6 @@ # Judgment Code judges -Use this reference when creating Python code judges. +Use this reference when creating Python Code judges. ## Current support @@ -14,7 +14,7 @@ curl -s https://docs.judgmentlabs.ai/documentation/evaluation/custom-scorers.md ## When to use a Code judge -Use a code judge when scoring needs: +Use a Code judge when scoring needs: - deterministic business logic - custom Python dependencies diff --git a/skills/judgment-tracing/references/evaluations.md b/skills/judgment-tracing/references/evaluations.md index 936e3b2..63c9cef 100644 --- a/skills/judgment-tracing/references/evaluations.md +++ b/skills/judgment-tracing/references/evaluations.md @@ -18,7 +18,7 @@ Use this reference when creating or debugging evaluations. 4. Pick one scorer first: - prompt or hosted scorer for rubric-based natural-language judgment - - Python code judge for deterministic logic, custom dependencies, or trace inspection + - Python Code judge for deterministic logic, custom dependencies, or trace inspection 5. Run locally while iterating, then expand the dataset and hosted workflows when the signal is useful. @@ -40,7 +40,7 @@ examples = [ ## Scorer Selection - Use prompt scorers for qualitative rubrics such as helpfulness, policy adherence, or answer quality. -- Use code judges for exact checks, business logic, custom library calls, or direct trace inspection. +- Use Code judges for exact checks, business logic, custom library calls, or direct trace inspection. - Use datasets when the user needs repeatable regression testing across many examples. ## Common Mistakes From 250e0022c4e853b796d07e3efcd32cb52b1678ea Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Wed, 7 Oct 2026 22:39:09 +0000 Subject: [PATCH 5/6] Call rubric judges Agent judges in skill prose. The prompt-scorers docs route stays in the link and the curl example. Co-authored-by: shunuen0 --- skills/judgment-tracing/references/docs.md | 2 +- skills/judgment-tracing/references/evaluations.md | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/skills/judgment-tracing/references/docs.md b/skills/judgment-tracing/references/docs.md index 8cfdf62..a67251c 100644 --- a/skills/judgment-tracing/references/docs.md +++ b/skills/judgment-tracing/references/docs.md @@ -32,7 +32,7 @@ curl -s https://docs.judgmentlabs.ai/sdk-reference/typescript.md ## Useful Topics - Tracing: `https://docs.judgmentlabs.ai/documentation/performance/tracing` -- Prompt scorers: `https://docs.judgmentlabs.ai/documentation/evaluation/prompt-scorers` +- Agent judges: `https://docs.judgmentlabs.ai/documentation/evaluation/prompt-scorers` - Code judges: `https://docs.judgmentlabs.ai/documentation/evaluation/custom-scorers` - Datasets: `https://docs.judgmentlabs.ai/documentation/evaluation/datasets` - Python SDK reference: `https://docs.judgmentlabs.ai/sdk-reference/python` diff --git a/skills/judgment-tracing/references/evaluations.md b/skills/judgment-tracing/references/evaluations.md index 63c9cef..a8d97d8 100644 --- a/skills/judgment-tracing/references/evaluations.md +++ b/skills/judgment-tracing/references/evaluations.md @@ -17,7 +17,7 @@ Use this reference when creating or debugging evaluations. 4. Pick one scorer first: - - prompt or hosted scorer for rubric-based natural-language judgment + - Agent judge or hosted scorer for rubric-based natural-language judgment - Python Code judge for deterministic logic, custom dependencies, or trace inspection 5. Run locally while iterating, then expand the dataset and hosted workflows when the signal is useful. @@ -39,7 +39,7 @@ examples = [ ## Scorer Selection -- Use prompt scorers for qualitative rubrics such as helpfulness, policy adherence, or answer quality. +- Use Agent judges for qualitative rubrics such as helpfulness, policy adherence, or answer quality. - Use Code judges for exact checks, business logic, custom library calls, or direct trace inspection. - Use datasets when the user needs repeatable regression testing across many examples. From 40eac42c7a8c1bfa279e0876f0733b0ffe6a4264 Mon Sep 17 00:00:00 2001 From: Shun Ueno Date: Mon, 12 Oct 2026 18:27:07 -0700 Subject: [PATCH 6/6] Revert unrelated heading and capitalization edits --- .claude-plugin/marketplace.json | 2 +- .codex-plugin/plugin.json | 2 +- README.md | 2 +- skills/judgment-tracing/SKILL.md | 10 +++++----- .../judgment-tracing/references/code-judges.md | 16 ++++++++-------- .../judgment-tracing/references/evaluations.md | 4 ++-- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json index eabe52e..a26b938 100644 --- a/.claude-plugin/marketplace.json +++ b/.claude-plugin/marketplace.json @@ -12,7 +12,7 @@ { "name": "judgment", "source": "./", - "description": "Judgment MCP server plus official skills (tracing, evaluations, Code judges, MCP usage best practices)." + "description": "Judgment MCP server plus official skills (tracing, evaluations, code judges, MCP usage best practices)." } ] } diff --git a/.codex-plugin/plugin.json b/.codex-plugin/plugin.json index 79d156a..ba8a759 100644 --- a/.codex-plugin/plugin.json +++ b/.codex-plugin/plugin.json @@ -25,7 +25,7 @@ "interface": { "displayName": "Judgment", "shortDescription": "Query Judgment traces, sessions, labels, judges, and docs.", - "longDescription": "Adds the Judgment MCP server and the official Judgment skills so Codex can inspect live agent data and follow Judgment best practices for tracing, evaluations, Code judges, datasets, and monitoring.", + "longDescription": "Adds the Judgment MCP server and the official Judgment skills so Codex can inspect live agent data and follow Judgment best practices for tracing, evaluations, code judges, datasets, and monitoring.", "developerName": "Judgment Labs", "category": "Productivity", "capabilities": [ diff --git a/README.md b/README.md index 5e9333a..072677b 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ One repo, installable in both Claude Code and OpenAI Codex, bundling: sessions, labels, judges, prompts, datasets, tests, automations, and agent memory directly from your editor. 2. **The `judgment-tracing` skill**: best-practice guidance for instrumenting - apps with tracing, writing evaluations, building Code judges, and running + apps with tracing, writing evaluations, building code judges, and running offline agent tests. 3. **The `mcp-server-best-practices` skill**: usage patterns for the Judgment MCP itself (search_traces batching, filter reference, fan-out queries). diff --git a/skills/judgment-tracing/SKILL.md b/skills/judgment-tracing/SKILL.md index eba65fa..457d1ec 100644 --- a/skills/judgment-tracing/SKILL.md +++ b/skills/judgment-tracing/SKILL.md @@ -1,6 +1,6 @@ --- name: judgment-tracing -description: Use Judgment for agent tracing, evaluations, Code judges, datasets, and monitoring. Use when integrating Judgment or judgeval, adding tracing to agents/workflows, creating evaluations or scorers, debugging traces, or looking up Judgment SDK usage and docs. +description: Use Judgment for agent tracing, evaluations, code judges, datasets, and monitoring. Use when integrating Judgment or judgeval, adding tracing to agents/workflows, creating evaluations or scorers, debugging traces, or looking up Judgment SDK usage and docs. allowed-tools: - WebFetch(domain:docs.judgmentlabs.ai) - Bash(curl *docs.judgmentlabs.ai/*) @@ -8,7 +8,7 @@ allowed-tools: # Judgment -This skill helps you use Judgment effectively across common agent development workflows: instrumenting applications, evaluating outputs, creating Code judges, and looking up current Judgment docs. +This skill helps you use Judgment effectively across common agent development workflows: instrumenting applications, evaluating outputs, creating code judges, and looking up current Judgment docs. ## Core Principles @@ -16,10 +16,10 @@ Follow these principles for all Judgment work: 1. **Docs first**: Fetch current Judgment docs and refer to reference files before implementing SDK patterns from memory. 2. **Instrument the real path**: Add tracing to the functions, tools, and LLM calls the app actually executes. -3. **Tracing first for integrations**: When the user asks to add Judgment to an app, start with [references/tracing.md](references/tracing.md) unless they explicitly ask for evaluations or Code judges. +3. **Tracing first for integrations**: When the user asks to add Judgment to an app, start with [references/tracing.md](references/tracing.md) unless they explicitly ask for evaluations or code judges. 4. **Stay on the tracer surface**: In most cases, use Judgment's tracer, wrappers, and documented integrations directly. Do not reach into underlying provider objects or create additional wrapper layers unless the current docs require it or a real instrumentation gap remains after using the supported integration. 5. **Start small**: For evaluations, begin with a focused example set and one scorer before expanding. -6. **Use the right scorer**: Use Agent judges for rubric-based judgment and Python Code judges for deterministic logic, custom dependencies, or trace inspection. +6. **Use the right scorer**: Use Agent judges for rubric-based judgment and Python code judges for deterministic logic, custom dependencies, or trace inspection. 7. **Keep credentials out of chat**: Ask the user to set `JUDGMENT_API_KEY` and `JUDGMENT_ORG_ID` locally rather than pasting secrets. 8. **Use offline agent tests before production changes**: For model, prompt, tool, or agent config changes, pull a stable dataset, collect fresh `OfflineTracer` traces for each input, then evaluate the generated offline examples in one batch. If the production agent is already traced with Judgment, leave that tracing intact and swap only the test harness initialization to `client.offline_tracer(...)`. @@ -28,5 +28,5 @@ Follow these principles for all Judgment work: - Adding or auditing tracing: [references/tracing.md](references/tracing.md) - Creating evaluations and choosing scorers: [references/evaluations.md](references/evaluations.md) - Testing agent changes with OfflineTracer: [references/agent-testing.md](references/agent-testing.md) -- Creating Python Code judges: [references/code-judges.md](references/code-judges.md) +- Creating Python code judges: [references/code-judges.md](references/code-judges.md) - Using Judgment docs and SDK references: [references/docs.md](references/docs.md) diff --git a/skills/judgment-tracing/references/code-judges.md b/skills/judgment-tracing/references/code-judges.md index 26abbf0..84aa237 100644 --- a/skills/judgment-tracing/references/code-judges.md +++ b/skills/judgment-tracing/references/code-judges.md @@ -1,8 +1,8 @@ -# Judgment Code judges +# Judgment Code Judges -Use this reference when creating Python Code judges. +Use this reference when creating Python code judges. -## Current support +## Current Support Code judges are Python-first. Do not invent a TypeScript `Judge` subclass workflow unless the current docs say it exists. @@ -12,16 +12,16 @@ Fetch the current docs before implementation: curl -s https://docs.judgmentlabs.ai/documentation/evaluation/custom-scorers.md ``` -## When to use a Code judge +## When To Use A Code Judge -Use a Code judge when scoring needs: +Use a code judge when scoring needs: - deterministic business logic - custom Python dependencies - direct trace inspection - structured numeric, binary, or categorical outputs -## Minimal pattern +## Minimal Pattern ```python from judgeval.data import Example @@ -37,7 +37,7 @@ class ResolutionScorer(Judge[BinaryResponse]): ) ``` -## Trace-aware pattern +## Trace-Aware Pattern Check for trace data before reading spans: @@ -61,7 +61,7 @@ class ToolCallScorer(Judge[NumericResponse]): ) ``` -## Packaging guidance +## Packaging Guidance - Keep the scorer class in the entrypoint file. - Add dependencies to `requirements.txt`. diff --git a/skills/judgment-tracing/references/evaluations.md b/skills/judgment-tracing/references/evaluations.md index a8d97d8..811f581 100644 --- a/skills/judgment-tracing/references/evaluations.md +++ b/skills/judgment-tracing/references/evaluations.md @@ -18,7 +18,7 @@ Use this reference when creating or debugging evaluations. 4. Pick one scorer first: - Agent judge or hosted scorer for rubric-based natural-language judgment - - Python Code judge for deterministic logic, custom dependencies, or trace inspection + - Python code judge for deterministic logic, custom dependencies, or trace inspection 5. Run locally while iterating, then expand the dataset and hosted workflows when the signal is useful. @@ -40,7 +40,7 @@ examples = [ ## Scorer Selection - Use Agent judges for qualitative rubrics such as helpfulness, policy adherence, or answer quality. -- Use Code judges for exact checks, business logic, custom library calls, or direct trace inspection. +- Use code judges for exact checks, business logic, custom library calls, or direct trace inspection. - Use datasets when the user needs repeatable regression testing across many examples. ## Common Mistakes