From ce2d8f69f5e6872eb9676866f8e896407e353467 Mon Sep 17 00:00:00 2001 From: Abhijeet Prasad Date: Mon, 26 Oct 2026 09:16:49 -0400 Subject: [PATCH 1/2] fix(cohere): capture rerank relevant_spans MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit > 🤖 **This text was AI-generated.** Cohere v2 rerank dropped the relevant_spans request option from trace metadata and discarded the matching passages returned by v5 models: ```text before: output[] = {index, relevance_score} ``` Capture the option in the metadata allowlist and include each returned passage's character offsets and relevance score: ```text after: metadata.relevant_spans = {mode, threshold?, max_spans?} output[] = {index, relevance_score, relevant_spans?: [{start, end, score}]} ``` Omit relevant_spans when absent from a result. Add a recorded v5 example covering both cases and skip it on SDK versions without the request option. Fixes #1004 Refs: https://linear.app/braintrustdata/issue/SDK-630 --- ...est_wrap_cohere_rerank_relevant_spans.yaml | 76 +++++++++++++++++++ .../integrations/cohere/test_cohere.py | 31 ++++++++ .../braintrust/integrations/cohere/tracing.py | 21 +++-- 3 files changed, 120 insertions(+), 8 deletions(-) create mode 100644 py/src/braintrust/integrations/cohere/cassettes/latest/test_wrap_cohere_rerank_relevant_spans.yaml diff --git a/py/src/braintrust/integrations/cohere/cassettes/latest/test_wrap_cohere_rerank_relevant_spans.yaml b/py/src/braintrust/integrations/cohere/cassettes/latest/test_wrap_cohere_rerank_relevant_spans.yaml new file mode 100644 index 000000000..e7c4d2258 --- /dev/null +++ b/py/src/braintrust/integrations/cohere/cassettes/latest/test_wrap_cohere_rerank_relevant_spans.yaml @@ -0,0 +1,76 @@ +interactions: +- request: + body: '{"model":"rerank-v5.0-fast","query":"What is the capital of France?","documents":["Paris + is the capital of France. The Eiffel Tower is a famous landmark in Paris.","Vienna + is the capital of Austria. The city is known for its classical music."],"top_n":2,"relevant_spans":{"mode":"enabled","threshold":0.3,"max_spans":10}}' + headers: + accept: + - '*/*' + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - '321' + content-type: + - application/json + host: + - api.cohere.com + user-agent: + - cohere/7.3.0 + x-fern-language: + - Python + x-fern-platform: + - darwin/25.6.0 + x-fern-runtime: + - python/3.14.7 + x-fern-sdk-name: + - cohere + x-fern-sdk-version: + - 7.3.0 + method: POST + uri: https://api.cohere.com/v2/rerank + response: + body: + string: '{"id":"a83ae71d-53df-47c4-9789-a82beaed4dc2","results":[{"index":0,"relevance_score":0.8421483,"relevant_spans":[{"start":0,"end":31,"score":0.9994439}]},{"index":1,"relevance_score":0.013234873}],"meta":{"api_version":{"version":"2"},"billed_units":{"input_tokens":49}}}' + headers: + access-control-expose-headers: + - X-Debug-Trace-ID + alt-svc: + - h3=":443"; ma=2592000,h3-29=":443"; ma=2592000 + cache-control: + - no-cache, no-store, no-transform, must-revalidate, private, max-age=0 + content-length: + - '271' + content-type: + - application/json + date: + - Mon, 26 Oct 2026 13:07:38 GMT + expires: + - Thu, 01 Jan 1970 00:00:00 GMT + pragma: + - no-cache + server: + - envoy + transfer-encoding: + - chunked + vary: + - Origin,Accept-Encoding + via: + - 1.1 google + x-accel-expires: + - '0' + x-debug-trace-id: + - e72147206125f7971f82cd91d5fd4c25 + x-endpoint-monthly-call-limit: + - '1000' + x-envoy-upstream-service-time: + - '97' + x-trial-endpoint-call-limit: + - '10' + x-trial-endpoint-call-remaining: + - '9' + status: + code: 200 + message: OK +version: 1 diff --git a/py/src/braintrust/integrations/cohere/test_cohere.py b/py/src/braintrust/integrations/cohere/test_cohere.py index ff66835ed..d779c6f14 100644 --- a/py/src/braintrust/integrations/cohere/test_cohere.py +++ b/py/src/braintrust/integrations/cohere/test_cohere.py @@ -452,6 +452,37 @@ def test_wrap_cohere_rerank_sync(memory_logger): assert span["metrics"].get("search_units") == 1 +@pytest.mark.vcr +def test_wrap_cohere_rerank_relevant_spans(memory_logger): + client = _v2_client(require_methods=("rerank",)) + if "relevant_spans" not in inspect.signature(client.rerank).parameters: + pytest.skip("Cohere rerank relevant_spans requires a newer SDK") + client = wrap_cohere(client) + + model = "rerank-v5.0-fast" + documents = [ + "Paris is the capital of France. The Eiffel Tower is a famous landmark in Paris.", + "Vienna is the capital of Austria. The city is known for its classical music.", + ] + relevant_spans = {"mode": "enabled", "threshold": 0.3, "max_spans": 10} + response = client.rerank( + model=model, + query="What is the capital of France?", + documents=documents, + top_n=2, + relevant_spans=relevant_spans, + ) + assert any(result.relevant_spans for result in response.results) + assert any(result.relevant_spans is None for result in response.results) + + spans = memory_logger.pop() + assert len(spans) == 1 + span = spans[0] + assert span["context"]["span_origin"]["instrumentation"]["name"] == "cohere-auto" + assert span["metadata"]["relevant_spans"] == relevant_spans + assert span["output"] == [result.model_dump(exclude_none=True) for result in response.results] + + @pytest.mark.vcr def test_wrap_cohere_chat_stream_v1_sync(memory_logger): assert not memory_logger.pop() diff --git a/py/src/braintrust/integrations/cohere/tracing.py b/py/src/braintrust/integrations/cohere/tracing.py index cbd73b4cf..174dda911 100644 --- a/py/src/braintrust/integrations/cohere/tracing.py +++ b/py/src/braintrust/integrations/cohere/tracing.py @@ -85,6 +85,7 @@ def start_span(*args, **kwargs): "max_chunks_per_doc", "max_tokens_per_doc", "rank_fields", + "relevant_spans", "priority", ) _AUDIO_TRANSCRIPTION_METADATA_KEYS = ( @@ -337,8 +338,8 @@ def _embed_output(result: Any) -> dict[str, Any] | None: def _rerank_output(result: Any) -> list[dict[str, Any]] | None: """Return the top-N rerank results (capped at 100) from a rerank response. - Each result is summarized to ``{index, relevance_score}`` — the document - payload (if present via ``return_documents=True``) is dropped on purpose. + Each result includes its index, relevance score, and relevant spans when + present. The document payload (via ``return_documents=True``) is dropped. """ results = _get_field(result, "results") if not isinstance(results, list): @@ -348,12 +349,16 @@ def _rerank_output(result: Any) -> list[dict[str, Any]] | None: for item in results[:100]: if item is None: continue - out.append( - { - "index": _get_field(item, "index"), - "relevance_score": _get_field(item, "relevance_score"), - } - ) + entry = { + "index": _get_field(item, "index"), + "relevance_score": _get_field(item, "relevance_score"), + } + relevant_spans = _get_field(item, "relevant_spans") + if relevant_spans is not None: + entry["relevant_spans"] = [ + {key: _get_field(span, key) for key in ("start", "end", "score")} for span in relevant_spans + ] + out.append(entry) return out From 83e3f6d8c0dbc4be95b0fc2f012046e293bfc169 Mon Sep 17 00:00:00 2001 From: Abhijeet Prasad Date: Mon, 26 Oct 2026 09:20:08 -0400 Subject: [PATCH 2/2] add commit message skill changes --- .agents/skills/commit-message/SKILL.md | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/.agents/skills/commit-message/SKILL.md b/.agents/skills/commit-message/SKILL.md index 23717f270..1cf2cf2f3 100644 --- a/.agents/skills/commit-message/SKILL.md +++ b/.agents/skills/commit-message/SKILL.md @@ -20,23 +20,21 @@ Use Conventional Commit form: ```text (): - + - + ``` - Choose the most specific suitable type: `feat`, `fix`, `ref` (repository convention for behavior-preserving restructuring), `perf`, `test`, `docs`, `chore`, `ci`, `style`, or `revert`. - Add a scope only when it meaningfully narrows the change. - Keep the subject at 72 characters or fewer, lowercase, imperative, and without a trailing period. -- Add a body when the subject alone does not explain the purpose or a meaningful behavior change. Keep it focused on what changed and why; wrap prose at about 100 characters. -- Add `BREAKING CHANGE: ...` or an issue-closing footer only when the diff or conversation supports it. - -Use direct, specific language. Name relevant APIs, versions, errors, or files when they clarify the change. Avoid restating the diff, unsupported claims, decorative examples, and filler. - -For substantive changes, prefer a compact code snippet or ASCII diagram in the body when it explains behavior or data flow more clearly than prose. Keep it focused and omit it when it adds no clarity. - -## Output and commit execution - -For a message suggestion, return one copyable message in a fenced code block. Add at most one brief sentence explaining a non-obvious type or scope choice. Ask a focused question only when the intent cannot reasonably be inferred from the diff and conversation. +- always add a body. The body must: + - start with a clear AI-generated notice: `> 🤖 **This text was AI-generated.**` + - explain the problem and the solution like you are writing docs for a user. Use simple english and link to issues/prs/docs as appropriate + - for both the problem and solution, try to include code snippets or ASCII diagrams that illustrates the change. ASCII diagrams should be in code snippet blocks + - mention limitations or next steps if applicable + - do not add validation or testing sections unless there is an extremely good reason to do so +- Add `BREAKING CHANGE: ...` or an issue-closing footer only when the diff or conversation supports it. If the change is breaking, make sure to append `!` to the type, like `feat!` +- add relevant github or linear issues to the footer if applicable Do not run `git commit` or `git commit --amend` unless the user explicitly asks you to commit. When committing a multiline message, preserve real newline characters: prefer writing the message to a temporary file and using `git commit -F ` (or `git commit --amend -F `). Multiple `-m` flags or ANSI-C quoting are acceptable for simple cases. Never assume literal `\n` inside ordinary quotes becomes a newline. After committing, verify the stored message with `git log -1 --format=medium`.