diff --git a/.agents/skills/commit-message/SKILL.md b/.agents/skills/commit-message/SKILL.md index 23717f270..1cf2cf2f3 100644 --- a/.agents/skills/commit-message/SKILL.md +++ b/.agents/skills/commit-message/SKILL.md @@ -20,23 +20,21 @@ Use Conventional Commit form: ```text (): - + - + ``` - Choose the most specific suitable type: `feat`, `fix`, `ref` (repository convention for behavior-preserving restructuring), `perf`, `test`, `docs`, `chore`, `ci`, `style`, or `revert`. - Add a scope only when it meaningfully narrows the change. - Keep the subject at 72 characters or fewer, lowercase, imperative, and without a trailing period. -- Add a body when the subject alone does not explain the purpose or a meaningful behavior change. Keep it focused on what changed and why; wrap prose at about 100 characters. -- Add `BREAKING CHANGE: ...` or an issue-closing footer only when the diff or conversation supports it. - -Use direct, specific language. Name relevant APIs, versions, errors, or files when they clarify the change. Avoid restating the diff, unsupported claims, decorative examples, and filler. - -For substantive changes, prefer a compact code snippet or ASCII diagram in the body when it explains behavior or data flow more clearly than prose. Keep it focused and omit it when it adds no clarity. - -## Output and commit execution - -For a message suggestion, return one copyable message in a fenced code block. Add at most one brief sentence explaining a non-obvious type or scope choice. Ask a focused question only when the intent cannot reasonably be inferred from the diff and conversation. +- always add a body. The body must: + - start with a clear AI-generated notice: `> 🤖 **This text was AI-generated.**` + - explain the problem and the solution like you are writing docs for a user. Use simple english and link to issues/prs/docs as appropriate + - for both the problem and solution, try to include code snippets or ASCII diagrams that illustrates the change. ASCII diagrams should be in code snippet blocks + - mention limitations or next steps if applicable + - do not add validation or testing sections unless there is an extremely good reason to do so +- Add `BREAKING CHANGE: ...` or an issue-closing footer only when the diff or conversation supports it. If the change is breaking, make sure to append `!` to the type, like `feat!` +- add relevant github or linear issues to the footer if applicable Do not run `git commit` or `git commit --amend` unless the user explicitly asks you to commit. When committing a multiline message, preserve real newline characters: prefer writing the message to a temporary file and using `git commit -F ` (or `git commit --amend -F `). Multiple `-m` flags or ANSI-C quoting are acceptable for simple cases. Never assume literal `\n` inside ordinary quotes becomes a newline. After committing, verify the stored message with `git log -1 --format=medium`. diff --git a/py/src/braintrust/integrations/cohere/cassettes/latest/test_wrap_cohere_rerank_relevant_spans.yaml b/py/src/braintrust/integrations/cohere/cassettes/latest/test_wrap_cohere_rerank_relevant_spans.yaml new file mode 100644 index 000000000..e7c4d2258 --- /dev/null +++ b/py/src/braintrust/integrations/cohere/cassettes/latest/test_wrap_cohere_rerank_relevant_spans.yaml @@ -0,0 +1,76 @@ +interactions: +- request: + body: '{"model":"rerank-v5.0-fast","query":"What is the capital of France?","documents":["Paris + is the capital of France. The Eiffel Tower is a famous landmark in Paris.","Vienna + is the capital of Austria. The city is known for its classical music."],"top_n":2,"relevant_spans":{"mode":"enabled","threshold":0.3,"max_spans":10}}' + headers: + accept: + - '*/*' + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - '321' + content-type: + - application/json + host: + - api.cohere.com + user-agent: + - cohere/7.3.0 + x-fern-language: + - Python + x-fern-platform: + - darwin/25.6.0 + x-fern-runtime: + - python/3.14.7 + x-fern-sdk-name: + - cohere + x-fern-sdk-version: + - 7.3.0 + method: POST + uri: https://api.cohere.com/v2/rerank + response: + body: + string: '{"id":"a83ae71d-53df-47c4-9789-a82beaed4dc2","results":[{"index":0,"relevance_score":0.8421483,"relevant_spans":[{"start":0,"end":31,"score":0.9994439}]},{"index":1,"relevance_score":0.013234873}],"meta":{"api_version":{"version":"2"},"billed_units":{"input_tokens":49}}}' + headers: + access-control-expose-headers: + - X-Debug-Trace-ID + alt-svc: + - h3=":443"; ma=2592000,h3-29=":443"; ma=2592000 + cache-control: + - no-cache, no-store, no-transform, must-revalidate, private, max-age=0 + content-length: + - '271' + content-type: + - application/json + date: + - Mon, 26 Oct 2026 13:07:38 GMT + expires: + - Thu, 01 Jan 1970 00:00:00 GMT + pragma: + - no-cache + server: + - envoy + transfer-encoding: + - chunked + vary: + - Origin,Accept-Encoding + via: + - 1.1 google + x-accel-expires: + - '0' + x-debug-trace-id: + - e72147206125f7971f82cd91d5fd4c25 + x-endpoint-monthly-call-limit: + - '1000' + x-envoy-upstream-service-time: + - '97' + x-trial-endpoint-call-limit: + - '10' + x-trial-endpoint-call-remaining: + - '9' + status: + code: 200 + message: OK +version: 1 diff --git a/py/src/braintrust/integrations/cohere/test_cohere.py b/py/src/braintrust/integrations/cohere/test_cohere.py index ff66835ed..d779c6f14 100644 --- a/py/src/braintrust/integrations/cohere/test_cohere.py +++ b/py/src/braintrust/integrations/cohere/test_cohere.py @@ -452,6 +452,37 @@ def test_wrap_cohere_rerank_sync(memory_logger): assert span["metrics"].get("search_units") == 1 +@pytest.mark.vcr +def test_wrap_cohere_rerank_relevant_spans(memory_logger): + client = _v2_client(require_methods=("rerank",)) + if "relevant_spans" not in inspect.signature(client.rerank).parameters: + pytest.skip("Cohere rerank relevant_spans requires a newer SDK") + client = wrap_cohere(client) + + model = "rerank-v5.0-fast" + documents = [ + "Paris is the capital of France. The Eiffel Tower is a famous landmark in Paris.", + "Vienna is the capital of Austria. The city is known for its classical music.", + ] + relevant_spans = {"mode": "enabled", "threshold": 0.3, "max_spans": 10} + response = client.rerank( + model=model, + query="What is the capital of France?", + documents=documents, + top_n=2, + relevant_spans=relevant_spans, + ) + assert any(result.relevant_spans for result in response.results) + assert any(result.relevant_spans is None for result in response.results) + + spans = memory_logger.pop() + assert len(spans) == 1 + span = spans[0] + assert span["context"]["span_origin"]["instrumentation"]["name"] == "cohere-auto" + assert span["metadata"]["relevant_spans"] == relevant_spans + assert span["output"] == [result.model_dump(exclude_none=True) for result in response.results] + + @pytest.mark.vcr def test_wrap_cohere_chat_stream_v1_sync(memory_logger): assert not memory_logger.pop() diff --git a/py/src/braintrust/integrations/cohere/tracing.py b/py/src/braintrust/integrations/cohere/tracing.py index cbd73b4cf..174dda911 100644 --- a/py/src/braintrust/integrations/cohere/tracing.py +++ b/py/src/braintrust/integrations/cohere/tracing.py @@ -85,6 +85,7 @@ def start_span(*args, **kwargs): "max_chunks_per_doc", "max_tokens_per_doc", "rank_fields", + "relevant_spans", "priority", ) _AUDIO_TRANSCRIPTION_METADATA_KEYS = ( @@ -337,8 +338,8 @@ def _embed_output(result: Any) -> dict[str, Any] | None: def _rerank_output(result: Any) -> list[dict[str, Any]] | None: """Return the top-N rerank results (capped at 100) from a rerank response. - Each result is summarized to ``{index, relevance_score}`` — the document - payload (if present via ``return_documents=True``) is dropped on purpose. + Each result includes its index, relevance score, and relevant spans when + present. The document payload (via ``return_documents=True``) is dropped. """ results = _get_field(result, "results") if not isinstance(results, list): @@ -348,12 +349,16 @@ def _rerank_output(result: Any) -> list[dict[str, Any]] | None: for item in results[:100]: if item is None: continue - out.append( - { - "index": _get_field(item, "index"), - "relevance_score": _get_field(item, "relevance_score"), - } - ) + entry = { + "index": _get_field(item, "index"), + "relevance_score": _get_field(item, "relevance_score"), + } + relevant_spans = _get_field(item, "relevant_spans") + if relevant_spans is not None: + entry["relevant_spans"] = [ + {key: _get_field(span, key) for key in ("start", "end", "score")} for span in relevant_spans + ] + out.append(entry) return out