diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5a9a35e..9460f5c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -10,7 +10,11 @@ env: jobs: test: - runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-latest, windows-latest] + runs-on: ${{ matrix.os }} steps: - name: Checkout @@ -44,6 +48,41 @@ jobs: - name: Run tests run: cargo test --verbose + msrv: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 + + - name: Read minimum supported Rust + id: msrv + run: | + python3 - <<'PYTHON' + import os + import tomllib + with open("Cargo.toml", "rb") as manifest: + version = tomllib.load(manifest)["package"]["rust-version"] + with open(os.environ["GITHUB_OUTPUT"], "a") as output: + output.write(f"version={version}\n") + PYTHON + + - name: Set up minimum supported Rust + uses: dtolnay/rust-toolchain@631a55b12751854ce901bb631d5902ceb48146f7 # stable + with: + toolchain: ${{ steps.msrv.outputs.version }} + + - name: Resolve dependencies using the declared MSRV + run: cargo generate-lockfile + + - name: Build all targets on MSRV + run: cargo check --locked --all-targets --all-features + + - name: Run tests and doctests on MSRV + run: cargo test --locked --all-features + + - name: Verify packaged crate on MSRV + run: cargo package --locked + # Check documentation builds docs: runs-on: ubuntu-latest diff --git a/Cargo.toml b/Cargo.toml index 22e77af..5ff62a8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -2,11 +2,14 @@ name = "braintrust-sdk-rust" version = "0.1.0-alpha.2" edition = "2021" +resolver = "3" rust-version = "1.86.0" authors = ["Braintrust Team "] description = "Rust SDK for Braintrust logging and tracing" repository = "https://github.com/braintrustdata/braintrust-sdk-rust" license = "Apache-2.0" +readme = "README.md" +include = ["src/**", "tests/**", "examples/**", "README.md", "LICENSE", "braintrust-logo.svg"] keywords = ["braintrust", "logging", "tracing", "observability"] categories = ["api-bindings", "asynchronous", "development-tools"] diff --git a/README.md b/README.md index 0fa848c..7dbe3f1 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -[![Braintrust](./braintrust-logo.svg)](https://www.braintrust.dev/) +[![Braintrust](https://raw.githubusercontent.com/braintrustdata/braintrust-sdk-rust/main/braintrust-logo.svg)](https://www.braintrust.dev/) # braintrust-sdk-rust @@ -8,55 +8,180 @@ Rust SDK for [Braintrust](https://braintrust.dev) logging and tracing. ## Installation -Add this to your `Cargo.toml`: +The API below is unreleased. To use this checkout, add it as a path dependency: ```toml [dependencies] -braintrust-sdk-rust = "0.1.0-alpha.1" +braintrust-sdk-rust = { path = "../braintrust-sdk-rust" } ``` -## Usage +## Rust version -```rust -use braintrust_sdk_rust::{BraintrustClient, BraintrustClientConfig, SpanLog}; +The minimum supported Rust version is **1.86.0**. This repository uses Cargo +resolver 3 to prefer dependency versions compatible with its declared MSRV. + +Consuming applications control their own resolver and lockfile. For Rust 1.86 +builds, select resolver 3 in the application's root manifest, for example: + +```toml +[workspace] +resolver = "3" +``` + +Regenerate an existing incompatible lockfile with the compatible resolver; +changing the resolver alone does not replace versions already locked. See +[Cargo's dependency resolver documentation](https://doc.rust-lang.org/cargo/reference/resolver.html#resolver-versions). + +## Logging and tracing + +Set `BRAINTRUST_API_KEY`, or pass a key with `.api_key(...)`. A client owns its +runtime by default, including when built inside an existing Tokio runtime. + +```rust,no_run +use braintrust_sdk_rust::{BraintrustClient, SpanLog}; use serde_json::json; +use std::time::Duration; #[tokio::main] async fn main() -> anyhow::Result<()> { - // Create a client pointing to your Braintrust instance - let client = BraintrustClient::new( - BraintrustClientConfig::new("https://api.braintrust.dev") - )?; + let client = BraintrustClient::builder() + .default_project("my-project") + .build() + .await?; - // Create a span for logging - let span = client - .span_builder("your-api-token", "your-org-id") - .project_name("my-project") - .build(); - - // Log input/output using span.log() with the builder pattern - span.log( + let span = client.span_builder().await?.name("greeting").build(); + let write = span.log( SpanLog::builder() .input(json!({"prompt": "Hello, world!"})) .output(json!({"response": "Hi there!"})) - .build() - .expect("Failed to build SpanLog") - ).await; + .build()?, + ); + write.await?; + span.end().await?; + client.shutdown(Duration::from_secs(10)).await?; + Ok(()) +} +``` + +`log()` submits immediately. `end()` finishes the span and returns a completion +for delivery of its final snapshot. Later mutations of a finished span are +ignored with a diagnostic. Create child spans for separate operations; use +`update_span()` or `log_feedback()` to patch an existing row explicitly. + +## Submission and completion + +Methods that schedule work return `Completion`. Calling the method submits +the operation; awaiting its token observes the result. Inspect `&*completion` +for the immediate admission `Result<(), E>`, such as a full or closed queue. +An admitted operation can still fail later. Dropping the token leaves accepted +work running. + +```rust,no_run +use braintrust_sdk_rust::{SpanHandle, SpanLog}; + +async fn log(span: &SpanHandle) -> anyhow::Result<()> { + let completion = span.log(SpanLog::default()); + if let Err(error) = &*completion { + eprintln!("submission rejected: {error}"); + } + completion.await?; + Ok(()) +} +``` + +Writes are batched and submitted automatically. `client.flush()` captures queued +and active writes through the point of the call and observes their delivery. +A terminal batch failure fails its write completions and covering flush, then +retires that batch. Later flushes process new work independently. Failures are +not retained for replay. + +Await `client.shutdown(timeout)` before exiting to close admission and drain +accepted work. A timeout fails remaining completions and discards pending SDK +state. Concurrent shutdown calls share the first call's deadline and outcome. + +## Synchronous applications and runtime selection - // Flush the span data to Braintrust - span.flush().await?; - client.flush().await?; +The same API works on a plain thread without creating a Tokio runtime. Use a +future executor to wait for values when needed; for example, with the `futures` +crate: +```rust,no_run +use braintrust_sdk_rust::BraintrustClient; +use std::{future::IntoFuture, time::Duration}; + +fn main() -> anyhow::Result<()> { + let client = futures::executor::block_on( + BraintrustClient::builder().build().into_future(), + )?; + futures::executor::block_on(client.shutdown(Duration::from_secs(10)).into_future())?; Ok(()) } ``` -## Features +To select your own Tokio runtime, pass `.runtime(current_runtime())` while +inside it, or `.runtime(Some(handle))`. With an explicit handle, keep that +runtime alive and driving work until completions finish. `current_runtime()` +returns `None` outside Tokio, which selects an SDK-owned runtime. + +## Evaluations + +An `EvalDefinition` contains reusable task and scorer callbacks. Each run supplies +its own case source and options. Local runs need no client, credentials or +serialization: + +```rust +use braintrust_sdk_rust::eval::{ + self, Case, EvalDefinition, EvalOpts, EvalRequest, ExactMatch, SyncFnTask, +}; + +fn main() -> anyhow::Result<()> { + let definition = EvalDefinition::new( + Box::new(SyncFnTask::new(|input: &String, _hooks| Ok(input.to_uppercase()))), + vec![Box::new(ExactMatch::new())], + )?; + let options = EvalOpts::builder() + .name("uppercase".into()) + .definition(definition) + .source(vec![Case { + input: "hello".to_string(), + expected: Some("HELLO".to_string()), + ..Default::default() + }].into()) + .quiet(true) + .build(); + let completion = eval::run(EvalRequest::local(options)); + assert!(completion.is_ok()); + let summary = futures::executor::block_on(async { completion.await })?; + assert_eq!(summary.successful_cases, 1); + assert_eq!(summary.results[0].output.as_deref(), Some("HELLO")); + Ok(()) +} +``` + +Use `EvalRequest::recorded(options, client)` to record an experiment on that +client's runtime. Recorded inputs and outputs must implement `Serialize`. +`DatasetCaseSource` reads Braintrust datasets and preserves dataset provenance +when recording. Run failures retain partial summaries and structured causes. + +Use `task(...)` and `scorer(...)` for async callbacks, or `SyncFnTask` and +`SyncFnScorer` for blocking callbacks. Attach a terminal, JSONL, silent or custom +reporter with `EvalRequest::with_reporter(...)`. Reporter failures preserve +computation and recording results. + +`BtEvalRunner` adapts the same execution engine to `bt eval`, including discovery, +filters and output transports. See the +[CLI example](https://github.com/braintrustdata/braintrust-sdk-rust/blob/main/examples/eval_async.rs) +and [synchronous example](https://github.com/braintrustdata/braintrust-sdk-rust/blob/main/examples/eval_sync.rs). + +## API organization -- **Span-based logging**: Create spans to track LLM calls and other operations -- **Usage metrics extraction**: Built-in extractors for OpenAI and Anthropic usage metrics -- **Async-first**: Built on Tokio for high-performance async operations -- **Background submission**: Logs are submitted in the background to minimize latency +- `BraintrustClient` creates concrete dataset, experiment and span handles with + shared credentials and lifecycle. +- `eval` separates definitions, case sources, execution, recording and reporting. +- `api::Client` exposes direct endpoint operations with one HTTP attempt per + call. Use it when your application owns delivery and retry policy. +- Usage extractors support OpenAI and Anthropic responses; streaming helpers + support provider response assembly. ## Preserving existing span provenance @@ -112,4 +237,4 @@ let usage = extract_anthropic_usage(&anthropic_response); ## License -This project is licensed under the [Apache License, Version 2.0](LICENSE). +This project is licensed under the [Apache License, Version 2.0](https://github.com/braintrustdata/braintrust-sdk-rust/blob/main/LICENSE). diff --git a/src/lib.rs b/src/lib.rs index 504c5b0..fc64fdd 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,3 +1,4 @@ +#![doc = include_str!("../README.md")] #![deny(private_bounds, private_interfaces)] pub mod api;