From b3aeef68bb818c091d89badf8e8eafff5b5537e9 Mon Sep 17 00:00:00 2001 From: Stephen Belanger Date: Wed, 21 Oct 2026 01:19:05 +0800 Subject: [PATCH] Separate reusable evaluation definitions from run options --- examples/eval_async.rs | 44 +++++---- examples/eval_sync.rs | 4 +- src/eval/bt_runner.rs | 10 +- src/eval/definition.rs | 197 ++++++++++++++++++++++++++++++++++++++++ src/eval/mod.rs | 14 ++- src/eval/runner.rs | 56 ++++-------- tests/eval_callables.rs | 32 ++++--- tests/eval_test.rs | 93 +++++++++++++++---- 8 files changed, 353 insertions(+), 97 deletions(-) create mode 100644 src/eval/definition.rs diff --git a/examples/eval_async.rs b/examples/eval_async.rs index f17894d..9b1f4cc 100644 --- a/examples/eval_async.rs +++ b/examples/eval_async.rs @@ -1,5 +1,4 @@ use braintrust_sdk_rust::eval::*; -use braintrust_sdk_rust::*; /// Demonstrates using BtEvalRunner, which is compatible with the `bt eval` CLI. /// @@ -10,7 +9,7 @@ use braintrust_sdk_rust::*; /// - BT_EVAL_SSE_SOCK=... — send SSE events over a Unix socket /// - BT_EVAL_SSE_ADDR=... — send SSE events over TCP #[tokio::main] -async fn main() -> Result<()> { +async fn main() -> anyhow::Result<()> { // BtEvalRunner reads BT_EVAL_* env vars automatically. // In non-list mode it creates a BraintrustClient from the environment. let mut runner = BtEvalRunner::from_env().await?; @@ -39,30 +38,35 @@ async fn main() -> Result<()> { }, ]; + // Share callbacks across runs; each run supplies its own source and options. + let definition = EvalDefinition::new( + Box::new(task(|input: &String, _hooks| { + Box::pin(async move { Ok(input.to_uppercase()) }) + })), + vec![ + Box::new(ExactMatch::new()), + Box::new(LevenshteinScorer::new()), + Box::new(scorer( + "nonempty", + |args: ScorerArgs<'_, String, String>| { + Box::pin(async move { + Ok(vec![Score::new( + "nonempty", + if args.output.is_empty() { 0.0 } else { 1.0 }, + )]) + }) + }, + )), + ], + )?; + runner .eval( "my-project", EvalOpts::builder() .name("uppercase-eval".to_string()) .source(cases.into()) - .task(Box::new(task(|input: &String, _hooks| { - Box::pin(async move { Ok(input.to_uppercase()) }) - }))) - .scorers(vec![ - Box::new(ExactMatch::new()), - Box::new(LevenshteinScorer::new()), - Box::new(scorer( - "nonempty", - |args: ScorerArgs<'_, String, String>| { - Box::pin(async move { - Ok(vec![Score::new( - "nonempty", - if args.output.is_empty() { 0.0 } else { 1.0 }, - )]) - }) - }, - )), - ]) + .definition(definition.clone()) .max_concurrency(2) .build(), ) diff --git a/examples/eval_sync.rs b/examples/eval_sync.rs index f6fec87..ac3c245 100644 --- a/examples/eval_sync.rs +++ b/examples/eval_sync.rs @@ -39,12 +39,12 @@ fn main() -> anyhow::Result<()> { // Run evaluation synchronously println!("Starting synchronous evaluation...\n"); + let definition = EvalDefinition::new(Box::new(task), scorers)?; let result = evaluator.run( EvalOpts::builder() .name("sync-uppercase-eval".to_string()) .source(cases.into()) - .task(Box::new(task)) - .scorers(scorers) + .definition(definition.clone()) .build(), )?; diff --git a/src/eval/bt_runner.rs b/src/eval/bt_runner.rs index fcb2eed..8662a52 100644 --- a/src/eval/bt_runner.rs +++ b/src/eval/bt_runner.rs @@ -186,9 +186,14 @@ fn create_sse_writer() -> Option> { /// ```no_run /// use braintrust_sdk_rust::eval::*; /// -/// # async fn example() -> braintrust_sdk_rust::Result<()> { +/// # async fn example() -> anyhow::Result<()> { /// let mut runner = BtEvalRunner::from_env().await?; /// +/// let definition = EvalDefinition::new( +/// Box::new(SimpleFnTask::new(|input: &String| Ok(input.to_uppercase()))), +/// vec![Box::new(ExactMatch::new())], +/// )?; +/// /// runner.eval( /// "my-project", /// EvalOpts::builder() @@ -198,8 +203,7 @@ fn create_sse_writer() -> Option> { /// expected: Some("HELLO".to_string()), /// ..Default::default() /// }].into()) -/// .task(Box::new(SimpleFnTask::new(|input: &String| Ok(input.to_uppercase())))) -/// .scorers(vec![Box::new(ExactMatch::new())]) +/// .definition(definition.clone()) /// .build(), /// ).await?; /// diff --git a/src/eval/definition.rs b/src/eval/definition.rs new file mode 100644 index 0000000..11adddd --- /dev/null +++ b/src/eval/definition.rs @@ -0,0 +1,197 @@ +//! Reusable computation shared by independent evaluation runs. +use std::sync::Arc; + +use async_trait::async_trait; + +use super::{EvalConfigError, Scorer, ScorerArgs, Scores, Task}; + +/// A validated task and ordered set of scorers, independent of any client or run. +/// +/// Cloning shares the callbacks without cloning inputs or outputs. Callback +/// state is shared across runs; sources, hooks and results belong to each run. +/// Callbacks must therefore support concurrent invocation. +pub struct EvalDefinition { + inner: Arc>, +} + +struct DefinitionInner { + task: Box>, + scorers: Vec>>, +} + +impl Clone for EvalDefinition { + fn clone(&self) -> Self { + Self { + inner: Arc::clone(&self.inner), + } + } +} + +impl EvalDefinition { + /// Validate scorer names and retain the callbacks for reuse. + /// + /// Names are captured once, including for custom scorers whose `name()` + /// could change through interior state. Empty or duplicate names are rejected. + pub fn new( + task: Box>, + scorers: Vec>>, + ) -> Result { + let mut names = std::collections::HashSet::new(); + let mut validated: Vec>> = Vec::with_capacity(scorers.len()); + for (index, scorer) in scorers.into_iter().enumerate() { + let name = scorer.name().to_owned(); + if name.trim().is_empty() { + return Err(EvalConfigError::EmptyScorerName { index }); + } + if !names.insert(name.clone()) { + return Err(EvalConfigError::DuplicateScorerName { name }); + } + validated.push(Box::new(NamedScorer { name, scorer })); + } + Ok(Self { + inner: Arc::new(DefinitionInner { + task, + scorers: validated, + }), + }) + } +} + +impl EvalDefinition { + /// Scorer names in definition order, fixed at construction. + pub fn scorer_names(&self) -> impl ExactSizeIterator { + self.inner.scorers.iter().map(|scorer| scorer.name()) + } + + pub(crate) fn task(&self) -> &dyn Task { + self.inner.task.as_ref() + } + + pub(crate) fn scorers(&self) -> &[Box>] { + &self.inner.scorers + } +} + +struct NamedScorer { + name: String, + scorer: Box>, +} + +#[async_trait] +impl Scorer for NamedScorer { + fn name(&self) -> &str { + &self.name + } + + async fn score(&self, args: ScorerArgs<'_, I, O>) -> anyhow::Result { + self.scorer.score(args).await + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::eval::{case_runner::CaseRunner, task, Case, CaseError, SimpleFnTask}; + use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; + + // No Clone, Serialize, equality or runtime is needed for computation. + struct Input(usize); + struct Output(usize); + + #[test] + fn cloned_definitions_share_callbacks_with_independent_case_state() { + let calls = Arc::new(AtomicUsize::new(0)); + let barrier = Arc::new(tokio::sync::Barrier::new(2)); + let definition = EvalDefinition::new( + Box::new(task({ + let calls = calls.clone(); + move |input: &Input, hooks: &crate::eval::TaskHooks<'_, Input, Output>| { + let barrier = barrier.clone(); + let calls = calls.clone(); + Box::pin(async move { + calls.fetch_add(1, Ordering::SeqCst); + hooks.add_tag(format!("case-{}", input.0)); + barrier.wait().await; + Ok(Output(input.0 + hooks.expected.unwrap().0)) + }) + } + })), + vec![], + ) + .unwrap(); + let other = definition.clone(); + drop(definition); + let first = other.clone(); + let case = |input, expected| Case { + input: Input(input), + expected: Some(Output(expected)), + metadata: None, + tags: None, + id: None, + xact_id: None, + created: None, + }; + let (one, two) = futures::executor::block_on(async { + futures::join!( + CaseRunner::run(case(1, 10), first.task(), first.scorers(), None), + CaseRunner::run(case(2, 20), other.task(), other.scorers(), None), + ) + }); + assert_eq!(calls.load(Ordering::SeqCst), 2); + assert_eq!(one.output.unwrap().0, 11); + assert_eq!(two.output.unwrap().0, 22); + assert_eq!(one.tags, Some(vec!["case-1".into()])); + assert_eq!(two.tags, Some(vec!["case-2".into()])); + // Reuse the same callbacks after both prior invocations have completed. + let (one, two) = futures::executor::block_on(async { + futures::join!( + CaseRunner::run(case(3, 30), first.task(), first.scorers(), None), + CaseRunner::run(case(4, 40), other.task(), other.scorers(), None), + ) + }); + assert_eq!(one.output.unwrap().0, 33); + assert_eq!(two.output.unwrap().0, 44); + assert_eq!(calls.load(Ordering::SeqCst), 4); + } + + struct ChangingName(Arc); + #[async_trait] + impl Scorer for ChangingName { + fn name(&self) -> &str { + if self.0.load(Ordering::SeqCst) { + "changed" + } else { + "original" + } + } + async fn score(&self, _: ScorerArgs<'_, usize, usize>) -> anyhow::Result { + anyhow::bail!("scoring failed") + } + } + + #[test] + fn scorer_identity_is_captured_once_for_every_run() { + let changed = Arc::new(AtomicBool::new(false)); + let definition = EvalDefinition::new( + Box::new(SimpleFnTask::new(|input: &usize| Ok(*input))), + vec![Box::new(ChangingName(changed.clone()))], + ) + .unwrap(); + changed.store(true, Ordering::SeqCst); + assert_eq!(definition.scorer_names().collect::>(), ["original"]); + for definition in [definition.clone(), definition] { + let result = futures::executor::block_on(CaseRunner::run( + Case { + input: 1, + ..Default::default() + }, + definition.task(), + definition.scorers(), + None, + )); + assert!( + matches!(&result.errors[..], [CaseError::Scorer {name, source}] if name == "original" && source.to_string() == "scoring failed") + ); + } + } +} diff --git a/src/eval/mod.rs b/src/eval/mod.rs index c785cd6..22d02cc 100644 --- a/src/eval/mod.rs +++ b/src/eval/mod.rs @@ -29,15 +29,19 @@ //! }, //! ]; //! +//! // Reuse this definition with a fresh source and options for every run. +//! let definition = EvalDefinition::new( +//! Box::new(task(|input: &String, _hooks| Box::pin(async move { +//! Ok(input.to_uppercase()) +//! }))), +//! vec![Box::new(ExactMatch::new())], +//! )?; //! let result = evaluator //! .run( //! EvalOpts::builder() //! .name("uppercase-eval".to_string()) //! .source(cases.into()) -//! .task(Box::new(task(|input: &String, _hooks| Box::pin(async move { -//! Ok(input.to_uppercase()) -//! })))) -//! .scorers(vec![Box::new(ExactMatch::new())]) +//! .definition(definition.clone()) //! .build(), //! ) //! .await?; @@ -49,6 +53,7 @@ mod bt_runner; mod case_runner; +mod definition; mod error; mod runner; mod scorer; @@ -56,6 +61,7 @@ mod source; mod task; mod types; +pub use definition::EvalDefinition; pub use error::{CaseError, EvalConfigError, EvalRunError, EvalRunFailure}; // Re-export public API diff --git a/src/eval/runner.rs b/src/eval/runner.rs index 19453c4..8263a28 100644 --- a/src/eval/runner.rs +++ b/src/eval/runner.rs @@ -8,11 +8,9 @@ use crate::{BraintrustClient, Result, SpanHandle}; use super::{ case_runner::CaseRunner, - scorer::Scorer, source::CaseSource, - task::Task, types::{Case, EvalResult, EvalSummary}, - EvalConfigError, EvalRunError, EvalRunFailure, + EvalConfigError, EvalDefinition, EvalRunError, EvalRunFailure, }; /// Options for running an evaluation @@ -22,10 +20,8 @@ pub struct EvalOpts { pub name: String, /// Owned source providing test cases pub source: Box>, - /// Task to execute on each input - pub task: Box>, - /// Scorers to evaluate outputs - pub scorers: Vec>>, + /// Reusable task and ordered scorers + pub definition: EvalDefinition, /// Braintrust project to log results to pub project_name: Option, /// Optional experiment name for Braintrust logging @@ -48,18 +44,6 @@ impl EvalOpts { if self.max_concurrency == 0 { return Err(EvalConfigError::ZeroConcurrency); } - let mut names = std::collections::HashSet::new(); - for (index, scorer) in self.scorers.iter().enumerate() { - let name = scorer.name(); - if name.trim().is_empty() { - return Err(EvalConfigError::EmptyScorerName { index }); - } - if !names.insert(name) { - return Err(EvalConfigError::DuplicateScorerName { - name: name.to_owned(), - }); - } - } Ok(()) } } @@ -158,20 +142,12 @@ where let index = case_count - 1; let client = Arc::clone(&self.client); let root_span = root_span.clone(); - let task = &opts.task; - let scorers = &opts.scorers; + let definition = &opts.definition; let quiet = opts.quiet; futures.push(async move { - let result = Self::process_case( - case, - task.as_ref(), - scorers, - &root_span, - &client, - index + 1, - quiet, - ) - .await; + let result = + Self::process_case(case, definition, &root_span, &client, index + 1, quiet) + .await; (index, result) }); if futures.len() >= opts.max_concurrency { @@ -255,8 +231,7 @@ where /// Process a single case async fn process_case( case: Case, - task: &dyn Task, - scorers: &[Box>], + definition: &EvalDefinition, _root_span: &SpanHandle, client: &BraintrustClient, case_num: usize, @@ -290,7 +265,13 @@ where } let initial_write = case_span.log(log_builder.build().unwrap()); - let mut result = CaseRunner::run(case, task, scorers, Some(&case_span)).await; + let mut result = CaseRunner::run( + case, + definition.task(), + definition.scorers(), + Some(&case_span), + ) + .await; result.span_id = case_span.export()?.span_id; // Recording adapts the completed outcome; computation never waits on @@ -378,7 +359,9 @@ where #[cfg(test)] mod tests { use super::*; - use crate::eval::{scorer::ExactMatch, source::VecCaseSource, task::SimpleFnTask, types::Case}; + use crate::eval::{ + scorer::ExactMatch, source::VecCaseSource, task::SimpleFnTask, types::Case, Scorer, Task, + }; // Note: Full integration tests require a BraintrustClient setup // These tests verify type correctness and compilation @@ -399,8 +382,7 @@ mod tests { let _opts = EvalOpts::builder() .name("test-eval".to_string()) .source(dataset) - .task(task) - .scorers(scorers) + .definition(crate::eval::EvalDefinition::new(task, scorers).unwrap()) .build(); } } diff --git a/tests/eval_callables.rs b/tests/eval_callables.rs index 9b6f47a..5ce314e 100644 --- a/tests/eval_callables.rs +++ b/tests/eval_callables.rs @@ -1,8 +1,8 @@ use async_trait::async_trait; use braintrust_sdk_rust::{ eval::{ - scorer, task, Case, CaseSource, EvalConfigError, EvalOpts, Evaluator, ExactMatch, Score, - Scorer, ScorerArgs, Scores, SimpleFnTask, Task, TaskHooks, + scorer, task, Case, CaseSource, EvalConfigError, EvalDefinition, EvalOpts, Evaluator, + ExactMatch, Score, Scorer, ScorerArgs, Scores, SimpleFnTask, Task, TaskHooks, }, BraintrustClient, }; @@ -180,13 +180,20 @@ async fn invalid_concurrency_and_names_fail_before_login_recording_or_source_acc as Box> }) .collect(); + let definition = EvalDefinition::new( + Box::new(SimpleFnTask::new(|input: &usize| Ok(*input))), + scorers, + ); + if concurrency != 0 { + assert!(matches!(definition, Err(actual) if actual == expected)); + continue; + } let opts = EvalOpts::builder() .name("invalid".into()) .source(Box::new(UntouchedSource(accessed.clone())) as Box>) - .task(Box::new(SimpleFnTask::new(|input: &usize| Ok(*input)))) - .scorers(scorers) + .definition(definition.unwrap()) .max_concurrency(concurrency) .quiet(true) .build(); @@ -217,17 +224,12 @@ fn duplicate_names_are_rejected_across_custom_tuple_and_builtin_scorers() { Box::pin(async { Ok(vec![]) }) })), ] { - let opts = EvalOpts::builder() - .name("duplicate".into()) - .source(Vec::>::new().into()) - .task(Box::new(SimpleFnTask::new(|input: &usize| Ok(*input)))) - .scorers(vec![Box::new(ExactMatch::new()), callback]) - .build(); - assert_eq!( - opts.validate(), - Err(EvalConfigError::DuplicateScorerName { - name: "exact_match".into() - }) + let definition = EvalDefinition::new( + Box::new(SimpleFnTask::new(|input: &usize| Ok(*input))), + vec![Box::new(ExactMatch::new()), callback], + ); + assert!( + matches!(definition, Err(EvalConfigError::DuplicateScorerName { name }) if name == "exact_match") ); } } diff --git a/tests/eval_test.rs b/tests/eval_test.rs index e139f1b..0c6fdc5 100644 --- a/tests/eval_test.rs +++ b/tests/eval_test.rs @@ -81,8 +81,7 @@ async fn test_basic_eval_flow() { EvalOpts::builder() .name("test-basic-eval".to_string()) .source(cases.into()) - .task(Box::new(callback)) - .scorers(scorers) + .definition(EvalDefinition::new(Box::new(callback), scorers).unwrap()) .quiet(true) .build(), ) @@ -191,8 +190,7 @@ fn test_eval_opts_builder() { let _opts = EvalOpts::builder() .name("test-eval".to_string()) .source(dataset) - .task(task) - .scorers(scorers) + .definition(EvalDefinition::new(task, scorers).unwrap()) .max_concurrency(10) .quiet(true) .build(); @@ -221,10 +219,13 @@ async fn recorded_scorer_failure_keeps_valid_scores_and_finishes_the_case() { }] .into(), ) - .task(Box::new(SimpleFnTask::new(|input: &String| { - Ok(input.to_uppercase()) - }))) - .scorers(scorers) + .definition( + EvalDefinition::new( + Box::new(SimpleFnTask::new(|input: &String| Ok(input.to_uppercase()))), + scorers, + ) + .unwrap(), + ) .quiet(true) .build(), ) @@ -347,8 +348,7 @@ async fn source_failure_drains_active_cases_preserves_provenance_and_finishes_re started, failed: failed.clone(), }) as Box>) - .task(Box::new(callback)) - .scorers(vec![]) + .definition(EvalDefinition::new(Box::new(callback), vec![]).unwrap()) .max_concurrency(3) .quiet(true) .build(); @@ -430,10 +430,15 @@ async fn failure_before_the_first_case_returns_an_empty_partial_run_and_finalize .source( Box::new(IterCaseSource::new(items.into_iter())) as Box> ) - .task(Box::new(SimpleFnTask::new( - |_: &String| -> anyhow::Result { panic!("no case was read") }, - ))) - .scorers(vec![]) + .definition( + EvalDefinition::new( + Box::new(SimpleFnTask::new(|_: &String| -> anyhow::Result { + panic!("no case was read") + })), + vec![], + ) + .unwrap(), + ) .quiet(true) .build(); let error = Evaluator::new(client).run(opts).await.unwrap_err(); @@ -514,8 +519,7 @@ async fn recording_failure_stops_new_cases_but_drains_already_scheduled_cases() .collect::>>() .into(), ) - .task(Box::new(callback)) - .scorers(vec![]) + .definition(EvalDefinition::new(Box::new(callback), vec![]).unwrap()) .max_concurrency(2) .quiet(true) .build(); @@ -553,3 +557,60 @@ async fn recording_failure_stops_new_cases_but_drains_already_scheduled_cases() assert_eq!(partial.failed_cases, 2); assert_eq!(partial.successful_cases, 0); } + +#[tokio::test] +async fn reusable_definition_keeps_recorded_runs_and_sources_independent() { + use std::sync::{ + atomic::{AtomicUsize, Ordering}, + Arc, + }; + let (client, _server) = recording_client().await; + let evaluator = Evaluator::new(client); + let calls = Arc::new(AtomicUsize::new(0)); + let definition = EvalDefinition::new( + Box::new(task({ + let calls = calls.clone(); + move |input: &usize, hooks| { + calls.fetch_add(1, Ordering::SeqCst); + hooks.add_tag(format!("input-{input}")); + Box::pin(async move { Ok(*input * 2) }) + } + })), + vec![Box::new(ExactMatch::new())], + ) + .unwrap(); + let options = |name: &str, input, expected| { + EvalOpts::builder() + .name(name.into()) + .source( + vec![Case { + input, + expected: Some(expected), + ..Default::default() + }] + .into(), + ) + .definition(definition.clone()) + .quiet(true) + .build() + }; + let (first, second) = tokio::join!( + evaluator.run(options("first", 2, 4)), + evaluator.run(options("second", 3, 100)), + ); + let first = first.unwrap(); + let second = second.unwrap(); + assert_eq!(first.name, "first"); + assert_eq!(second.name, "second"); + assert_eq!(first.results[0].output, Some(4)); + assert_eq!(second.results[0].output, Some(6)); + assert_eq!(first.results[0].scores[0].score, 1.0); + assert_eq!(second.results[0].scores[0].score, 0.0); + assert_eq!(first.results[0].tags, Some(vec!["input-2".into()])); + assert_eq!(second.results[0].tags, Some(vec!["input-3".into()])); + assert_ne!(first.results[0].span_id, second.results[0].span_id); + let later = evaluator.run(options("later", 5, 10)).await.unwrap(); + assert_eq!(later.total_cases, 1); + assert_eq!(later.results[0].output, Some(10)); + assert_eq!(calls.load(Ordering::SeqCst), 3); +}