Separate reusable evaluation definitions from run options

examples/eval_async.rs

@@ -1,5 +1,4 @@
 use braintrust_sdk_rust::eval::*;
-use braintrust_sdk_rust::*;
 
 /// Demonstrates using BtEvalRunner, which is compatible with the `bt eval` CLI.
 ///
@@ -10,7 +9,7 @@ use braintrust_sdk_rust::*;
 ///   - BT_EVAL_SSE_SOCK=...  — send SSE events over a Unix socket
 ///   - BT_EVAL_SSE_ADDR=...  — send SSE events over TCP
 #[tokio::main]
-async fn main() -> Result<()> {
+async fn main() -> anyhow::Result<()> {
     // BtEvalRunner reads BT_EVAL_* env vars automatically.
     // In non-list mode it creates a BraintrustClient from the environment.
     let mut runner = BtEvalRunner::from_env().await?;
@@ -39,30 +38,35 @@ async fn main() -> Result<()> {
         },
     ];
 
+    // Share callbacks across runs; each run supplies its own source and options.
+    let definition = EvalDefinition::new(
+        Box::new(task(|input: &String, _hooks| {
+            Box::pin(async move { Ok(input.to_uppercase()) })
+        })),
+        vec![
+            Box::new(ExactMatch::new()),
+            Box::new(LevenshteinScorer::new()),
+            Box::new(scorer(
+                "nonempty",
+                |args: ScorerArgs<'_, String, String>| {
+                    Box::pin(async move {
+                        Ok(vec![Score::new(
+                            "nonempty",
+                            if args.output.is_empty() { 0.0 } else { 1.0 },
+                        )])
+                    })
+                },
+            )),
+        ],
+    )?;
+
     runner
         .eval(
             "my-project",
             EvalOpts::builder()
                 .name("uppercase-eval".to_string())
                 .source(cases.into())
-                .task(Box::new(task(|input: &String, _hooks| {
-                    Box::pin(async move { Ok(input.to_uppercase()) })
-                })))
-                .scorers(vec![
-                    Box::new(ExactMatch::new()),
-                    Box::new(LevenshteinScorer::new()),
-                    Box::new(scorer(
-                        "nonempty",
-                        |args: ScorerArgs<'_, String, String>| {
-                            Box::pin(async move {
-                                Ok(vec![Score::new(
-                                    "nonempty",
-                                    if args.output.is_empty() { 0.0 } else { 1.0 },
-                                )])
-                            })
-                        },
-                    )),
-                ])
+                .definition(definition.clone())
                 .max_concurrency(2)
                 .build(),
         )

examples/eval_sync.rs

@@ -39,12 +39,12 @@ fn main() -> anyhow::Result<()> {
     // Run evaluation synchronously
     println!("Starting synchronous evaluation...\n");
 
+    let definition = EvalDefinition::new(Box::new(task), scorers)?;
     let result = evaluator.run(
         EvalOpts::builder()
             .name("sync-uppercase-eval".to_string())
             .source(cases.into())
-            .task(Box::new(task))
-            .scorers(scorers)
+            .definition(definition.clone())
             .build(),
     )?;
 

src/eval/bt_runner.rs

@@ -186,9 +186,14 @@ fn create_sse_writer() -> Option<Arc<SseWriter>> {
 /// ```no_run
 /// use braintrust_sdk_rust::eval::*;
 ///
-/// # async fn example() -> braintrust_sdk_rust::Result<()> {
+/// # async fn example() -> anyhow::Result<()> {
 /// let mut runner = BtEvalRunner::from_env().await?;
 ///
+/// let definition = EvalDefinition::new(
+///     Box::new(SimpleFnTask::new(|input: &String| Ok(input.to_uppercase()))),
+///     vec![Box::new(ExactMatch::new())],
+/// )?;
+///
 /// runner.eval(
 ///     "my-project",
 ///     EvalOpts::builder()
@@ -198,8 +203,7 @@ fn create_sse_writer() -> Option<Arc<SseWriter>> {
 ///             expected: Some("HELLO".to_string()),
 ///             ..Default::default()
 ///         }].into())
-///         .task(Box::new(SimpleFnTask::new(|input: &String| Ok(input.to_uppercase()))))
-///         .scorers(vec![Box::new(ExactMatch::new())])
+///         .definition(definition.clone())
 ///         .build(),
 /// ).await?;
 ///

src/eval/definition.rs

@@ -0,0 +1,197 @@
+//! Reusable computation shared by independent evaluation runs.
+use std::sync::Arc;
+
+use async_trait::async_trait;
+
+use super::{EvalConfigError, Scorer, ScorerArgs, Scores, Task};
+
+/// A validated task and ordered set of scorers, independent of any client or run.
+///
+/// Cloning shares the callbacks without cloning inputs or outputs. Callback
+/// state is shared across runs; sources, hooks and results belong to each run.
+/// Callbacks must therefore support concurrent invocation.
+pub struct EvalDefinition<I, O> {
+    inner: Arc<DefinitionInner<I, O>>,
+}
+
+struct DefinitionInner<I, O> {
+    task: Box<dyn Task<I, O>>,
+    scorers: Vec<Box<dyn Scorer<I, O>>>,
+}
+
+impl<I, O> Clone for EvalDefinition<I, O> {
+    fn clone(&self) -> Self {
+        Self {
+            inner: Arc::clone(&self.inner),
+        }
+    }
+}
+
+impl<I: Send + Sync + 'static, O: Send + Sync + 'static> EvalDefinition<I, O> {
+    /// Validate scorer names and retain the callbacks for reuse.
+    ///
+    /// Names are captured once, including for custom scorers whose `name()`
+    /// could change through interior state. Empty or duplicate names are rejected.
+    pub fn new(
+        task: Box<dyn Task<I, O>>,
+        scorers: Vec<Box<dyn Scorer<I, O>>>,
+    ) -> Result<Self, EvalConfigError> {
+        let mut names = std::collections::HashSet::new();
+        let mut validated: Vec<Box<dyn Scorer<I, O>>> = Vec::with_capacity(scorers.len());
+        for (index, scorer) in scorers.into_iter().enumerate() {
+            let name = scorer.name().to_owned();
+            if name.trim().is_empty() {
+                return Err(EvalConfigError::EmptyScorerName { index });
+            }
+            if !names.insert(name.clone()) {
+                return Err(EvalConfigError::DuplicateScorerName { name });
+            }
+            validated.push(Box::new(NamedScorer { name, scorer }));
+        }
+        Ok(Self {
+            inner: Arc::new(DefinitionInner {
+                task,
+                scorers: validated,
+            }),
+        })
+    }
+}
+
+impl<I, O> EvalDefinition<I, O> {
+    /// Scorer names in definition order, fixed at construction.
+    pub fn scorer_names(&self) -> impl ExactSizeIterator<Item = &str> {
+        self.inner.scorers.iter().map(|scorer| scorer.name())
+    }
+
+    pub(crate) fn task(&self) -> &dyn Task<I, O> {
+        self.inner.task.as_ref()
+    }
+
+    pub(crate) fn scorers(&self) -> &[Box<dyn Scorer<I, O>>] {
+        &self.inner.scorers
+    }
+}
+
+struct NamedScorer<I, O> {
+    name: String,
+    scorer: Box<dyn Scorer<I, O>>,
+}
+
+#[async_trait]
+impl<I: Send + Sync, O: Send + Sync> Scorer<I, O> for NamedScorer<I, O> {
+    fn name(&self) -> &str {
+        &self.name
+    }
+
+    async fn score(&self, args: ScorerArgs<'_, I, O>) -> anyhow::Result<Scores> {
+        self.scorer.score(args).await
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::eval::{case_runner::CaseRunner, task, Case, CaseError, SimpleFnTask};
+    use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
+
+    // No Clone, Serialize, equality or runtime is needed for computation.
+    struct Input(usize);
+    struct Output(usize);
+
+    #[test]
+    fn cloned_definitions_share_callbacks_with_independent_case_state() {
+        let calls = Arc::new(AtomicUsize::new(0));
+        let barrier = Arc::new(tokio::sync::Barrier::new(2));
+        let definition = EvalDefinition::new(
+            Box::new(task({
+                let calls = calls.clone();
+                move |input: &Input, hooks: &crate::eval::TaskHooks<'_, Input, Output>| {
+                    let barrier = barrier.clone();
+                    let calls = calls.clone();
+                    Box::pin(async move {
+                        calls.fetch_add(1, Ordering::SeqCst);
+                        hooks.add_tag(format!("case-{}", input.0));
+                        barrier.wait().await;
+                        Ok(Output(input.0 + hooks.expected.unwrap().0))
+                    })
+                }
+            })),
+            vec![],
+        )
+        .unwrap();
+        let other = definition.clone();
+        drop(definition);
+        let first = other.clone();
+        let case = |input, expected| Case {
+            input: Input(input),
+            expected: Some(Output(expected)),
+            metadata: None,
+            tags: None,
+            id: None,
+            xact_id: None,
+            created: None,
+        };
+        let (one, two) = futures::executor::block_on(async {
+            futures::join!(
+                CaseRunner::run(case(1, 10), first.task(), first.scorers(), None),
+                CaseRunner::run(case(2, 20), other.task(), other.scorers(), None),
+            )
+        });
+        assert_eq!(calls.load(Ordering::SeqCst), 2);
+        assert_eq!(one.output.unwrap().0, 11);
+        assert_eq!(two.output.unwrap().0, 22);
+        assert_eq!(one.tags, Some(vec!["case-1".into()]));
+        assert_eq!(two.tags, Some(vec!["case-2".into()]));
+        // Reuse the same callbacks after both prior invocations have completed.
+        let (one, two) = futures::executor::block_on(async {
+            futures::join!(
+                CaseRunner::run(case(3, 30), first.task(), first.scorers(), None),
+                CaseRunner::run(case(4, 40), other.task(), other.scorers(), None),
+            )
+        });
+        assert_eq!(one.output.unwrap().0, 33);
+        assert_eq!(two.output.unwrap().0, 44);
+        assert_eq!(calls.load(Ordering::SeqCst), 4);
+    }
+
+    struct ChangingName(Arc<AtomicBool>);
+    #[async_trait]
+    impl Scorer<usize, usize> for ChangingName {
+        fn name(&self) -> &str {
+            if self.0.load(Ordering::SeqCst) {
+                "changed"
+            } else {
+                "original"
+            }
+        }
+        async fn score(&self, _: ScorerArgs<'_, usize, usize>) -> anyhow::Result<Scores> {
+            anyhow::bail!("scoring failed")
+        }
+    }
+
+    #[test]
+    fn scorer_identity_is_captured_once_for_every_run() {
+        let changed = Arc::new(AtomicBool::new(false));
+        let definition = EvalDefinition::new(
+            Box::new(SimpleFnTask::new(|input: &usize| Ok(*input))),
+            vec![Box::new(ChangingName(changed.clone()))],
+        )
+        .unwrap();
+        changed.store(true, Ordering::SeqCst);
+        assert_eq!(definition.scorer_names().collect::<Vec<_>>(), ["original"]);
+        for definition in [definition.clone(), definition] {
+            let result = futures::executor::block_on(CaseRunner::run(
+                Case {
+                    input: 1,
+                    ..Default::default()
+                },
+                definition.task(),
+                definition.scorers(),
+                None,
+            ));
+            assert!(
+                matches!(&result.errors[..], [CaseError::Scorer {name, source}] if name == "original" && source.to_string() == "scoring failed")
+            );
+        }
+    }
+}

src/eval/mod.rs

@@ -29,15 +29,19 @@
 //!     },
 //! ];
 //!
+//! // Reuse this definition with a fresh source and options for every run.
+//! let definition = EvalDefinition::new(
+//!     Box::new(task(|input: &String, _hooks| Box::pin(async move {
+//!         Ok(input.to_uppercase())
+//!     }))),
+//!     vec![Box::new(ExactMatch::new())],
+//! )?;
 //! let result = evaluator
 //!     .run(
 //!         EvalOpts::builder()
 //!             .name("uppercase-eval".to_string())
 //!             .source(cases.into())
-//!             .task(Box::new(task(|input: &String, _hooks| Box::pin(async move {
-//!                 Ok(input.to_uppercase())
-//!             }))))
-//!             .scorers(vec![Box::new(ExactMatch::new())])
+//!             .definition(definition.clone())
 //!             .build(),
 //!     )
 //!     .await?;
@@ -49,13 +53,15 @@
 
 mod bt_runner;
 mod case_runner;
+mod definition;
 mod error;
 mod runner;
 mod scorer;
 mod source;
 mod task;
 mod types;
 
+pub use definition::EvalDefinition;
 pub use error::{CaseError, EvalConfigError, EvalRunError, EvalRunFailure};
 
 // Re-export public API