Own evaluation case sources and preserve partial run failures

examples/eval_async.rs

@@ -44,7 +44,7 @@ async fn main() -> Result<()> {
             "my-project",
             EvalOpts::builder()
                 .name("uppercase-eval".to_string())
-                .dataset(cases.into())
+                .source(cases.into())
                 .task(Box::new(task(|input: &String, _hooks| {
                     Box::pin(async move { Ok(input.to_uppercase()) })
                 })))

examples/eval_sync.rs

@@ -42,7 +42,7 @@ fn main() -> anyhow::Result<()> {
     let result = evaluator.run(
         EvalOpts::builder()
             .name("sync-uppercase-eval".to_string())
-            .dataset(cases.into())
+            .source(cases.into())
             .task(Box::new(task))
             .scorers(scorers)
             .build(),

src/eval/bt_runner.rs

@@ -10,7 +10,7 @@
 use std::io::Write;
 use std::sync::{Arc, Mutex};
 
-use serde::{de::DeserializeOwned, Serialize};
+use serde::Serialize;
 use serde_json::{json, Value};
 
 use crate::{BraintrustClient, BraintrustError, Result};
@@ -193,7 +193,7 @@ fn create_sse_writer() -> Option<Arc<SseWriter>> {
 ///     "my-project",
 ///     EvalOpts::builder()
 ///         .name("my-eval".to_string())
-///         .dataset(vec![Case {
+///         .source(vec![Case {
 ///             input: "hello".to_string(),
 ///             expected: Some("HELLO".to_string()),
 ///             ..Default::default()
@@ -288,8 +288,8 @@ impl BtEvalRunner {
     /// If the eval name does not pass the configured filters, it is skipped silently.
     pub async fn eval<I, O>(&mut self, project: &str, mut opts: EvalOpts<I, O>) -> Result<()>
     where
-        I: Serialize + DeserializeOwned + Send + Sync + Clone + 'static,
-        O: Serialize + DeserializeOwned + Send + Sync + PartialEq + Clone + 'static,
+        I: Serialize + Send + Sync + 'static,
+        O: Serialize + Send + Sync + 'static,
     {
         let name = opts.name.clone();
 
@@ -349,7 +349,7 @@ impl BtEvalRunner {
                     eprintln!("Evaluator '{}' failed: {}", name, e);
                 }
                 if self.config.terminate_on_failure {
-                    return Err(e);
+                    return Err(BraintrustError::Background(e.to_string()));
                 }
             }
         }

src/eval/error.rs

@@ -48,3 +48,94 @@ pub enum EvalConfigError {
     #[error("duplicate scorer name: {name}")]
     DuplicateScorerName { name: String },
 }
+
+/// Run-level failures retain their boundary and source location.
+/// Case indices are zero-based; None denotes run-level recording work.
+#[derive(Debug, thiserror::Error)]
+pub enum EvalRunFailure {
+    #[error(transparent)]
+    Config(#[from] EvalConfigError),
+    #[error("case source failed at index {case_index}: {source}")]
+    Source {
+        case_index: usize,
+        #[source]
+        source: super::SourceError,
+    },
+    #[error("recording failed for case {case_index:?}: {source}")]
+    Recording {
+        case_index: Option<usize>,
+        #[source]
+        source: crate::BraintrustError,
+    },
+    #[error(transparent)]
+    Execution(#[from] crate::ExecutionError),
+    #[error(transparent)]
+    Admission(#[from] crate::EnqueueError),
+}
+
+/// An incomplete run preserves finished case results and every run-level failure.
+/// Case computation errors remain in those results rather than aborting the run.
+pub struct EvalRunError<I, O> {
+    pub failures: Vec<EvalRunFailure>,
+    pub partial: Option<Box<super::EvalSummary<I, O>>>,
+}
+impl<I, O> EvalRunError<I, O> {
+    fn new(failure: EvalRunFailure) -> Self {
+        Self {
+            failures: vec![failure],
+            partial: None,
+        }
+    }
+}
+impl<I, O> std::fmt::Debug for EvalRunError<I, O> {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("EvalRunError")
+            .field("failures", &self.failures)
+            .field(
+                "completed_cases",
+                &self.partial.as_ref().map(|summary| summary.results.len()),
+            )
+            .finish()
+    }
+}
+impl<I, O> std::fmt::Display for EvalRunError<I, O> {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        if let Some(first) = self.failures.first() {
+            write!(f, "{first}")?;
+        }
+        if self.failures.len() > 1 {
+            write!(f, " ({} run failures)", self.failures.len())?;
+        }
+        Ok(())
+    }
+}
+impl<I, O> std::error::Error for EvalRunError<I, O> {
+    fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
+        self.failures
+            .first()
+            .map(|failure| failure as &dyn std::error::Error)
+    }
+}
+impl<I, O> From<EvalConfigError> for EvalRunError<I, O> {
+    fn from(error: EvalConfigError) -> Self {
+        Self::new(EvalRunFailure::Config(error))
+    }
+}
+impl<I, O> From<crate::BraintrustError> for EvalRunError<I, O> {
+    fn from(source: crate::BraintrustError) -> Self {
+        Self::new(EvalRunFailure::Recording {
+            case_index: None,
+            source,
+        })
+    }
+}
+impl<I, O> From<crate::ExecutionError> for EvalRunError<I, O> {
+    fn from(error: crate::ExecutionError) -> Self {
+        Self::new(EvalRunFailure::Execution(error))
+    }
+}
+impl<I, O> From<crate::EnqueueError> for EvalRunError<I, O> {
+    fn from(error: crate::EnqueueError) -> Self {
+        Self::new(EvalRunFailure::Admission(error))
+    }
+}

src/eval/mod.rs

@@ -13,7 +13,7 @@
 //! use braintrust_sdk_rust::*;
 //! use braintrust_sdk_rust::eval::*;
 //!
-//! # async fn example() -> Result<()> {
+//! # async fn example() -> anyhow::Result<()> {
 //! let client = BraintrustClient::builder()
 //!     .api_key("sk-...")
 //!     .build()
@@ -33,7 +33,7 @@
 //!     .run(
 //!         EvalOpts::builder()
 //!             .name("uppercase-eval".to_string())
-//!             .dataset(cases.into())
+//!             .source(cases.into())
 //!             .task(Box::new(task(|input: &String, _hooks| Box::pin(async move {
 //!                 Ok(input.to_uppercase())
 //!             }))))
@@ -49,19 +49,21 @@
 
 mod bt_runner;
 mod case_runner;
-mod dataset;
 mod error;
 mod runner;
 mod scorer;
+mod source;
 mod task;
 mod types;
 
-pub use error::{CaseError, EvalConfigError};
+pub use error::{CaseError, EvalConfigError, EvalRunError, EvalRunFailure};
 
 // Re-export public API
 pub use bt_runner::{summary_to_value, BtEvalRunner};
-pub use dataset::{Dataset, IterDataset, StreamDataset, VecDataset};
 pub use runner::{EvalOpts, Evaluator, SyncEvaluator};
 pub use scorer::{scorer, ExactMatch, LevenshteinScorer, NumericDiffScorer, Scorer, SyncFnScorer};
+pub use source::{
+    CaseSource, DatasetVersionRef, IterCaseSource, SourceError, StreamCaseSource, VecCaseSource,
+};
 pub use task::{task, SimpleFnTask, SyncFnTask, Task};
 pub use types::{Case, EvalResult, EvalSummary, Score, ScoreStats, ScorerArgs, Scores, TaskHooks};