Submit local and recorded evaluation runs eagerly

examples/eval_sync.rs

@@ -1,15 +1,11 @@
 use braintrust_sdk_rust::eval::*;
 use braintrust_sdk_rust::*;
+use std::future::IntoFuture;
 
 fn main() -> anyhow::Result<()> {
-    // Note: No #[tokio::main] needed! SyncEvaluator handles async internally
-
-    // Create Braintrust client
-    let rt = tokio::runtime::Runtime::new().map_err(|e| anyhow::anyhow!(e))?;
-    let client = rt.block_on(async { BraintrustClient::builder().build().await })?;
-
-    // Create sync evaluator
-    let evaluator = SyncEvaluator::new(client)?;
+    // The client's owned runtime drives work; this thread only waits for tokens.
+    let client = futures::executor::block_on(BraintrustClient::builder().build().into_future())?;
+    let evaluator = Evaluator::new(client);
 
     // Define test cases
     let cases = vec![
@@ -40,12 +36,16 @@ fn main() -> anyhow::Result<()> {
     println!("Starting synchronous evaluation...\n");
 
     let definition = EvalDefinition::new(Box::new(task), scorers)?;
-    let result = evaluator.run(
-        EvalOpts::builder()
-            .name("sync-uppercase-eval".to_string())
-            .source(cases.into())
-            .definition(definition.clone())
-            .build(),
+    let result = futures::executor::block_on(
+        evaluator
+            .run(
+                EvalOpts::builder()
+                    .name("sync-uppercase-eval".to_string())
+                    .source(cases.into())
+                    .definition(definition.clone())
+                    .build(),
+            )
+            .into_future(),
     )?;
 
     // Print results

src/eval/error.rs

@@ -43,6 +43,8 @@ impl Serialize for CaseError {
 pub enum EvalConfigError {
     #[error("evaluation concurrency must be greater than zero")]
     ZeroConcurrency,
+    #[error("recorded evaluations inherit their client runtime; omit the run runtime override")]
+    RecordedRuntime,
     #[error("scorer at index {index} must have a nonblank name")]
     EmptyScorerName { index: usize },
     #[error("duplicate scorer name: {name}")]
@@ -53,6 +55,8 @@ pub enum EvalConfigError {
 /// Case indices are zero-based; None denotes run-level recording work.
 #[derive(Debug, thiserror::Error)]
 pub enum EvalRunFailure {
+    #[error("failed to initialize evaluation runtime: {0}")]
+    Runtime(#[source] crate::BraintrustError),
     #[error(transparent)]
     Config(#[from] EvalConfigError),
     #[error("case source failed at index {case_index}: {source}")]
@@ -81,7 +85,7 @@ pub struct EvalRunError<I, O> {
     pub partial: Option<Box<super::EvalSummary<I, O>>>,
 }
 impl<I, O> EvalRunError<I, O> {
-    fn new(failure: EvalRunFailure) -> Self {
+    pub(crate) fn new(failure: EvalRunFailure) -> Self {
         Self {
             failures: vec![failure],
             partial: None,

src/eval/mod.rs

@@ -19,8 +19,6 @@
 //!     .build()
 //!     .await?;
 //!
-//! let evaluator = Evaluator::new(client);
-//!
 //! let cases = vec![
 //!     Case {
 //!         input: "hello".to_string(),
@@ -36,15 +34,15 @@
 //!     }))),
 //!     vec![Box::new(ExactMatch::new())],
 //! )?;
-//! let result = evaluator
-//!     .run(
+//! let completion = run(EvalRequest::recorded(
 //!         EvalOpts::builder()
 //!             .name("uppercase-eval".to_string())
 //!             .source(cases.into())
 //!             .definition(definition.clone())
 //!             .build(),
-//!     )
-//!     .await?;
+//!     client,
+//! ));
+//! let result = completion.await?;
 //!
 //! println!("Results: {:?}", result);
 //! # Ok(())
@@ -58,6 +56,7 @@ mod error;
 mod executor;
 mod recorder;
 mod reporter;
+mod run;
 mod runner;
 mod scorer;
 mod source;
@@ -69,7 +68,8 @@ pub use error::{CaseError, EvalConfigError, EvalRunError, EvalRunFailure};
 
 // Re-export public API
 pub use bt_runner::{summary_to_value, BtEvalRunner};
-pub use runner::{EvalOpts, Evaluator, SyncEvaluator};
+pub use run::{run, EvalRequest};
+pub use runner::{EvalOpts, Evaluator};
 pub use scorer::{scorer, ExactMatch, LevenshteinScorer, NumericDiffScorer, Scorer, SyncFnScorer};
 pub use source::{
     CaseSource, DatasetVersionRef, IterCaseSource, SourceError, StreamCaseSource, VecCaseSource,

src/eval/recorder.rs

@@ -15,19 +15,21 @@ use std::sync::Arc;
 type Write = Completion<DeliveryReceipt, WriteError>;
 
 pub(crate) struct BraintrustRecorder {
-    client: Arc<BraintrustClient>,
+    client: BraintrustClient,
+    submitter: Arc<dyn crate::internal::ports::SpanSubmitter>,
     root: SpanHandle,
     root_write: tokio::sync::Mutex<Option<Write>>,
 }
 impl BraintrustRecorder {
     pub async fn new(
-        client: Arc<BraintrustClient>,
+        client: BraintrustClient,
+        submitter: Arc<dyn crate::internal::ports::SpanSubmitter>,
         name: String,
         metadata: Option<Map<String, Value>>,
         tags: Option<Vec<String>>,
     ) -> Result<Self> {
         let root = client
-            .span_builder()
+            .span_builder_for_operation(submitter.clone())
             .await?
             .span_type(crate::SpanType::Eval)
             .build();
@@ -41,6 +43,7 @@ impl BraintrustRecorder {
         let root_write = tokio::sync::Mutex::new(Some(root.log(log.build().unwrap())));
         Ok(Self {
             client,
+            submitter,
             root,
             root_write,
         })
@@ -74,7 +77,7 @@ impl<I: Serialize + Send + Sync + 'static, O: Serialize + Send + Sync + 'static>
     async fn start(&self, index: usize, case: &Case<I, O>) -> Result<Box<dyn CaseRecording<I, O>>> {
         let span = self
             .client
-            .span_builder()
+            .span_builder_for_operation(self.submitter.clone())
             .await?
             .span_type(crate::SpanType::Eval)
             .build();

src/eval/run.rs

@@ -0,0 +1,128 @@
+//! Immediate submission of local and recorded evaluations.
+use super::{
+    executor::{EvalExecutor, EvalPlan},
+    recorder::BraintrustRecorder,
+    reporter::TerminalReporter,
+    EvalConfigError, EvalOpts, EvalRunError, EvalRunFailure, EvalSummary,
+};
+use crate::{internal::execution::ExecutionBinding, BraintrustClient, Completion};
+use serde::Serialize;
+
+type RunCompletion<I, O> = Completion<EvalSummary<I, O>, EvalRunError<I, O>>;
+type SubmitRecorded<I, O> = fn(BraintrustClient, EvalOpts<I, O>) -> RunCompletion<I, O>;
+
+/// An owned invocation of a reusable definition with its own source and options.
+///
+/// Local requests need no client or serialization. Recorded requests require
+/// serialization and inherit their client's execution binding and lifecycle.
+pub struct EvalRequest<I, O> {
+    options: EvalOpts<I, O>,
+    target: Target<I, O>,
+}
+enum Target<I, O> {
+    Local,
+    Recorded {
+        client: BraintrustClient,
+        submit: SubmitRecorded<I, O>,
+    },
+}
+impl<I, O> EvalRequest<I, O> {
+    /// Run without login, HTTP or recording, on `options.runtime` or an owned runtime.
+    pub fn local(options: EvalOpts<I, O>) -> Self {
+        Self {
+            options,
+            target: Target::Local,
+        }
+    }
+}
+impl<I: Serialize + Send + Sync + 'static, O: Serialize + Send + Sync + 'static> EvalRequest<I, O> {
+    /// Record with this client's runtime. Supplying `options.runtime` is an error.
+    pub fn recorded(options: EvalOpts<I, O>, client: BraintrustClient) -> Self {
+        Self {
+            options,
+            target: Target::Recorded {
+                client,
+                submit: recorded::<I, O>,
+            },
+        }
+    }
+}
+
+/// Submit an evaluation now; awaiting observes the admitted run's result.
+/// Dropping the token does not cancel accepted computation or recording.
+///
+/// ```
+/// use braintrust_sdk_rust::eval::{self, Case, EvalDefinition, EvalOpts, EvalRequest, SimpleFnTask};
+/// # fn main() -> anyhow::Result<()> {
+/// let definition = EvalDefinition::new(
+///     Box::new(SimpleFnTask::new(|input: &usize| Ok(input * 2))), vec![],
+/// )?;
+/// let options = EvalOpts::builder().name("local".into())
+///     .definition(definition).source(vec![Case { input: 3, ..Default::default() }].into())
+///     .quiet(true).build();
+/// let completion = eval::run(EvalRequest::local(options));
+/// assert!(completion.is_ok()); // admission already happened
+/// let summary = futures::executor::block_on(async { completion.await })?;
+/// assert_eq!(summary.results[0].output, Some(6));
+/// # Ok(())
+/// # }
+/// ```
+pub fn run<I: Send + Sync + 'static, O: Send + Sync + 'static>(
+    request: EvalRequest<I, O>,
+) -> RunCompletion<I, O> {
+    if let Err(error) = request.options.validate() {
+        return Completion::rejected(error.into());
+    }
+    match request.target {
+        Target::Local => {
+            let mut options = request.options;
+            let binding = match ExecutionBinding::new(options.runtime.take()) {
+                Ok(binding) => binding,
+                Err(source) => {
+                    return Completion::rejected(EvalRunError::new(EvalRunFailure::Runtime(source)))
+                }
+            };
+            binding.response(async move {
+                let mut reporter = TerminalReporter {
+                    quiet: options.quiet,
+                };
+                EvalExecutor::run(EvalPlan::new(options)?, None, &mut reporter).await
+            })
+        }
+        Target::Recorded { client, submit } => {
+            if request.options.runtime.is_some() {
+                return Completion::rejected(EvalConfigError::RecordedRuntime.into());
+            }
+            submit(client, request.options)
+        }
+    }
+}
+
+fn recorded<I: Serialize + Send + Sync + 'static, O: Serialize + Send + Sync + 'static>(
+    client: BraintrustClient,
+    options: EvalOpts<I, O>,
+) -> RunCompletion<I, O> {
+    let binding = client.execution_binding();
+    binding.response_try(|| {
+        let operation = client.span_operation()?;
+        Ok(async move {
+            let mut reporter = TerminalReporter {
+                quiet: options.quiet,
+            };
+            let recorder = BraintrustRecorder::new(
+                client,
+                operation.submitter.clone(),
+                options.name.clone(),
+                options.metadata.clone(),
+                options.tags.clone(),
+            )
+            .await?;
+            let result =
+                EvalExecutor::run(EvalPlan::new(options)?, Some(&recorder), &mut reporter).await;
+            drop(recorder);
+            // Revoke write permission before publishing, including escaped spans.
+            drop(operation);
+            result
+        })
+    })
+}

src/eval/runner.rs

@@ -3,7 +3,7 @@ use serde_json::{Map, Value};
 use std::marker::PhantomData;
 use std::sync::Arc;
 
-use crate::{BraintrustClient, Result};
+use crate::BraintrustClient;
 
 use super::{
     source::CaseSource, types::EvalSummary, EvalConfigError, EvalDefinition, EvalRunError,
@@ -32,6 +32,8 @@ pub struct EvalOpts<I, O> {
     /// Suppress progress output (default: false)
     #[builder(default = false)]
     pub quiet: bool,
+    /// Explicit runtime for local runs. Recorded runs inherit their client binding.
+    pub runtime: Option<tokio::runtime::Handle>,
 }
 
 impl<I, O> EvalOpts<I, O> {
@@ -44,7 +46,7 @@ impl<I, O> EvalOpts<I, O> {
     }
 }
 
-/// Async evaluator for running evaluations
+/// Client-bound convenience facade for recorded evaluations.
 pub struct Evaluator<I, O> {
     client: Arc<BraintrustClient>,
     _phantom: PhantomData<(I, O)>,
@@ -71,50 +73,16 @@ where
         }
     }
 
-    /// Run the evaluation
-    pub async fn run(
-        &self,
-        opts: EvalOpts<I, O>,
-    ) -> std::result::Result<EvalSummary<I, O>, EvalRunError<I, O>> {
-        let name = opts.name.clone();
-        let metadata = opts.metadata.clone();
-        let tags = opts.tags.clone();
-        let mut reporter = super::reporter::TerminalReporter { quiet: opts.quiet };
-        let plan = super::executor::EvalPlan::new(opts)?;
-        let recorder =
-            super::recorder::BraintrustRecorder::new(self.client.clone(), name, metadata, tags)
-                .await?;
-        super::executor::EvalExecutor::run(plan, Some(&recorder), &mut reporter).await
-    }
-}
-
-/// Sync evaluator (wraps async evaluator with Runtime::block_on)
-pub struct SyncEvaluator<I, O> {
-    evaluator: Evaluator<I, O>,
-    runtime: tokio::runtime::Runtime,
-}
-
-impl<I, O> SyncEvaluator<I, O>
-where
-    I: Serialize + Send + Sync + 'static,
-    O: Serialize + Send + Sync + 'static,
-{
-    /// Create a new sync evaluator
-    pub fn new(client: BraintrustClient) -> Result<Self> {
-        let runtime = tokio::runtime::Runtime::new()
-            .map_err(|e| crate::BraintrustError::Background(e.to_string()))?;
-        Ok(Self {
-            evaluator: Evaluator::new(client),
-            runtime,
-        })
-    }
-
-    /// Run the evaluation synchronously
+    /// Submit a recorded run immediately on the client binding.
+    /// Dropping the completion does not cancel accepted work.
     pub fn run(
         &self,
         opts: EvalOpts<I, O>,
-    ) -> std::result::Result<EvalSummary<I, O>, EvalRunError<I, O>> {
-        self.runtime.block_on(self.evaluator.run(opts))
+    ) -> crate::Completion<EvalSummary<I, O>, EvalRunError<I, O>> {
+        super::run(super::EvalRequest::recorded(
+            opts,
+            self.client.as_ref().clone(),
+        ))
     }
 }