Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
55 changes: 55 additions & 0 deletions crates/llm-runtime/tests/anthropic_messages_live.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1752,6 +1752,61 @@ async fn anthropic_messages_live_adapter_sees_oversized_image() {
assert!(answer.contains("blue"), "expected blue, got {answer:?}");
}

// Advertise the real built-in definitions so provider schema validation runs
// through the same materialization and rejection mapping as a hosted session.
// No tools are executed; each request isolates one potentially invalid schema.
async fn assert_live_builtin_input_schema_accepted(tool_id: &str) {
use harness::CoreAgentLlm as _;

let blobs = Arc::new(InMemoryBlobStore::new());
let question = user_entry(
1,
text_blob(&blobs, "Reply with exactly OK. Do not call any tools.").await,
);
let mut request = intent_request("live-anthropic-builtin-input-schema", vec![question]);
request.tools = vec![tools::definitions::register(
tool_id,
Default::default(),
harness::ToolParallelism::ParallelSafe,
Default::default(),
)];
let model = request.model.model.clone();
let adapter = Arc::new(AnthropicMessagesLlmAdapter::new(
retrying_anthropic_messages_client(live_client()),
blobs,
));
let runtime = llm_runtime::LlmRuntime::new(
llm_runtime::LlmAdapterRegistry::new()
.with_generation_adapter(ProviderApiKind::AnthropicMessages, adapter),
);

let result = runtime
.generate(generation_request(1, request))
.await
.unwrap_or_else(|error| {
panic!("{model} must accept the {tool_id} input schema: {error:?}")
});
assert_eq!(result.status, LlmGenerationStatus::Succeeded, "{tool_id}");
}

#[tokio::test(flavor = "current_thread")]
#[ignore = "requires ANTHROPIC_API_KEY (costs real money)"]
async fn anthropic_messages_live_accepts_builtin_input_schema_blob_put() {
assert_live_builtin_input_schema_accepted("blob.put").await;
}

#[tokio::test(flavor = "current_thread")]
#[ignore = "requires ANTHROPIC_API_KEY (costs real money)"]
async fn anthropic_messages_live_accepts_builtin_input_schema_env_write_file() {
assert_live_builtin_input_schema_accepted("env.write_file").await;
}

#[tokio::test(flavor = "current_thread")]
#[ignore = "requires ANTHROPIC_API_KEY (costs real money)"]
async fn anthropic_messages_live_accepts_builtin_input_schema_vfs_write_file() {
assert_live_builtin_input_schema_accepted("vfs.write_file").await;
}

/// A request the provider refuses crosses the runtime boundary as
/// `Rejected`, carrying the provider's own message rather than a wrapped
/// runtime error, so the run fails as `request_rejected`.
Expand Down
65 changes: 60 additions & 5 deletions crates/llm-runtime/tests/builtin_catalog_parity.rs
Original file line number Diff line number Diff line change
Expand Up @@ -75,7 +75,7 @@ fn registered(case: &str, api: &ProviderApiKind) -> Vec<ToolSpec> {
registered.tools.into_values().collect()
}

async fn fixture(api: ProviderApiKind, case: &str) -> Value {
async fn fixture(api: ProviderApiKind, case: &str, code_mode: bool) -> Value {
let blobs = InMemoryBlobStore::new();
let model = ModelSelection {
provider_id: if api == ProviderApiKind::AnthropicMessages {
Expand All @@ -92,9 +92,22 @@ async fn fixture(api: ProviderApiKind, case: &str) -> Value {
.into(),
api_kind: api.clone(),
};
let tools = registered(case, &api);
let mut tools = registered(case, &api);
let code_mode = code_mode.then(|| {
let presentation = harness::CodeModePresentation {
allowed_tools: tools.iter().map(|tool| tool.name.clone()).collect(),
..Default::default()
};
tools.push(tools::definitions::register(
"code.execute",
Default::default(),
harness::ToolParallelism::ParallelSafe,
Default::default(),
));
presentation
});
let request = LlmRequest {
code_mode: None,
code_mode,
model,
request_fingerprint: "sha256:catalog-parity".into(),
context: ContextSnapshot {
Expand Down Expand Up @@ -143,11 +156,53 @@ async fn builtin_requests_match_captured_provider_contracts() {
for entry in baseline {
let api: ProviderApiKind = serde_json::from_value(entry["api"].clone()).expect("API kind");
let case = entry["case"].as_str().expect("case");
let actual = fixture(api.clone(), case).await;
let actual = fixture(api.clone(), case, false).await;
assert_eq!(actual, entry["request"], "{api:?}: {case}");
}
}

#[tokio::test(flavor = "current_thread")]
async fn anthropic_builtin_inputs_avoid_top_level_combinators_with_and_without_code_mode() {
for case in [
"workspace",
"environment",
"canonical",
"one_shot",
"web",
"workflow",
] {
for code_mode in [false, true] {
let wire = fixture(ProviderApiKind::AnthropicMessages, case, code_mode).await;
let tools = wire["tools"].as_array().expect("advertised tools");
assert!(tools.iter().any(|tool| tool["name"] == "blob_put"));
for tool in tools {
let Some(schema) = tool.get("input_schema") else {
continue; // Provider-native tools have no custom input schema.
};
assert_eq!(schema["type"], "object", "{case}: {}", tool["name"]);
for keyword in ["oneOf", "anyOf", "allOf"] {
assert!(
schema.get(keyword).is_none(),
"{case}, code_mode={code_mode}: {} has top-level {keyword}",
tool["name"]
);
}
if matches!(
tool["name"].as_str(),
Some("blob_put" | "Write" | "VfsWrite" | "write_file" | "vfs_write_file")
) {
assert!(
tool["description"]
.as_str()
.unwrap()
.contains("exactly one")
);
}
}
}
}
}

/// Run explicitly after an intentional change to provider-visible tool definitions.
#[tokio::test(flavor = "current_thread")]
#[ignore = "regenerates the committed provider request fixture"]
Expand All @@ -156,7 +211,7 @@ async fn regenerate_builtin_provider_contracts() {
serde_json::from_str(include_str!("fixtures/builtin_catalogs.json")).expect("baseline");
for entry in &mut baseline {
let api = serde_json::from_value(entry["api"].clone()).expect("API kind");
entry["request"] = fixture(api, entry["case"].as_str().expect("case")).await;
entry["request"] = fixture(api, entry["case"].as_str().expect("case"), false).await;
}
std::fs::write(
concat!(
Expand Down
Loading
Loading