From 3b130b9b6ce8cd2958bd2be7b55434845845d5e1 Mon Sep 17 00:00:00 2001 From: jariy17 Date: Fri, 28 Aug 2026 21:54:53 +0000 Subject: [PATCH 1/7] feat(project): add `project add evaluator code-based` Declarative code-based evaluators via projects. Mode is inferred from flags (mirrors CodeBasedConfigSchema managed XOR external): --lambda-arn -> external (BYO Lambda) --metric -> managed 3P (deepeval/autoevals), scaffolded neither -> managed empty stub you fill in Scaffolds app// from ported evaluator templates (python/deepeval/autoevals lambda), hardcodes codeLocation, and auto-wires additionalPolicies= [execution-role-policy.json]. Also enables `project remove evaluator`. --- .../execution-role-policy.json | 15 ++ .../autoevals-lambda/lambda_function.py | 37 +++ .../autoevals-lambda/pyproject.toml | 22 ++ .../execution-role-policy.json | 15 ++ .../deepeval-lambda/lambda_function.py | 29 ++ .../evaluators/deepeval-lambda/pyproject.toml | 19 ++ .../python-lambda/execution-role-policy.json | 10 + .../python-lambda/lambda_function.py | 19 ++ .../evaluators/python-lambda/pyproject.toml | 15 ++ src/core/project/manager.tsx | 23 +- src/core/project/templates/evaluator.ts | 37 +++ src/core/project/templates/types.ts | 2 + .../add/evaluator/code-based/index.test.ts | 255 ++++++++++++++++++ .../project/add/evaluator/code-based/index.ts | 168 ++++++++++++ src/handlers/project/add/evaluator/index.ts | 2 + src/handlers/project/remove/index.ts | 1 + src/handlers/project/types.ts | 7 + 17 files changed, 675 insertions(+), 1 deletion(-) create mode 100644 src/assets/evaluators/autoevals-lambda/execution-role-policy.json create mode 100644 src/assets/evaluators/autoevals-lambda/lambda_function.py create mode 100644 src/assets/evaluators/autoevals-lambda/pyproject.toml create mode 100644 src/assets/evaluators/deepeval-lambda/execution-role-policy.json create mode 100644 src/assets/evaluators/deepeval-lambda/lambda_function.py create mode 100644 src/assets/evaluators/deepeval-lambda/pyproject.toml create mode 100644 src/assets/evaluators/python-lambda/execution-role-policy.json create mode 100644 src/assets/evaluators/python-lambda/lambda_function.py create mode 100644 src/assets/evaluators/python-lambda/pyproject.toml create mode 100644 src/core/project/templates/evaluator.ts create mode 100644 src/handlers/project/add/evaluator/code-based/index.test.ts create mode 100644 src/handlers/project/add/evaluator/code-based/index.ts diff --git a/src/assets/evaluators/autoevals-lambda/execution-role-policy.json b/src/assets/evaluators/autoevals-lambda/execution-role-policy.json new file mode 100644 index 000000000..6b47af830 --- /dev/null +++ b/src/assets/evaluators/autoevals-lambda/execution-role-policy.json @@ -0,0 +1,15 @@ +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": ["logs:CreateLogGroup", "logs:CreateLogStream", "logs:PutLogEvents"], + "Resource": "arn:*:logs:*:*:log-group:/aws/lambda/*" + }, + { + "Effect": "Allow", + "Action": ["bedrock:InvokeModel"], + "Resource": "*" + } + ] +} diff --git a/src/assets/evaluators/autoevals-lambda/lambda_function.py b/src/assets/evaluators/autoevals-lambda/lambda_function.py new file mode 100644 index 000000000..410f056a6 --- /dev/null +++ b/src/assets/evaluators/autoevals-lambda/lambda_function.py @@ -0,0 +1,37 @@ +{{#if ModelProviderBedrock}} +import os + +# litellm's Bedrock provider reads AWS_REGION_NAME; Lambda only sets AWS_REGION/AWS_DEFAULT_REGION. +os.environ.setdefault("AWS_REGION_NAME", os.environ.get("AWS_REGION", "us-west-2")) + +from autoevals import {{ EvaluatorClass }}, init +from autoevals.litellm import LiteLLMClient + +from bedrock_agentcore.evaluation.custom_code_based_evaluators import ( + EvaluatorInput, + EvaluatorOutput, + custom_code_based_evaluator, +) +from bedrock_agentcore.evaluation.custom_code_based_evaluators.third_party.autoevals import AutoEvalsAdapter + +client = LiteLLMClient() +init(client=client, default_model="{{ Model }}") + +adapter = AutoEvalsAdapter(metric={{ EvaluatorClass }}(client=client, model="{{ Model }}"){{#if EvaluatorParams}}, {{{ EvaluatorParams }}}{{/if}}) +{{else}} +from autoevals import {{ EvaluatorClass }} + +from bedrock_agentcore.evaluation.custom_code_based_evaluators import ( + EvaluatorInput, + EvaluatorOutput, + custom_code_based_evaluator, +) +from bedrock_agentcore.evaluation.custom_code_based_evaluators.third_party.autoevals import AutoEvalsAdapter + +adapter = AutoEvalsAdapter(metric={{ EvaluatorClass }}({{#if Model}}model="{{ Model }}"{{/if}}){{#if EvaluatorParams}}, {{{ EvaluatorParams }}}{{/if}}) +{{/if}} + + +@custom_code_based_evaluator() +def handler(evaluator_input: EvaluatorInput, context) -> EvaluatorOutput: + return adapter(evaluator_input, context) diff --git a/src/assets/evaluators/autoevals-lambda/pyproject.toml b/src/assets/evaluators/autoevals-lambda/pyproject.toml new file mode 100644 index 000000000..b37442fd1 --- /dev/null +++ b/src/assets/evaluators/autoevals-lambda/pyproject.toml @@ -0,0 +1,22 @@ +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[project] +name = "{{ Name }}" +version = "0.1.0" +description = "AgentCore Code-Based Evaluator (Autoevals)" +requires-python = ">=3.10" +dependencies = [ + "bedrock-agentcore[autoevals]", + "autoevals>=0.0.80,<1.0.0", +{{#if ModelProviderBedrock}} + # autoevals grades via LiteLLMClient -> Bedrock (Converse); litellm replaces the openai judge + "litellm>=1.60,<1.85", +{{else}} + "openai>=1.0.0", +{{/if}} +] + +[tool.hatch.build.targets.wheel] +packages = ["."] diff --git a/src/assets/evaluators/deepeval-lambda/execution-role-policy.json b/src/assets/evaluators/deepeval-lambda/execution-role-policy.json new file mode 100644 index 000000000..6b47af830 --- /dev/null +++ b/src/assets/evaluators/deepeval-lambda/execution-role-policy.json @@ -0,0 +1,15 @@ +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": ["logs:CreateLogGroup", "logs:CreateLogStream", "logs:PutLogEvents"], + "Resource": "arn:*:logs:*:*:log-group:/aws/lambda/*" + }, + { + "Effect": "Allow", + "Action": ["bedrock:InvokeModel"], + "Resource": "*" + } + ] +} diff --git a/src/assets/evaluators/deepeval-lambda/lambda_function.py b/src/assets/evaluators/deepeval-lambda/lambda_function.py new file mode 100644 index 000000000..a89b22051 --- /dev/null +++ b/src/assets/evaluators/deepeval-lambda/lambda_function.py @@ -0,0 +1,29 @@ +import os + +os.environ.setdefault("DEEPEVAL_RESULTS_FOLDER", "/tmp/.deepeval") +os.environ.setdefault("DEEPEVAL_TELEMETRY_OPT_OUT", "YES") +os.chdir("/tmp") + +{{#if ModelProviderBedrock}} +from deepeval.models import AmazonBedrockModel +{{/if}} +from deepeval.metrics import {{ EvaluatorClass }} + +from bedrock_agentcore.evaluation.custom_code_based_evaluators import ( + EvaluatorInput, + EvaluatorOutput, + custom_code_based_evaluator, +) +from bedrock_agentcore.evaluation.custom_code_based_evaluators.third_party.deepeval import DeepEvalAdapter + +{{#if ModelProviderBedrock}} +model = AmazonBedrockModel(model="{{ Model }}", region=os.environ.get("AWS_REGION", "us-west-2")) +adapter = DeepEvalAdapter(metric={{ EvaluatorClass }}(model=model{{#if EvaluatorParams}}, {{{ EvaluatorParams }}}{{/if}})) +{{else}} +adapter = DeepEvalAdapter(metric={{ EvaluatorClass }}({{{ EvaluatorParams }}})) +{{/if}} + + +@custom_code_based_evaluator() +def handler(evaluator_input: EvaluatorInput, context) -> EvaluatorOutput: + return adapter(evaluator_input, context) diff --git a/src/assets/evaluators/deepeval-lambda/pyproject.toml b/src/assets/evaluators/deepeval-lambda/pyproject.toml new file mode 100644 index 000000000..7385ccfc2 --- /dev/null +++ b/src/assets/evaluators/deepeval-lambda/pyproject.toml @@ -0,0 +1,19 @@ +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[project] +name = "{{ Name }}" +version = "0.1.0" +description = "AgentCore Code-Based Evaluator (DeepEval)" +requires-python = ">=3.10" +dependencies = [ + "bedrock-agentcore[deepeval]", + "deepeval>=2.0.0,<3.0.0", +{{#if ModelProviderBedrock}} + "aiobotocore>=2.13.0", +{{/if}} +] + +[tool.hatch.build.targets.wheel] +packages = ["."] diff --git a/src/assets/evaluators/python-lambda/execution-role-policy.json b/src/assets/evaluators/python-lambda/execution-role-policy.json new file mode 100644 index 000000000..b3b98be42 --- /dev/null +++ b/src/assets/evaluators/python-lambda/execution-role-policy.json @@ -0,0 +1,10 @@ +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": ["logs:CreateLogGroup", "logs:CreateLogStream", "logs:PutLogEvents"], + "Resource": "arn:*:logs:*:*:log-group:/aws/lambda/*" + } + ] +} diff --git a/src/assets/evaluators/python-lambda/lambda_function.py b/src/assets/evaluators/python-lambda/lambda_function.py new file mode 100644 index 000000000..0f8bd5c6b --- /dev/null +++ b/src/assets/evaluators/python-lambda/lambda_function.py @@ -0,0 +1,19 @@ +from bedrock_agentcore.evaluation.custom_code_based_evaluators import ( + custom_code_based_evaluator, + EvaluatorInput, + EvaluatorOutput, +) + + +@custom_code_based_evaluator() +def handler(input: EvaluatorInput, context) -> EvaluatorOutput: + """Evaluate agent behavior with custom logic. + + Args: + input: Contains evaluation_level, session_spans, target_trace_id, target_span_id + + Returns: + EvaluatorOutput with value/label for success, or errorCode/errorMessage for failure. + """ + # TODO: Replace with your evaluation logic + return EvaluatorOutput(value=1.0, label="Pass", explanation="Evaluation passed") diff --git a/src/assets/evaluators/python-lambda/pyproject.toml b/src/assets/evaluators/python-lambda/pyproject.toml new file mode 100644 index 000000000..69ad99b43 --- /dev/null +++ b/src/assets/evaluators/python-lambda/pyproject.toml @@ -0,0 +1,15 @@ +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[project] +name = "{{ Name }}" +version = "0.1.0" +description = "AgentCore Code-Based Evaluator" +requires-python = ">=3.10" +dependencies = [ + "bedrock-agentcore>=1.6.0", +] + +[tool.hatch.build.targets.wheel] +packages = ["."] diff --git a/src/core/project/manager.tsx b/src/core/project/manager.tsx index 45b99d4d6..251d25c04 100644 --- a/src/core/project/manager.tsx +++ b/src/core/project/manager.tsx @@ -42,6 +42,7 @@ import { } from "./templates/export"; import { HarnessSpecSchema } from "../../projectSchemas/harness"; import { FsTreeNode } from "./templates/fsTree"; +import { getEvaluatorTemplateResolver } from "./templates/evaluator"; import { ProjectSpecSchema, type ManagedBy } from "../../projectSchemas/project"; import { ConfigBundleSchema } from "../../projectSchemas/config-bundle"; import { @@ -336,7 +337,27 @@ export class FsProjectManager implements ProjectManager { break; } case "evaluator": { - projectSpec.evaluators.push(parseResource(EvaluatorSchema, input.resourceConfig)); + const evaluator = parseResource(EvaluatorSchema, input.resourceConfig); + // Managed code-based evaluators ship generated Lambda source; external + // and llm-as-a-judge evaluators are spec-only. + if (input.scaffold) { + yield { message: "Scaffolding evaluator in project" }; + const outputPath = join(project.rootPath, "app", evaluator.name); + scaffoldedPaths.push(outputPath); + const resolver = getEvaluatorTemplateResolver({ + assetSource: this.assetSource, + templateRenderer: this.templateRenderer, + }); + const result = await resolver.resolve({ + evaluator, + assetDir: input.scaffold.assetDir, + context: input.scaffold.context, + }); + await result.tree.write(dirname(outputPath)); + projectSpec.evaluators.push(...(result.spec.evaluators ?? [])); + } else { + projectSpec.evaluators.push(evaluator); + } break; } case "gateway": diff --git a/src/core/project/templates/evaluator.ts b/src/core/project/templates/evaluator.ts new file mode 100644 index 000000000..5c091e657 --- /dev/null +++ b/src/core/project/templates/evaluator.ts @@ -0,0 +1,37 @@ +import { FsTreeNode } from "./fsTree"; +import type { AssetSource } from "../source"; +import type { Evaluator } from "../../../projectSchemas/evaluator"; +import type { TemplateRenderer, TemplateResolver } from "./types"; + +/** Inputs for scaffolding a managed code-based evaluator's Lambda source. */ +export type EvaluatorScaffoldInput = { + evaluator: Evaluator; + /** Template directory under src/assets/evaluators, e.g. "evaluators/deepeval-lambda". */ + assetDir: string; + /** Handlebars variables for the template (EvaluatorClass, Model, ModelProviderBedrock, ...). */ + context: Record; +}; + +type GetEvaluatorTemplateResolverConfig = { + assetSource: AssetSource; + templateRenderer: TemplateRenderer; +}; + +/** Resolves the template that renders a managed code-based evaluator's code directory. */ +export function getEvaluatorTemplateResolver( + config: GetEvaluatorTemplateResolverConfig, +): TemplateResolver { + return { + async resolve(input) { + const tree = await FsTreeNode.fromAssetSource( + { assetSource: config.assetSource }, + { assetDir: input.assetDir }, + { + rootDirName: input.evaluator.name, + transformContent: (raw) => config.templateRenderer.render(raw, input.context), + }, + ); + return { tree, spec: { evaluators: [input.evaluator] } }; + }, + }; +} diff --git a/src/core/project/templates/types.ts b/src/core/project/templates/types.ts index f472be503..798e4ba4b 100644 --- a/src/core/project/templates/types.ts +++ b/src/core/project/templates/types.ts @@ -3,6 +3,7 @@ import type { ProjectRuntime } from "../../../projectSchemas/runtime"; import type { MemorySchema } from "../../../projectSchemas/memory"; import type { CredentialSchema } from "../../../projectSchemas/credential"; import type { HarnessRegistryEntry } from "../../../projectSchemas/harness"; +import type { Evaluator } from "../../../projectSchemas/evaluator"; import type z from "zod"; /** AgentCore Project Spec Entries that rendered as part of a {@link Template} **/ @@ -11,6 +12,7 @@ export type SpecEntries = { credentials?: z.infer[]; memories?: z.infer[]; harnesses?: HarnessRegistryEntry[]; + evaluators?: Evaluator[]; }; /** A group of files and resources that can be rendered into a project **/ diff --git a/src/handlers/project/add/evaluator/code-based/index.test.ts b/src/handlers/project/add/evaluator/code-based/index.test.ts new file mode 100644 index 000000000..307d15878 --- /dev/null +++ b/src/handlers/project/add/evaluator/code-based/index.test.ts @@ -0,0 +1,255 @@ +import { afterEach, describe, expect, test } from "bun:test"; +import { mkdtemp, rm } from "node:fs/promises"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; +import { createRootHandler } from "../../../../index"; +import { + createSilentLogger, + TestCoreClient, + TestGlobalConfigAccessor, + testIO, +} from "../../../../../testing"; +import { InputValidationError } from "../../../../../errors"; + +const originalCwd = process.cwd(); +const tempDirectories: string[] = []; + +async function inTempDirectory(): Promise { + const directory = await mkdtemp(join(tmpdir(), "agentcore-code-eval-")); + tempDirectories.push(directory); + process.chdir(directory); + return process.cwd(); +} + +afterEach(async () => { + process.chdir(originalCwd); + await Promise.all( + tempDirectories.splice(0).map((directory) => rm(directory, { recursive: true, force: true })), + ); +}); + +async function run(args: string[]) { + const io = testIO(); + const root = createRootHandler(new TestCoreClient(), { + io: io.io, + globalConfigAccessor: new TestGlobalConfigAccessor(), + logger: createSilentLogger(), + }); + await root.route(["node", "agentcore", "project", ...args]); + return { io }; +} + +async function inProject(name = "TestProject"): Promise { + const directory = await inTempDirectory(); + await run(["create", "--name", name, "--skip-install", "--skip-git"]); + const projectRoot = join(directory, name); + process.chdir(projectRoot); + return projectRoot; +} + +const spec = (projectRoot: string) => + Bun.file(join(projectRoot, "agentcore", "agentcore.json")).json(); +const evaluator = async (projectRoot: string, name: string) => + (await spec(projectRoot)).evaluators.find((e: { name: string }) => e.name === name); + +describe("project add evaluator code-based", () => { + test("3P metric → managed config + scaffolded, rendered Lambda source", async () => { + const projectRoot = await inProject(); + await run([ + "add", + "evaluator", + "code-based", + "--name", + "answer_faithfulness", + "--level", + "SESSION", + "--metric", + "deepeval.FaithfulnessMetric", + "--model", + "bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0", + ]); + + expect(await evaluator(projectRoot, "answer_faithfulness")).toMatchObject({ + name: "answer_faithfulness", + level: "SESSION", + config: { + codeBased: { + managed: { + codeLocation: "app/answer_faithfulness", + entrypoint: "lambda_function.handler", + timeoutSeconds: 300, + additionalPolicies: ["execution-role-policy.json"], + }, + }, + }, + }); + + const appDir = join(projectRoot, "app", "answer_faithfulness"); + const handler = await Bun.file(join(appDir, "lambda_function.py")).text(); + expect(handler).toContain("FaithfulnessMetric"); + expect(handler).toContain("AmazonBedrockModel"); + expect(handler).toContain("anthropic.claude-3-5-sonnet-20240620-v1:0"); + expect(await Bun.file(join(appDir, "execution-role-policy.json")).exists()).toBe(true); + // no unrendered Handlebars left behind + expect(handler).not.toContain("{{"); + }); + + test("autoevals metric with default (non-bedrock) model", async () => { + const projectRoot = await inProject(); + await run([ + "add", + "evaluator", + "code-based", + "--name", + "factuality", + "--level", + "TRACE", + "--metric", + "autoevals.Factuality", + ]); + + expect( + (await evaluator(projectRoot, "factuality")).config.codeBased.managed.timeoutSeconds, + ).toBe(60); + const handler = await Bun.file( + join(projectRoot, "app", "factuality", "lambda_function.py"), + ).text(); + expect(handler).toContain("Factuality"); + expect(handler).not.toContain("{{"); + }); + + test("no metric, no lambda → empty managed stub", async () => { + const projectRoot = await inProject(); + await run(["add", "evaluator", "code-based", "--name", "custom_eval", "--level", "TOOL_CALL"]); + + expect((await evaluator(projectRoot, "custom_eval")).config.codeBased.managed).toMatchObject({ + codeLocation: "app/custom_eval", + timeoutSeconds: 60, + }); + const handler = await Bun.file( + join(projectRoot, "app", "custom_eval", "lambda_function.py"), + ).text(); + expect(handler).toContain("TODO"); + expect(handler).toContain("custom_code_based_evaluator"); + }); + + test("--lambda-arn → external config, no scaffold", async () => { + const projectRoot = await inProject(); + const arn = "arn:aws:lambda:us-west-2:123456789012:function:refund-policy"; + await run([ + "add", + "evaluator", + "code-based", + "--name", + "refund_policy", + "--level", + "SESSION", + "--lambda-arn", + arn, + ]); + + expect((await evaluator(projectRoot, "refund_policy")).config).toEqual({ + codeBased: { external: { lambdaArn: arn } }, + }); + expect( + await Bun.file(join(projectRoot, "app", "refund_policy", "lambda_function.py")).exists(), + ).toBe(false); + }); + + test("persists description, kms key, and tags", async () => { + const projectRoot = await inProject(); + const kms = "arn:aws:kms:us-east-1:123456789012:key/12345678-1234-1234-1234-123456789012"; + await run([ + "add", + "evaluator", + "code-based", + "--name", + "full", + "--level", + "SESSION", + "--lambda-arn", + "arn:aws:lambda:us-west-2:123456789012:function:f", + "--description", + "external scorer", + "--kms-key-arn", + kms, + "--tags", + '{"team":"ml"}', + ]); + + expect(await evaluator(projectRoot, "full")).toMatchObject({ + description: "external scorer", + kmsKeyArn: kms, + tags: { team: "ml" }, + }); + }); + + test.each<[string, string[]]>([ + ["missing --name", ["--level", "SESSION"]], + ["missing --level", ["--name", "x"]], + [ + "--metric and --lambda-arn together", + [ + "--name", + "x", + "--level", + "SESSION", + "--metric", + "deepeval.FaithfulnessMetric", + "--lambda-arn", + "arn:aws:lambda:us-west-2:123456789012:function:f", + ], + ], + [ + "unknown metric library", + ["--name", "x", "--level", "SESSION", "--metric", "ragas.Faithfulness"], + ], + ["metric without a class", ["--name", "x", "--level", "SESSION", "--metric", "deepeval"]], + ["--model without --metric", ["--name", "x", "--level", "SESSION", "--model", "bedrock/foo"]], + [ + "managed flag with --lambda-arn", + [ + "--name", + "x", + "--level", + "SESSION", + "--lambda-arn", + "arn:aws:lambda:us-west-2:123456789012:function:f", + "--timeout-seconds", + "30", + ], + ], + ["invalid --level", ["--name", "x", "--level", "NOPE"]], + ["invalid --lambda-arn", ["--name", "x", "--level", "SESSION", "--lambda-arn", "not-an-arn"]], + ])("%s", async (_label, flags) => { + await inProject(); + await expect(run(["add", "evaluator", "code-based", ...flags])).rejects.toBeInstanceOf( + InputValidationError, + ); + }); + + test("rejects a duplicate evaluator name", async () => { + await inProject(); + const flags = [ + "add", + "evaluator", + "code-based", + "--name", + "dup", + "--level", + "SESSION", + "--lambda-arn", + "arn:aws:lambda:us-west-2:123456789012:function:f", + ]; + await run(flags); + await expect(run(flags)).rejects.toBeInstanceOf(InputValidationError); + }); + + test("remove evaluator drops it from the spec", async () => { + const projectRoot = await inProject(); + await run(["add", "evaluator", "code-based", "--name", "gone", "--level", "SESSION"]); + expect(await evaluator(projectRoot, "gone")).toBeDefined(); + await run(["remove", "evaluator", "--name", "gone"]); + expect(await evaluator(projectRoot, "gone")).toBeUndefined(); + }); +}); diff --git a/src/handlers/project/add/evaluator/code-based/index.ts b/src/handlers/project/add/evaluator/code-based/index.ts new file mode 100644 index 000000000..c92da7c42 --- /dev/null +++ b/src/handlers/project/add/evaluator/code-based/index.ts @@ -0,0 +1,168 @@ +import z from "zod"; +import { createHandler, flag, ProjectKey } from "../../../../../router"; +import { InputValidationError } from "../../../../../errors"; +import { EvaluatorSchema } from "../../../../../projectSchemas/evaluator"; +import { TagsSchema } from "../../../../../projectSchemas/tags"; +import { parseJsonFlagWithSchema } from "../../../../utils"; +import type { AddProjectResourceConfig } from "../../types"; + +// 3P evaluator libraries the CLI can scaffold. The metric class is passed +// through to the library (not allowlisted here) — only the library prefix is +// validated. Default timeouts mirror the old CLI's THIRD_PARTY_EVALUATOR_LIBRARIES. +const LIBRARIES: Record = { + deepeval: { assetDir: "evaluators/deepeval-lambda", defaultTimeoutSeconds: 300 }, + autoevals: { assetDir: "evaluators/autoevals-lambda", defaultTimeoutSeconds: 60 }, +}; + +const EMPTY_ASSET_DIR = "evaluators/python-lambda"; +const EMPTY_DEFAULT_TIMEOUT_SECONDS = 60; + +export const createAddCodeBasedEvaluatorHandler = (config: AddProjectResourceConfig) => + createHandler({ + name: "code-based", + description: + "add a code-based evaluator — a Lambda that scores a session. Pass a 3P metric, an existing Lambda, or neither to scaffold an empty evaluator you fill in", + flags: [ + flag("name", "the name of the evaluator", z.string().optional()), + flag("level", "what to score: SESSION, TRACE, or TOOL_CALL", z.string().optional()), + flag( + "metric", + "3P metric to scaffold as , e.g. deepeval.FaithfulnessMetric or autoevals.Factuality", + z.string().optional(), + ), + flag( + "model", + "judge model for the 3P metric, e.g. bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0", + z.string().optional(), + ), + flag("lambda-arn", "ARN of an existing Lambda that scores a session", z.string().optional()), + flag( + "timeout-seconds", + "Lambda timeout in seconds (1-300)", + z.number().int().min(1).max(300).optional(), + ), + flag("description", "a description of what this evaluator measures", z.string().optional()), + flag( + "kms-key-arn", + "customer-managed KMS key ARN to encrypt the evaluator", + z.string().optional(), + ), + flag("tags", "tags to apply (JSON object of key/value strings)", z.string().optional()), + ], + handle: async (ctx, flags) => { + if (!flags["name"]) + throw new InputValidationError("required option '--name ' not specified"); + if (!flags["level"]) + throw new InputValidationError("required option '--level ' not specified"); + + const hasMetric = flags["metric"] !== undefined; + const hasLambda = flags["lambda-arn"] !== undefined; + if (hasMetric && hasLambda) + throw new InputValidationError( + "provide either --metric (managed) or --lambda-arn (external), not both", + ); + + const tags = parseJsonFlagWithSchema("tags", flags["tags"], TagsSchema); + const base = { + name: flags["name"], + level: flags["level"], + description: flags["description"], + kmsKeyArn: flags["kms-key-arn"], + tags, + }; + + // Kept loose so `--level` stays a plain string for EvaluatorSchema to + // validate (mirrors the llm-as-a-judge handler); safeParse narrows it. + let candidate: Record; + let scaffold: { assetDir: string; context: Record } | undefined; + + if (hasLambda) { + if (flags["metric"] || flags["model"] || flags["timeout-seconds"] !== undefined) + throw new InputValidationError( + "--metric, --model, and --timeout-seconds are managed-only and not valid with --lambda-arn", + ); + candidate = { + ...base, + config: { codeBased: { external: { lambdaArn: flags["lambda-arn"] } } }, + }; + } else { + // managed: 3P metric, or empty stub when no metric is given. + if (flags["model"] && !hasMetric) + throw new InputValidationError("--model requires --metric"); + + let assetDir = EMPTY_ASSET_DIR; + let defaultTimeout = EMPTY_DEFAULT_TIMEOUT_SECONDS; + const context: Record = { Name: toPythonPackageName(flags["name"]) }; + + if (hasMetric) { + const raw = flags["metric"]!; + const dot = raw.indexOf("."); + const library = dot > 0 ? raw.slice(0, dot) : ""; + const metricClass = dot > 0 ? raw.slice(dot + 1) : ""; + const lib = library && metricClass ? LIBRARIES[library] : undefined; + if (!lib) + throw new InputValidationError( + `invalid --metric "${raw}": expected where library is one of ${Object.keys(LIBRARIES).join(", ")} (e.g. deepeval.FaithfulnessMetric)`, + ); + assetDir = lib.assetDir; + defaultTimeout = lib.defaultTimeoutSeconds; + + const { modelProviderBedrock, model } = parseModel(flags["model"]); + context["EvaluatorClass"] = metricClass; + context["Model"] = model; + context["ModelProviderBedrock"] = modelProviderBedrock; + context["EvaluatorParams"] = ""; + } + + const timeoutSeconds = flags["timeout-seconds"] ?? defaultTimeout; + candidate = { + ...base, + config: { + codeBased: { + managed: { + codeLocation: `app/${flags["name"]}`, + entrypoint: "lambda_function.handler", + timeoutSeconds, + additionalPolicies: ["execution-role-policy.json"], + }, + }, + }, + }; + scaffold = { assetDir, context }; + } + + const parsed = EvaluatorSchema.safeParse(candidate); + if (!parsed.success) throw new InputValidationError(z.prettifyError(parsed.error)); + + const project = ctx.require(ProjectKey); + for await (const event of config.projectManager.addResource(project, { + resourceType: "evaluator", + resourceConfig: parsed.data, + scaffold, + })) { + config.io.stderr.write(`${event.message}\n`); + } + + config.io.stderr.write(`added evaluator '${flags["name"]}' to '${project.name}'\n`); + }, + }); + +// `bedrock/` selects the Bedrock judge backend (Model = the id); anything +// else (openai/gpt-4o, a bare name, or unset) falls through to the library's +// default model. +function parseModel(model: string | undefined): { modelProviderBedrock: boolean; model: string } { + if (!model) return { modelProviderBedrock: false, model: "" }; + const slash = model.indexOf("/"); + const provider = slash > 0 ? model.slice(0, slash) : ""; + if (provider === "bedrock") return { modelProviderBedrock: true, model: model.slice(slash + 1) }; + return { modelProviderBedrock: false, model }; +} + +// PEP 508 package name: ASCII letters/numbers/period/underscore/hyphen, must +// start and end alphanumeric. Mirrors the runtime template helper. +function toPythonPackageName(name: string): string { + return name + .replace(/[^a-zA-Z0-9._-]/g, "-") + .replace(/^[^a-zA-Z0-9]+/, "") + .replace(/[^a-zA-Z0-9]+$/, ""); +} diff --git a/src/handlers/project/add/evaluator/index.ts b/src/handlers/project/add/evaluator/index.ts index 5dbc413ab..1c05ea3be 100644 --- a/src/handlers/project/add/evaluator/index.ts +++ b/src/handlers/project/add/evaluator/index.ts @@ -1,9 +1,11 @@ import { Router } from "../../../../router"; import type { AddProjectResourceConfig } from "../types"; import { createAddLlmAsAJudgeEvaluatorHandler } from "./llm-as-a-judge"; +import { createAddCodeBasedEvaluatorHandler } from "./code-based"; export function createAddEvaluatorHandler(config: AddProjectResourceConfig): Router { const evaluator = new Router("evaluator", "add a custom evaluator to the current project"); evaluator.handler(createAddLlmAsAJudgeEvaluatorHandler(config)); + evaluator.handler(createAddCodeBasedEvaluatorHandler(config)); return evaluator; } diff --git a/src/handlers/project/remove/index.ts b/src/handlers/project/remove/index.ts index 40bfc132a..5c0a49f79 100644 --- a/src/handlers/project/remove/index.ts +++ b/src/handlers/project/remove/index.ts @@ -44,6 +44,7 @@ export const createRemoveProjectHandler = (config: RemoveProjectResourceConfig) "online-eval", "online-insight", "memory", + "evaluator", "gateway", "gateway-target", "gateway-connector", diff --git a/src/handlers/project/types.ts b/src/handlers/project/types.ts index 4e142deb5..ecc9b289c 100644 --- a/src/handlers/project/types.ts +++ b/src/handlers/project/types.ts @@ -210,6 +210,13 @@ export type AddResourceInput = | { resourceType: "evaluator"; resourceConfig: z.input; + /** + * Present only for managed code-based evaluators, whose code the CLI + * generates. `assetDir` picks the template under src/assets/evaluators; + * `context` holds its Handlebars variables. External and llm-as-a-judge + * evaluators omit this and are spec-only. + */ + scaffold?: { assetDir: string; context: Record }; } | { resourceType: "gateway"; From bb281d89765fcbd983940cc961236ac8a63a672b Mon Sep 17 00:00:00 2001 From: jariy17 Date: Fri, 28 Aug 2026 22:09:06 +0000 Subject: [PATCH 2/7] fix(project): guard against app/ collisions when scaffolding evaluators Runtimes, harnesses, and evaluators all scaffold into app/, but the duplicate-name guard is per-resource-type and the tree write happens outside the rollback try/catch. An evaluator whose name matches an existing runtime/ harness dir (or a leftover from a removed evaluator) threw a raw 'File already exists' mid-write and orphaned partial files. Fail up front with a clear InputValidationError when app/ already exists. --- src/core/project/manager.tsx | 8 ++++++++ .../add/evaluator/code-based/index.test.ts | 20 +++++++++++++++++-- 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/src/core/project/manager.tsx b/src/core/project/manager.tsx index 251d25c04..9396c58c9 100644 --- a/src/core/project/manager.tsx +++ b/src/core/project/manager.tsx @@ -343,6 +343,14 @@ export class FsProjectManager implements ProjectManager { if (input.scaffold) { yield { message: "Scaffolding evaluator in project" }; const outputPath = join(project.rootPath, "app", evaluator.name); + // Runtimes, harnesses, and evaluators all scaffold into app/, + // but the dup-name guard above is per-resource-type. Fail up front + // rather than let FsTreeNode.write throw mid-write (outside the + // rollback try/catch below) and orphan partial files. + if (existsSync(outputPath)) + throw new InputValidationError( + `cannot scaffold evaluator '${evaluator.name}': 'app/${evaluator.name}' already exists (another resource may use this name, or a previous scaffold was left behind)`, + ); scaffoldedPaths.push(outputPath); const resolver = getEvaluatorTemplateResolver({ assetSource: this.assetSource, diff --git a/src/handlers/project/add/evaluator/code-based/index.test.ts b/src/handlers/project/add/evaluator/code-based/index.test.ts index 307d15878..a60f1f0e7 100644 --- a/src/handlers/project/add/evaluator/code-based/index.test.ts +++ b/src/handlers/project/add/evaluator/code-based/index.test.ts @@ -1,5 +1,5 @@ import { afterEach, describe, expect, test } from "bun:test"; -import { mkdtemp, rm } from "node:fs/promises"; +import { mkdir, mkdtemp, rm } from "node:fs/promises"; import { join } from "node:path"; import { tmpdir } from "node:os"; import { createRootHandler } from "../../../../index"; @@ -50,7 +50,7 @@ async function inProject(name = "TestProject"): Promise { const spec = (projectRoot: string) => Bun.file(join(projectRoot, "agentcore", "agentcore.json")).json(); const evaluator = async (projectRoot: string, name: string) => - (await spec(projectRoot)).evaluators.find((e: { name: string }) => e.name === name); + ((await spec(projectRoot)).evaluators ?? []).find((e: { name: string }) => e.name === name); describe("project add evaluator code-based", () => { test("3P metric → managed config + scaffolded, rendered Lambda source", async () => { @@ -245,6 +245,22 @@ describe("project add evaluator code-based", () => { await expect(run(flags)).rejects.toBeInstanceOf(InputValidationError); }); + test("errors before writing when app/ already exists (cross-resource collision)", async () => { + const projectRoot = await inProject(); + const appDir = join(projectRoot, "app", "collide"); + await mkdir(appDir, { recursive: true }); + await Bun.write(join(appDir, "pyproject.toml"), "# pre-existing\n"); + + await expect( + run(["add", "evaluator", "code-based", "--name", "collide", "--level", "SESSION"]), + ).rejects.toBeInstanceOf(InputValidationError); + + // no spec entry, and the pre-existing dir is left untouched (no mid-write orphans) + expect(await evaluator(projectRoot, "collide")).toBeUndefined(); + expect(await Bun.file(join(appDir, "pyproject.toml")).text()).toBe("# pre-existing\n"); + expect(await Bun.file(join(appDir, "lambda_function.py")).exists()).toBe(false); + }); + test("remove evaluator drops it from the spec", async () => { const projectRoot = await inProject(); await run(["add", "evaluator", "code-based", "--name", "gone", "--level", "SESSION"]); From 8acc19fc7e2fb21a1468632efd5624b97a5a5a44 Mon Sep 17 00:00:00 2001 From: jariy17 Date: Fri, 28 Aug 2026 22:12:15 +0000 Subject: [PATCH 3/7] fix(project): validate --metric class and require a Bedrock --model for code-based evaluators - Reject a namespaced/multi-dot metric class (e.g. deepeval.metrics.Faithfulness) that would render invalid Python; require a single class identifier. - --model is Bedrock-only: accept a bare model id / inference-profile-or- foundation-model ARN, optionally prefixed with bedrock/, validated via isValidBedrockModelId (same forms the llm-as-a-judge handler accepts). Non-Bedrock or slashless values now error instead of being silently dropped (deepeval) or passed to the wrong client (autoevals). - autoevals template prefixes bedrock/ for litellm routing now that Model is the bare id. --- .../autoevals-lambda/lambda_function.py | 4 +- .../add/evaluator/code-based/index.test.ts | 50 +++++++++++++++++++ .../project/add/evaluator/code-based/index.ts | 39 ++++++++++----- 3 files changed, 78 insertions(+), 15 deletions(-) diff --git a/src/assets/evaluators/autoevals-lambda/lambda_function.py b/src/assets/evaluators/autoevals-lambda/lambda_function.py index 410f056a6..4161dbbf7 100644 --- a/src/assets/evaluators/autoevals-lambda/lambda_function.py +++ b/src/assets/evaluators/autoevals-lambda/lambda_function.py @@ -15,9 +15,9 @@ from bedrock_agentcore.evaluation.custom_code_based_evaluators.third_party.autoevals import AutoEvalsAdapter client = LiteLLMClient() -init(client=client, default_model="{{ Model }}") +init(client=client, default_model="bedrock/{{ Model }}") -adapter = AutoEvalsAdapter(metric={{ EvaluatorClass }}(client=client, model="{{ Model }}"){{#if EvaluatorParams}}, {{{ EvaluatorParams }}}{{/if}}) +adapter = AutoEvalsAdapter(metric={{ EvaluatorClass }}(client=client, model="bedrock/{{ Model }}"){{#if EvaluatorParams}}, {{{ EvaluatorParams }}}{{/if}}) {{else}} from autoevals import {{ EvaluatorClass }} diff --git a/src/handlers/project/add/evaluator/code-based/index.test.ts b/src/handlers/project/add/evaluator/code-based/index.test.ts index a60f1f0e7..846f36194 100644 --- a/src/handlers/project/add/evaluator/code-based/index.test.ts +++ b/src/handlers/project/add/evaluator/code-based/index.test.ts @@ -205,6 +205,36 @@ describe("project add evaluator code-based", () => { ["--name", "x", "--level", "SESSION", "--metric", "ragas.Faithfulness"], ], ["metric without a class", ["--name", "x", "--level", "SESSION", "--metric", "deepeval"]], + [ + "namespaced (multi-dot) metric class", + ["--name", "x", "--level", "SESSION", "--metric", "deepeval.metrics.Faithfulness"], + ], + [ + "non-Bedrock --model", + [ + "--name", + "x", + "--level", + "SESSION", + "--metric", + "deepeval.FaithfulnessMetric", + "--model", + "gpt-4o", + ], + ], + [ + "--model bedrock with no slash/id", + [ + "--name", + "x", + "--level", + "SESSION", + "--metric", + "autoevals.Factuality", + "--model", + "bedrock", + ], + ], ["--model without --metric", ["--name", "x", "--level", "SESSION", "--model", "bedrock/foo"]], [ "managed flag with --lambda-arn", @@ -228,6 +258,26 @@ describe("project add evaluator code-based", () => { ); }); + test("accepts a bare Bedrock inference-profile model id and renders it into the source", async () => { + const projectRoot = await inProject(); + await run([ + "add", + "evaluator", + "code-based", + "--name", + "prof", + "--level", + "SESSION", + "--metric", + "deepeval.FaithfulnessMetric", + "--model", + "us.anthropic.claude-sonnet-4-5-20250929-v1:0", + ]); + expect(await evaluator(projectRoot, "prof")).toBeDefined(); + const src = await Bun.file(join(projectRoot, "app", "prof", "lambda_function.py")).text(); + expect(src).toContain("us.anthropic.claude-sonnet-4-5-20250929-v1:0"); + }); + test("rejects a duplicate evaluator name", async () => { await inProject(); const flags = [ diff --git a/src/handlers/project/add/evaluator/code-based/index.ts b/src/handlers/project/add/evaluator/code-based/index.ts index c92da7c42..3b780707b 100644 --- a/src/handlers/project/add/evaluator/code-based/index.ts +++ b/src/handlers/project/add/evaluator/code-based/index.ts @@ -1,7 +1,7 @@ import z from "zod"; import { createHandler, flag, ProjectKey } from "../../../../../router"; import { InputValidationError } from "../../../../../errors"; -import { EvaluatorSchema } from "../../../../../projectSchemas/evaluator"; +import { EvaluatorSchema, isValidBedrockModelId } from "../../../../../projectSchemas/evaluator"; import { TagsSchema } from "../../../../../projectSchemas/tags"; import { parseJsonFlagWithSchema } from "../../../../utils"; import type { AddProjectResourceConfig } from "../../types"; @@ -104,13 +104,21 @@ export const createAddCodeBasedEvaluatorHandler = (config: AddProjectResourceCon throw new InputValidationError( `invalid --metric "${raw}": expected where library is one of ${Object.keys(LIBRARIES).join(", ")} (e.g. deepeval.FaithfulnessMetric)`, ); + // The class is rendered straight into `from import ` and + // `(...)`, so it must be a single Python identifier. Reject + // dotted/namespaced values (e.g. deepeval.metrics.Faithfulness) that + // would emit invalid Python and only fail at deploy/runtime. + if (!/^[A-Za-z_][A-Za-z0-9_]*$/.test(metricClass)) + throw new InputValidationError( + `invalid metric class "${metricClass}" in --metric "${raw}": expected a single class name like FaithfulnessMetric`, + ); assetDir = lib.assetDir; defaultTimeout = lib.defaultTimeoutSeconds; - const { modelProviderBedrock, model } = parseModel(flags["model"]); + const model = resolveBedrockModel(flags["model"]); context["EvaluatorClass"] = metricClass; - context["Model"] = model; - context["ModelProviderBedrock"] = modelProviderBedrock; + context["Model"] = model ?? ""; + context["ModelProviderBedrock"] = model !== undefined; context["EvaluatorParams"] = ""; } @@ -147,15 +155,20 @@ export const createAddCodeBasedEvaluatorHandler = (config: AddProjectResourceCon }, }); -// `bedrock/` selects the Bedrock judge backend (Model = the id); anything -// else (openai/gpt-4o, a bare name, or unset) falls through to the library's -// default model. -function parseModel(model: string | undefined): { modelProviderBedrock: boolean; model: string } { - if (!model) return { modelProviderBedrock: false, model: "" }; - const slash = model.indexOf("/"); - const provider = slash > 0 ? model.slice(0, slash) : ""; - if (provider === "bedrock") return { modelProviderBedrock: true, model: model.slice(slash + 1) }; - return { modelProviderBedrock: false, model }; +// Judge model for a 3P metric. Bedrock-only: accepts a bare Bedrock model id / +// inference-profile-or-foundation-model ARN, optionally prefixed with +// `bedrock/`, and returns the id with the prefix stripped. Anything else errors +// rather than silently degrading — a non-Bedrock value is dropped by the +// deepeval template and passed to the wrong client by autoevals. Returns +// undefined when no --model was given (library default). +function resolveBedrockModel(model: string | undefined): string | undefined { + if (!model) return undefined; + const id = model.startsWith("bedrock/") ? model.slice("bedrock/".length) : model; + if (!isValidBedrockModelId(id)) + throw new InputValidationError( + `invalid --model "${model}": expected a Bedrock model ID (e.g. anthropic.claude-3-5-sonnet-20240620-v1:0) or an inference-profile/foundation-model ARN, optionally prefixed with "bedrock/"`, + ); + return id; } // PEP 508 package name: ASCII letters/numbers/period/underscore/hyphen, must From a9e41a968dba933a86ba63826bcaf42883ca325a Mon Sep 17 00:00:00 2001 From: jariy17 Date: Fri, 28 Aug 2026 22:13:03 +0000 Subject: [PATCH 4/7] fix(project): echo inferred mode caveats for code-based evaluators Print notes after add: the empty stub returns Pass for every session until implemented, and managed evaluators are scaffolded but not yet provisioned by 'project deploy' (no CDK/L3 support). External (--lambda-arn) prints neither. --- .../add/evaluator/code-based/index.test.ts | 32 +++++++++++++++++++ .../project/add/evaluator/code-based/index.ts | 12 +++++++ 2 files changed, 44 insertions(+) diff --git a/src/handlers/project/add/evaluator/code-based/index.test.ts b/src/handlers/project/add/evaluator/code-based/index.test.ts index 846f36194..3d67f6e79 100644 --- a/src/handlers/project/add/evaluator/code-based/index.test.ts +++ b/src/handlers/project/add/evaluator/code-based/index.test.ts @@ -311,6 +311,38 @@ describe("project add evaluator code-based", () => { expect(await Bun.file(join(appDir, "lambda_function.py")).exists()).toBe(false); }); + test("empty stub warns it returns Pass until implemented, plus the not-deployed note", async () => { + await inProject(); + const { io } = await run([ + "add", + "evaluator", + "code-based", + "--name", + "stub", + "--level", + "SESSION", + ]); + expect(io.stderr()).toContain("returns Pass for every session"); + expect(io.stderr()).toContain("not yet provisioned"); + }); + + test("external mode prints neither managed note", async () => { + await inProject(); + const { io } = await run([ + "add", + "evaluator", + "code-based", + "--name", + "ext", + "--level", + "SESSION", + "--lambda-arn", + "arn:aws:lambda:us-west-2:123456789012:function:f", + ]); + expect(io.stderr()).not.toContain("returns Pass for every session"); + expect(io.stderr()).not.toContain("not yet provisioned"); + }); + test("remove evaluator drops it from the spec", async () => { const projectRoot = await inProject(); await run(["add", "evaluator", "code-based", "--name", "gone", "--level", "SESSION"]); diff --git a/src/handlers/project/add/evaluator/code-based/index.ts b/src/handlers/project/add/evaluator/code-based/index.ts index 3b780707b..60bfe62fc 100644 --- a/src/handlers/project/add/evaluator/code-based/index.ts +++ b/src/handlers/project/add/evaluator/code-based/index.ts @@ -152,6 +152,18 @@ export const createAddCodeBasedEvaluatorHandler = (config: AddProjectResourceCon } config.io.stderr.write(`added evaluator '${flags["name"]}' to '${project.name}'\n`); + // Make the inferred mode and its caveats visible: the empty stub silently + // passes every session, and no managed evaluator is provisioned by deploy + // yet (no CDK/L3 support) — both are silent footguns otherwise. + if (!hasLambda) { + if (!hasMetric) + config.io.stderr.write( + `note: this evaluator returns Pass for every session until you implement app/${flags["name"]}/lambda_function.py\n`, + ); + config.io.stderr.write( + `note: managed code-based evaluators are scaffolded locally but not yet provisioned by 'project deploy' (pending CDK/L3 support)\n`, + ); + } }, }); From 6763a26819bb353a4cdc556922aad2fa948c7f72 Mon Sep 17 00:00:00 2001 From: jariy17 Date: Mon, 31 Aug 2026 17:25:24 +0000 Subject: [PATCH 5/7] refactor(project): share toPythonPackageName via fsUtils; DEFAULT_TIMEOUT const; drop code comments --- src/core/project/fsUtils.ts | 7 ++++ src/core/project/manager.tsx | 6 ---- src/core/project/templates/evaluator.ts | 4 --- src/core/project/templates/export.ts | 2 +- src/core/project/templates/runtime.ts | 13 +------ .../add/evaluator/code-based/index.test.ts | 2 -- .../project/add/evaluator/code-based/index.ts | 36 +++---------------- 7 files changed, 14 insertions(+), 56 deletions(-) diff --git a/src/core/project/fsUtils.ts b/src/core/project/fsUtils.ts index a9e6a4a15..714205349 100644 --- a/src/core/project/fsUtils.ts +++ b/src/core/project/fsUtils.ts @@ -20,3 +20,10 @@ export function enclosingProjectRoot(directory: string): string | undefined { } } } + +export function toPythonPackageName(name: string): string { + return name + .replace(/[^a-zA-Z0-9._-]/g, "-") + .replace(/^[^a-zA-Z0-9]+/, "") + .replace(/[^a-zA-Z0-9]+$/, ""); +} diff --git a/src/core/project/manager.tsx b/src/core/project/manager.tsx index 9396c58c9..10f11e029 100644 --- a/src/core/project/manager.tsx +++ b/src/core/project/manager.tsx @@ -338,15 +338,9 @@ export class FsProjectManager implements ProjectManager { } case "evaluator": { const evaluator = parseResource(EvaluatorSchema, input.resourceConfig); - // Managed code-based evaluators ship generated Lambda source; external - // and llm-as-a-judge evaluators are spec-only. if (input.scaffold) { yield { message: "Scaffolding evaluator in project" }; const outputPath = join(project.rootPath, "app", evaluator.name); - // Runtimes, harnesses, and evaluators all scaffold into app/, - // but the dup-name guard above is per-resource-type. Fail up front - // rather than let FsTreeNode.write throw mid-write (outside the - // rollback try/catch below) and orphan partial files. if (existsSync(outputPath)) throw new InputValidationError( `cannot scaffold evaluator '${evaluator.name}': 'app/${evaluator.name}' already exists (another resource may use this name, or a previous scaffold was left behind)`, diff --git a/src/core/project/templates/evaluator.ts b/src/core/project/templates/evaluator.ts index 5c091e657..2b1968705 100644 --- a/src/core/project/templates/evaluator.ts +++ b/src/core/project/templates/evaluator.ts @@ -3,12 +3,9 @@ import type { AssetSource } from "../source"; import type { Evaluator } from "../../../projectSchemas/evaluator"; import type { TemplateRenderer, TemplateResolver } from "./types"; -/** Inputs for scaffolding a managed code-based evaluator's Lambda source. */ export type EvaluatorScaffoldInput = { evaluator: Evaluator; - /** Template directory under src/assets/evaluators, e.g. "evaluators/deepeval-lambda". */ assetDir: string; - /** Handlebars variables for the template (EvaluatorClass, Model, ModelProviderBedrock, ...). */ context: Record; }; @@ -17,7 +14,6 @@ type GetEvaluatorTemplateResolverConfig = { templateRenderer: TemplateRenderer; }; -/** Resolves the template that renders a managed code-based evaluator's code directory. */ export function getEvaluatorTemplateResolver( config: GetEvaluatorTemplateResolverConfig, ): TemplateResolver { diff --git a/src/core/project/templates/export.ts b/src/core/project/templates/export.ts index 5cc289010..f83d89302 100644 --- a/src/core/project/templates/export.ts +++ b/src/core/project/templates/export.ts @@ -16,7 +16,7 @@ import { credentialEnvVarName, type Credential } from "../../../projectSchemas/c import type { Memory } from "../../../projectSchemas/memory"; import type { EnvLocalEntry } from "../../../handlers/project/types"; import { InputValidationError } from "../../../errors/errors"; -import { toPythonPackageName } from "./runtime"; +import { toPythonPackageName } from "../fsUtils"; type ProjectSpec = z.infer; diff --git a/src/core/project/templates/runtime.ts b/src/core/project/templates/runtime.ts index 10ab9cd7d..add21b561 100644 --- a/src/core/project/templates/runtime.ts +++ b/src/core/project/templates/runtime.ts @@ -5,6 +5,7 @@ import type { ProjectRuntime } from "../../../projectSchemas/runtime"; import type { TemplateRenderer, TemplateResolver } from "./types"; import type { ScaffoldRuntimeInput } from "../../../handlers/project/types"; import { InputValidationError } from "../../../errors"; +import { toPythonPackageName } from "../fsUtils"; function buildRuntimeSpec(input: RuntimeResourceConfig): ProjectRuntime { const { scaffoldRuntimeInput, name, ...infra } = input; @@ -38,18 +39,6 @@ function buildRuntimeSpec(input: RuntimeResourceConfig): ProjectRuntime { }; } -/** - * Normalize a name for use as a Python package name per PEP 508. - * Valid names consist only of ASCII letters, numbers, period, underscore, and - * hyphen, and must start and end with a letter or number. - */ -export function toPythonPackageName(name: string): string { - return name - .replace(/[^a-zA-Z0-9._-]/g, "-") - .replace(/^[^a-zA-Z0-9]+/, "") - .replace(/[^a-zA-Z0-9]+$/, ""); -} - /** * Normalize a name for use as an npm package name. * diff --git a/src/handlers/project/add/evaluator/code-based/index.test.ts b/src/handlers/project/add/evaluator/code-based/index.test.ts index 3d67f6e79..96daf2472 100644 --- a/src/handlers/project/add/evaluator/code-based/index.test.ts +++ b/src/handlers/project/add/evaluator/code-based/index.test.ts @@ -90,7 +90,6 @@ describe("project add evaluator code-based", () => { expect(handler).toContain("AmazonBedrockModel"); expect(handler).toContain("anthropic.claude-3-5-sonnet-20240620-v1:0"); expect(await Bun.file(join(appDir, "execution-role-policy.json")).exists()).toBe(true); - // no unrendered Handlebars left behind expect(handler).not.toContain("{{"); }); @@ -305,7 +304,6 @@ describe("project add evaluator code-based", () => { run(["add", "evaluator", "code-based", "--name", "collide", "--level", "SESSION"]), ).rejects.toBeInstanceOf(InputValidationError); - // no spec entry, and the pre-existing dir is left untouched (no mid-write orphans) expect(await evaluator(projectRoot, "collide")).toBeUndefined(); expect(await Bun.file(join(appDir, "pyproject.toml")).text()).toBe("# pre-existing\n"); expect(await Bun.file(join(appDir, "lambda_function.py")).exists()).toBe(false); diff --git a/src/handlers/project/add/evaluator/code-based/index.ts b/src/handlers/project/add/evaluator/code-based/index.ts index 60bfe62fc..ac9b66eb1 100644 --- a/src/handlers/project/add/evaluator/code-based/index.ts +++ b/src/handlers/project/add/evaluator/code-based/index.ts @@ -3,19 +3,18 @@ import { createHandler, flag, ProjectKey } from "../../../../../router"; import { InputValidationError } from "../../../../../errors"; import { EvaluatorSchema, isValidBedrockModelId } from "../../../../../projectSchemas/evaluator"; import { TagsSchema } from "../../../../../projectSchemas/tags"; +import { toPythonPackageName } from "../../../../../core/project/fsUtils"; import { parseJsonFlagWithSchema } from "../../../../utils"; import type { AddProjectResourceConfig } from "../../types"; -// 3P evaluator libraries the CLI can scaffold. The metric class is passed -// through to the library (not allowlisted here) — only the library prefix is -// validated. Default timeouts mirror the old CLI's THIRD_PARTY_EVALUATOR_LIBRARIES. +const DEFAULT_TIMEOUT = 60; + const LIBRARIES: Record = { deepeval: { assetDir: "evaluators/deepeval-lambda", defaultTimeoutSeconds: 300 }, - autoevals: { assetDir: "evaluators/autoevals-lambda", defaultTimeoutSeconds: 60 }, + autoevals: { assetDir: "evaluators/autoevals-lambda", defaultTimeoutSeconds: DEFAULT_TIMEOUT }, }; const EMPTY_ASSET_DIR = "evaluators/python-lambda"; -const EMPTY_DEFAULT_TIMEOUT_SECONDS = 60; export const createAddCodeBasedEvaluatorHandler = (config: AddProjectResourceConfig) => createHandler({ @@ -71,8 +70,6 @@ export const createAddCodeBasedEvaluatorHandler = (config: AddProjectResourceCon tags, }; - // Kept loose so `--level` stays a plain string for EvaluatorSchema to - // validate (mirrors the llm-as-a-judge handler); safeParse narrows it. let candidate: Record; let scaffold: { assetDir: string; context: Record } | undefined; @@ -86,12 +83,11 @@ export const createAddCodeBasedEvaluatorHandler = (config: AddProjectResourceCon config: { codeBased: { external: { lambdaArn: flags["lambda-arn"] } } }, }; } else { - // managed: 3P metric, or empty stub when no metric is given. if (flags["model"] && !hasMetric) throw new InputValidationError("--model requires --metric"); let assetDir = EMPTY_ASSET_DIR; - let defaultTimeout = EMPTY_DEFAULT_TIMEOUT_SECONDS; + let defaultTimeout = DEFAULT_TIMEOUT; const context: Record = { Name: toPythonPackageName(flags["name"]) }; if (hasMetric) { @@ -104,10 +100,6 @@ export const createAddCodeBasedEvaluatorHandler = (config: AddProjectResourceCon throw new InputValidationError( `invalid --metric "${raw}": expected where library is one of ${Object.keys(LIBRARIES).join(", ")} (e.g. deepeval.FaithfulnessMetric)`, ); - // The class is rendered straight into `from import ` and - // `(...)`, so it must be a single Python identifier. Reject - // dotted/namespaced values (e.g. deepeval.metrics.Faithfulness) that - // would emit invalid Python and only fail at deploy/runtime. if (!/^[A-Za-z_][A-Za-z0-9_]*$/.test(metricClass)) throw new InputValidationError( `invalid metric class "${metricClass}" in --metric "${raw}": expected a single class name like FaithfulnessMetric`, @@ -152,9 +144,6 @@ export const createAddCodeBasedEvaluatorHandler = (config: AddProjectResourceCon } config.io.stderr.write(`added evaluator '${flags["name"]}' to '${project.name}'\n`); - // Make the inferred mode and its caveats visible: the empty stub silently - // passes every session, and no managed evaluator is provisioned by deploy - // yet (no CDK/L3 support) — both are silent footguns otherwise. if (!hasLambda) { if (!hasMetric) config.io.stderr.write( @@ -167,12 +156,6 @@ export const createAddCodeBasedEvaluatorHandler = (config: AddProjectResourceCon }, }); -// Judge model for a 3P metric. Bedrock-only: accepts a bare Bedrock model id / -// inference-profile-or-foundation-model ARN, optionally prefixed with -// `bedrock/`, and returns the id with the prefix stripped. Anything else errors -// rather than silently degrading — a non-Bedrock value is dropped by the -// deepeval template and passed to the wrong client by autoevals. Returns -// undefined when no --model was given (library default). function resolveBedrockModel(model: string | undefined): string | undefined { if (!model) return undefined; const id = model.startsWith("bedrock/") ? model.slice("bedrock/".length) : model; @@ -182,12 +165,3 @@ function resolveBedrockModel(model: string | undefined): string | undefined { ); return id; } - -// PEP 508 package name: ASCII letters/numbers/period/underscore/hyphen, must -// start and end alphanumeric. Mirrors the runtime template helper. -function toPythonPackageName(name: string): string { - return name - .replace(/[^a-zA-Z0-9._-]/g, "-") - .replace(/^[^a-zA-Z0-9]+/, "") - .replace(/[^a-zA-Z0-9]+$/, ""); -} From 6f3034a429b3ead20e9c5cd936c8bf7b075b3ba4 Mon Sep 17 00:00:00 2001 From: jariy17 Date: Mon, 31 Aug 2026 17:50:20 +0000 Subject: [PATCH 6/7] refactor(project): move code-based evaluator template knowledge into templates layer Mirror the runtime layering: the handler now just parses/validates flags and passes a ManagedEvaluatorScaffoldInput; templates/evaluator.ts owns the library registry, per-library timeouts, render context, and buildManagedEvaluatorSpec (parallels buildRuntimeSpec). Also adds "evaluator" to RemoveResourceInput. --- src/core/project/manager.tsx | 16 +-- src/core/project/templates/evaluator.ts | 76 +++++++++-- .../project/add/evaluator/code-based/index.ts | 128 ++++++++---------- src/handlers/project/types.ts | 15 +- 4 files changed, 133 insertions(+), 102 deletions(-) diff --git a/src/core/project/manager.tsx b/src/core/project/manager.tsx index 10f11e029..f32b75c31 100644 --- a/src/core/project/manager.tsx +++ b/src/core/project/manager.tsx @@ -337,28 +337,22 @@ export class FsProjectManager implements ProjectManager { break; } case "evaluator": { - const evaluator = parseResource(EvaluatorSchema, input.resourceConfig); if (input.scaffold) { yield { message: "Scaffolding evaluator in project" }; - const outputPath = join(project.rootPath, "app", evaluator.name); + const outputPath = join(project.rootPath, "app", input.scaffold.name); if (existsSync(outputPath)) throw new InputValidationError( - `cannot scaffold evaluator '${evaluator.name}': 'app/${evaluator.name}' already exists (another resource may use this name, or a previous scaffold was left behind)`, + `cannot scaffold evaluator '${input.scaffold.name}': 'app/${input.scaffold.name}' already exists (another resource may use this name, or a previous scaffold was left behind)`, ); scaffoldedPaths.push(outputPath); - const resolver = getEvaluatorTemplateResolver({ + const result = await getEvaluatorTemplateResolver({ assetSource: this.assetSource, templateRenderer: this.templateRenderer, - }); - const result = await resolver.resolve({ - evaluator, - assetDir: input.scaffold.assetDir, - context: input.scaffold.context, - }); + }).resolve(input.scaffold); await result.tree.write(dirname(outputPath)); projectSpec.evaluators.push(...(result.spec.evaluators ?? [])); } else { - projectSpec.evaluators.push(evaluator); + projectSpec.evaluators.push(parseResource(EvaluatorSchema, input.resourceConfig)); } break; } diff --git a/src/core/project/templates/evaluator.ts b/src/core/project/templates/evaluator.ts index 2b1968705..0940a2d64 100644 --- a/src/core/project/templates/evaluator.ts +++ b/src/core/project/templates/evaluator.ts @@ -1,14 +1,67 @@ import { FsTreeNode } from "./fsTree"; import type { AssetSource } from "../source"; -import type { Evaluator } from "../../../projectSchemas/evaluator"; +import type { Evaluator, EvaluationLevel } from "../../../projectSchemas/evaluator"; import type { TemplateRenderer, TemplateResolver } from "./types"; +import { toPythonPackageName } from "../fsUtils"; -export type EvaluatorScaffoldInput = { - evaluator: Evaluator; - assetDir: string; - context: Record; +const DEFAULT_TIMEOUT = 60; + +export const EVALUATOR_LIBRARIES = { + deepeval: { assetDir: "evaluators/deepeval-lambda", defaultTimeoutSeconds: 300 }, + autoevals: { assetDir: "evaluators/autoevals-lambda", defaultTimeoutSeconds: DEFAULT_TIMEOUT }, +} as const; + +export type EvaluatorLibrary = keyof typeof EVALUATOR_LIBRARIES; + +const EMPTY_ASSET_DIR = "evaluators/python-lambda"; + +export type ManagedEvaluatorScaffoldInput = { + name: string; + level: EvaluationLevel; + description?: string; + kmsKeyArn?: string; + tags?: Record; + metric?: { library: EvaluatorLibrary; metricClass: string }; + model?: string; + timeoutSeconds?: number; }; +function buildManagedEvaluatorSpec(input: ManagedEvaluatorScaffoldInput): Evaluator { + const timeoutSeconds = + input.timeoutSeconds ?? + (input.metric + ? EVALUATOR_LIBRARIES[input.metric.library].defaultTimeoutSeconds + : DEFAULT_TIMEOUT); + return { + name: input.name, + level: input.level, + ...(input.description && { description: input.description }), + config: { + codeBased: { + managed: { + codeLocation: `app/${input.name}`, + entrypoint: "lambda_function.handler", + timeoutSeconds, + additionalPolicies: ["execution-role-policy.json"], + }, + }, + }, + ...(input.kmsKeyArn && { kmsKeyArn: input.kmsKeyArn }), + ...(input.tags && { tags: input.tags }), + }; +} + +function buildRenderContext(input: ManagedEvaluatorScaffoldInput): Record { + const context: Record = { Name: toPythonPackageName(input.name) }; + if (input.metric) { + context["EvaluatorClass"] = input.metric.metricClass; + context["Model"] = input.model ?? ""; + context["ModelProviderBedrock"] = input.model !== undefined; + context["EvaluatorParams"] = ""; + } + return context; +} + type GetEvaluatorTemplateResolverConfig = { assetSource: AssetSource; templateRenderer: TemplateRenderer; @@ -16,18 +69,21 @@ type GetEvaluatorTemplateResolverConfig = { export function getEvaluatorTemplateResolver( config: GetEvaluatorTemplateResolverConfig, -): TemplateResolver { +): TemplateResolver { return { async resolve(input) { + const assetDir = input.metric + ? EVALUATOR_LIBRARIES[input.metric.library].assetDir + : EMPTY_ASSET_DIR; const tree = await FsTreeNode.fromAssetSource( { assetSource: config.assetSource }, - { assetDir: input.assetDir }, + { assetDir }, { - rootDirName: input.evaluator.name, - transformContent: (raw) => config.templateRenderer.render(raw, input.context), + rootDirName: input.name, + transformContent: (raw) => config.templateRenderer.render(raw, buildRenderContext(input)), }, ); - return { tree, spec: { evaluators: [input.evaluator] } }; + return { tree, spec: { evaluators: [buildManagedEvaluatorSpec(input)] } }; }, }; } diff --git a/src/handlers/project/add/evaluator/code-based/index.ts b/src/handlers/project/add/evaluator/code-based/index.ts index ac9b66eb1..f34ef086a 100644 --- a/src/handlers/project/add/evaluator/code-based/index.ts +++ b/src/handlers/project/add/evaluator/code-based/index.ts @@ -1,21 +1,20 @@ import z from "zod"; import { createHandler, flag, ProjectKey } from "../../../../../router"; import { InputValidationError } from "../../../../../errors"; -import { EvaluatorSchema, isValidBedrockModelId } from "../../../../../projectSchemas/evaluator"; +import { + EvaluatorSchema, + EvaluationLevelSchema, + isValidBedrockModelId, +} from "../../../../../projectSchemas/evaluator"; import { TagsSchema } from "../../../../../projectSchemas/tags"; -import { toPythonPackageName } from "../../../../../core/project/fsUtils"; +import { + EVALUATOR_LIBRARIES, + type EvaluatorLibrary, + type ManagedEvaluatorScaffoldInput, +} from "../../../../../core/project/templates/evaluator"; import { parseJsonFlagWithSchema } from "../../../../utils"; import type { AddProjectResourceConfig } from "../../types"; -const DEFAULT_TIMEOUT = 60; - -const LIBRARIES: Record = { - deepeval: { assetDir: "evaluators/deepeval-lambda", defaultTimeoutSeconds: 300 }, - autoevals: { assetDir: "evaluators/autoevals-lambda", defaultTimeoutSeconds: DEFAULT_TIMEOUT }, -}; - -const EMPTY_ASSET_DIR = "evaluators/python-lambda"; - export const createAddCodeBasedEvaluatorHandler = (config: AddProjectResourceConfig) => createHandler({ name: "code-based", @@ -53,6 +52,9 @@ export const createAddCodeBasedEvaluatorHandler = (config: AddProjectResourceCon throw new InputValidationError("required option '--name ' not specified"); if (!flags["level"]) throw new InputValidationError("required option '--level ' not specified"); + const levelParsed = EvaluationLevelSchema.safeParse(flags["level"]); + if (!levelParsed.success) throw new InputValidationError(z.prettifyError(levelParsed.error)); + const level = levelParsed.data; const hasMetric = flags["metric"] !== undefined; const hasLambda = flags["lambda-arn"] !== undefined; @@ -64,98 +66,76 @@ export const createAddCodeBasedEvaluatorHandler = (config: AddProjectResourceCon const tags = parseJsonFlagWithSchema("tags", flags["tags"], TagsSchema); const base = { name: flags["name"], - level: flags["level"], + level, description: flags["description"], kmsKeyArn: flags["kms-key-arn"], tags, }; - - let candidate: Record; - let scaffold: { assetDir: string; context: Record } | undefined; + const project = ctx.require(ProjectKey); if (hasLambda) { if (flags["metric"] || flags["model"] || flags["timeout-seconds"] !== undefined) throw new InputValidationError( "--metric, --model, and --timeout-seconds are managed-only and not valid with --lambda-arn", ); - candidate = { + const parsed = EvaluatorSchema.safeParse({ ...base, config: { codeBased: { external: { lambdaArn: flags["lambda-arn"] } } }, - }; - } else { - if (flags["model"] && !hasMetric) - throw new InputValidationError("--model requires --metric"); - - let assetDir = EMPTY_ASSET_DIR; - let defaultTimeout = DEFAULT_TIMEOUT; - const context: Record = { Name: toPythonPackageName(flags["name"]) }; - - if (hasMetric) { - const raw = flags["metric"]!; - const dot = raw.indexOf("."); - const library = dot > 0 ? raw.slice(0, dot) : ""; - const metricClass = dot > 0 ? raw.slice(dot + 1) : ""; - const lib = library && metricClass ? LIBRARIES[library] : undefined; - if (!lib) - throw new InputValidationError( - `invalid --metric "${raw}": expected where library is one of ${Object.keys(LIBRARIES).join(", ")} (e.g. deepeval.FaithfulnessMetric)`, - ); - if (!/^[A-Za-z_][A-Za-z0-9_]*$/.test(metricClass)) - throw new InputValidationError( - `invalid metric class "${metricClass}" in --metric "${raw}": expected a single class name like FaithfulnessMetric`, - ); - assetDir = lib.assetDir; - defaultTimeout = lib.defaultTimeoutSeconds; - - const model = resolveBedrockModel(flags["model"]); - context["EvaluatorClass"] = metricClass; - context["Model"] = model ?? ""; - context["ModelProviderBedrock"] = model !== undefined; - context["EvaluatorParams"] = ""; + }); + if (!parsed.success) throw new InputValidationError(z.prettifyError(parsed.error)); + for await (const event of config.projectManager.addResource(project, { + resourceType: "evaluator", + resourceConfig: parsed.data, + })) { + config.io.stderr.write(`${event.message}\n`); } - - const timeoutSeconds = flags["timeout-seconds"] ?? defaultTimeout; - candidate = { - ...base, - config: { - codeBased: { - managed: { - codeLocation: `app/${flags["name"]}`, - entrypoint: "lambda_function.handler", - timeoutSeconds, - additionalPolicies: ["execution-role-policy.json"], - }, - }, - }, - }; - scaffold = { assetDir, context }; + config.io.stderr.write(`added evaluator '${flags["name"]}' to '${project.name}'\n`); + return; } - const parsed = EvaluatorSchema.safeParse(candidate); - if (!parsed.success) throw new InputValidationError(z.prettifyError(parsed.error)); + if (flags["model"] && !hasMetric) throw new InputValidationError("--model requires --metric"); + + const scaffold: ManagedEvaluatorScaffoldInput = { + ...base, + ...(hasMetric && { metric: parseMetric(flags["metric"]!) }), + ...(flags["model"] !== undefined && { model: resolveBedrockModel(flags["model"]) }), + ...(flags["timeout-seconds"] !== undefined && { timeoutSeconds: flags["timeout-seconds"] }), + }; - const project = ctx.require(ProjectKey); for await (const event of config.projectManager.addResource(project, { resourceType: "evaluator", - resourceConfig: parsed.data, + resourceConfig: { name: scaffold.name }, scaffold, })) { config.io.stderr.write(`${event.message}\n`); } config.io.stderr.write(`added evaluator '${flags["name"]}' to '${project.name}'\n`); - if (!hasLambda) { - if (!hasMetric) - config.io.stderr.write( - `note: this evaluator returns Pass for every session until you implement app/${flags["name"]}/lambda_function.py\n`, - ); + if (!hasMetric) config.io.stderr.write( - `note: managed code-based evaluators are scaffolded locally but not yet provisioned by 'project deploy' (pending CDK/L3 support)\n`, + `note: this evaluator returns Pass for every session until you implement app/${flags["name"]}/lambda_function.py\n`, ); - } + config.io.stderr.write( + `note: managed code-based evaluators are scaffolded locally but not yet provisioned by 'project deploy' (pending CDK/L3 support)\n`, + ); }, }); +function parseMetric(raw: string): { library: EvaluatorLibrary; metricClass: string } { + const dot = raw.indexOf("."); + const library = dot > 0 ? raw.slice(0, dot) : ""; + const metricClass = dot > 0 ? raw.slice(dot + 1) : ""; + if (!(library in EVALUATOR_LIBRARIES)) + throw new InputValidationError( + `invalid --metric "${raw}": expected where library is one of ${Object.keys(EVALUATOR_LIBRARIES).join(", ")} (e.g. deepeval.FaithfulnessMetric)`, + ); + if (!/^[A-Za-z_][A-Za-z0-9_]*$/.test(metricClass)) + throw new InputValidationError( + `invalid metric class "${metricClass}" in --metric "${raw}": expected a single class name like FaithfulnessMetric`, + ); + return { library: library as EvaluatorLibrary, metricClass }; +} + function resolveBedrockModel(model: string | undefined): string | undefined { if (!model) return undefined; const id = model.startsWith("bedrock/") ? model.slice("bedrock/".length) : model; diff --git a/src/handlers/project/types.ts b/src/handlers/project/types.ts index ecc9b289c..fb255068b 100644 --- a/src/handlers/project/types.ts +++ b/src/handlers/project/types.ts @@ -6,6 +6,7 @@ import type { PaymentConnectorSchema, PaymentManagerSchema } from "../../project import type { ConfigBundleSchema } from "../../projectSchemas/config-bundle"; import { MemorySchema } from "../../projectSchemas/memory"; import type { EvaluatorSchema } from "../../projectSchemas/evaluator"; +import type { ManagedEvaluatorScaffoldInput } from "../../core/project/templates/evaluator"; import type { ProjectSpecSchema } from "../../projectSchemas/project"; import z from "zod"; import type { ImportBedrockAgentInput, RuntimeResourceConfig } from "./add/runtime/types"; @@ -210,13 +211,12 @@ export type AddResourceInput = | { resourceType: "evaluator"; resourceConfig: z.input; - /** - * Present only for managed code-based evaluators, whose code the CLI - * generates. `assetDir` picks the template under src/assets/evaluators; - * `context` holds its Handlebars variables. External and llm-as-a-judge - * evaluators omit this and are spec-only. - */ - scaffold?: { assetDir: string; context: Record }; + scaffold?: undefined; + } + | { + resourceType: "evaluator"; + resourceConfig: { name: string }; + scaffold: ManagedEvaluatorScaffoldInput; } | { resourceType: "gateway"; @@ -288,6 +288,7 @@ export type RemoveResourceInput = | "online-eval" | "online-insight" | "memory" + | "evaluator" | "gateway" | "policy-engine" | "payment-manager"; From ef9f5cab21574fa24b59e3bd37cf8f5b2713d749 Mon Sep 17 00:00:00 2001 From: jariy17 Date: Mon, 31 Aug 2026 18:17:31 +0000 Subject: [PATCH 7/7] fix(project): drop stale 'pending CDK/L3 support' note for managed evaluators MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The L3 (@aws/agentcore-cdk) does provision spec.evaluators — synth emits AWS::BedrockAgentCore::Evaluator + Lambda — so the note was inaccurate. Keep the empty-stub 'returns Pass until implemented' note, which is still true. --- src/handlers/project/add/evaluator/code-based/index.test.ts | 6 ++---- src/handlers/project/add/evaluator/code-based/index.ts | 3 --- 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/src/handlers/project/add/evaluator/code-based/index.test.ts b/src/handlers/project/add/evaluator/code-based/index.test.ts index 96daf2472..08faac121 100644 --- a/src/handlers/project/add/evaluator/code-based/index.test.ts +++ b/src/handlers/project/add/evaluator/code-based/index.test.ts @@ -309,7 +309,7 @@ describe("project add evaluator code-based", () => { expect(await Bun.file(join(appDir, "lambda_function.py")).exists()).toBe(false); }); - test("empty stub warns it returns Pass until implemented, plus the not-deployed note", async () => { + test("empty stub warns it returns Pass until implemented", async () => { await inProject(); const { io } = await run([ "add", @@ -321,10 +321,9 @@ describe("project add evaluator code-based", () => { "SESSION", ]); expect(io.stderr()).toContain("returns Pass for every session"); - expect(io.stderr()).toContain("not yet provisioned"); }); - test("external mode prints neither managed note", async () => { + test("external mode prints no stub note", async () => { await inProject(); const { io } = await run([ "add", @@ -338,7 +337,6 @@ describe("project add evaluator code-based", () => { "arn:aws:lambda:us-west-2:123456789012:function:f", ]); expect(io.stderr()).not.toContain("returns Pass for every session"); - expect(io.stderr()).not.toContain("not yet provisioned"); }); test("remove evaluator drops it from the spec", async () => { diff --git a/src/handlers/project/add/evaluator/code-based/index.ts b/src/handlers/project/add/evaluator/code-based/index.ts index f34ef086a..58e02b7f3 100644 --- a/src/handlers/project/add/evaluator/code-based/index.ts +++ b/src/handlers/project/add/evaluator/code-based/index.ts @@ -115,9 +115,6 @@ export const createAddCodeBasedEvaluatorHandler = (config: AddProjectResourceCon config.io.stderr.write( `note: this evaluator returns Pass for every session until you implement app/${flags["name"]}/lambda_function.py\n`, ); - config.io.stderr.write( - `note: managed code-based evaluators are scaffolded locally but not yet provisioned by 'project deploy' (pending CDK/L3 support)\n`, - ); }, });