From 660bdd522acb8f9e0f6fff514d8a5d2d9a4b464f Mon Sep 17 00:00:00 2001 From: Neha Phani Bethapudi Date: Mon, 7 Sep 2026 06:07:57 -0700 Subject: [PATCH 1/3] fix(setup): persist selected environment id Carry the selected Power Platform environment ID through onboarding, refresh, and config writes so evaluation runs do not need to re-derive it from Dataverse. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../scripts/fetch_and_setup.py | 51 +++++++++++++--- solutions/ess-maker-skills/scripts/setup.py | 12 ++++ .../ess-maker-skills/scripts/setup_state.py | 1 + .../src/skills/onboarding/step1.md | 18 ++++-- .../src/skills/onboarding/step1b.md | 2 +- .../src/skills/onboarding/step2.md | 6 +- tests/scripts/test_fetch_and_setup.py | 46 ++++++++++++--- tests/scripts/test_setup_evaluations.py | 58 +++++++++++++++++++ tests/setup/test_foundation_setup_router.py | 1 + tests/setup/test_setup_state.py | 2 +- 10 files changed, 171 insertions(+), 26 deletions(-) diff --git a/solutions/ess-maker-skills/scripts/fetch_and_setup.py b/solutions/ess-maker-skills/scripts/fetch_and_setup.py index 019a97f6f..0a1d84dba 100644 --- a/solutions/ess-maker-skills/scripts/fetch_and_setup.py +++ b/solutions/ess-maker-skills/scripts/fetch_and_setup.py @@ -240,7 +240,7 @@ def save_temp_files(components, template_configs, workflows): def _resolve_refresh_target(args, config): - """Resolve the (env, bot, name, schema, managed) target for a refresh. + """Resolve the (env URL/ID, bot, name, schema, managed) refresh target. Explicit CLI overrides win over the stored config so ``--refresh`` can retarget an agent to a second environment. Without ``--url`` this is a @@ -254,15 +254,20 @@ def _resolve_refresh_target(args, config): agent = config.get("agent", {}) retargeting = bool(args.url) env_url = args.url.rstrip("/") if args.url else config["dataverseEndpoint"] + environment_id = ( + args.environment_id + or agent.get("environmentId") + or config.get("environmentId") + ) bot_id = args.bot_id or agent.get("botId") name = args.name or agent.get("name") schema = args.schema or agent.get("schemaName") managed = args.managed if retargeting else agent.get("isManaged", False) - return env_url, bot_id, name, schema, managed + return env_url, environment_id, bot_id, name, schema, managed -def run_setup(env_url, args_bot_id, args_name, args_schema, args_managed, - paths, extra_flags=None): +def run_setup(env_url, environment_id, args_bot_id, args_name, args_schema, + args_managed, paths, extra_flags=None): """Run setup.py with the given temp file paths.""" print("\nRunning setup...\n") cmd = [ @@ -273,6 +278,8 @@ def run_setup(env_url, args_bot_id, args_name, args_schema, args_managed, "--schema", args_schema, "--components", paths["components"], ] + if environment_id: + cmd.extend(["--environment-id", environment_id]) if args_managed: cmd.append("--managed") if "template_configs" in paths: @@ -292,6 +299,10 @@ def main(): parser.add_argument("--url", help="Power Platform environment URL " "(e.g. https://org.crm.dynamics.com)") + parser.add_argument( + "--environment-id", + help="Power Platform environment ID selected during discovery", + ) parser.add_argument("--bot-id", help="Bot ID (GUID) from Dataverse") parser.add_argument("--name", @@ -308,7 +319,14 @@ def main(): # retarget to a different env/bot (see _resolve_refresh_target) --- if args.refresh: config = load_config() - env_url, bot_id, name, schema, managed = _resolve_refresh_target( + ( + env_url, + environment_id, + bot_id, + name, + schema, + managed, + ) = _resolve_refresh_target( args, config) if args.url: @@ -332,8 +350,16 @@ def main(): print(e.format_for_terminal()) sys.exit(1) paths = save_temp_files(components, template_configs, workflows) - rc = run_setup(env_url, bot_id, name, schema, managed, - paths, extra_flags=["--refresh"]) + rc = run_setup( + env_url, + environment_id, + bot_id, + name, + schema, + managed, + paths, + extra_flags=["--refresh"], + ) sys.exit(rc) # --- Normal mode: requires all arguments --- @@ -362,8 +388,15 @@ def main(): print(e.format_for_terminal()) sys.exit(1) paths = save_temp_files(components, template_configs, workflows) - rc = run_setup(env_url, args.bot_id, args.name, args.schema, - args.managed, paths) + rc = run_setup( + env_url, + args.environment_id, + args.bot_id, + args.name, + args.schema, + args.managed, + paths, + ) sys.exit(rc) diff --git a/solutions/ess-maker-skills/scripts/setup.py b/solutions/ess-maker-skills/scripts/setup.py index 469c0fc26..4e72a8469 100644 --- a/solutions/ess-maker-skills/scripts/setup.py +++ b/solutions/ess-maker-skills/scripts/setup.py @@ -584,6 +584,9 @@ def write_config(agent_info, slug, output_dir, template_configs_discovered, "slug": slug, "folder": output_dir.replace("\\", "/"), } + environment_id = str(agent_info.get("environmentId") or "").strip() + if environment_id: + agent_entry["environmentId"] = environment_id # Load existing config to preserve other agents and connections local_dir = ".local" @@ -630,6 +633,10 @@ def write_config(agent_info, slug, output_dir, template_configs_discovered, "workflowCount": workflow_count, "evaluationCount": evaluation_count, } + if environment_id: + config["environmentId"] = environment_id + elif existing.get("environmentId"): + config["environmentId"] = existing["environmentId"] # Preserve existing connections and other user-set fields for key in ("connections", "workdayTestEmployeeId", "referenceSource", "environmentSku"): @@ -753,6 +760,10 @@ def main(): description="ESS Maker Kit — one-shot setup") parser.add_argument("--url", required=True, help="Power Platform environment URL") + parser.add_argument( + "--environment-id", + help="Power Platform environment ID selected during discovery", + ) parser.add_argument("--bot-id", required=True, help="Selected bot ID from Dataverse") parser.add_argument("--name", required=True, @@ -781,6 +792,7 @@ def main(): "schema": args.schema, "managed": args.managed, "url": args.url, + "environmentId": args.environment_id, } # --- Idempotency gate: refuse silent overwrite of existing agent dir --- diff --git a/solutions/ess-maker-skills/scripts/setup_state.py b/solutions/ess-maker-skills/scripts/setup_state.py index 481ca79f3..0140d4d47 100644 --- a/solutions/ess-maker-skills/scripts/setup_state.py +++ b/solutions/ess-maker-skills/scripts/setup_state.py @@ -1187,6 +1187,7 @@ def _state_view(state: SetupState, view: str) -> dict[str, Any]: "connect_ready": state.connect_ready, "environment": { "locked": bool(state.environment.get("locked")), + "id": state.environment.get("id"), "tenant_endpoint": state.environment.get("tenant_endpoint"), }, "completed_steps": [ diff --git a/solutions/ess-maker-skills/src/skills/onboarding/step1.md b/solutions/ess-maker-skills/src/skills/onboarding/step1.md index d907d26d8..4ea72cb16 100644 --- a/solutions/ess-maker-skills/src/skills/onboarding/step1.md +++ b/solutions/ess-maker-skills/src/skills/onboarding/step1.md @@ -17,10 +17,11 @@ Parse the printed state. If `connect_ready` is true and `environment.locked` is true: 1. Set ENV_URL to `environment.tenant_endpoint`, stripping any trailing slash. -2. Do not list environments, ask how to provide an environment, or ask the +2. Set ENVIRONMENT_ID to `environment.id`. +3. Do not list environments, ask how to provide an environment, or ask the maker to select it again. -3. Set FOUNDATION_REUSED to true. -4. Continue directly to section 1.2. +4. Set FOUNDATION_REUSED to true. +5. Continue directly to section 1.2. Only continue to section 1.0 when no completed foundation state with a locked environment exists. @@ -102,7 +103,8 @@ Map the selected URL to the unique matching `instanceUrl` in ## 1.1b — Use selection -Read the selected object's `instanceUrl` field. Save it as ENV_URL. +Read the selected object's `instanceUrl` field. Save it as ENV_URL. Read its +`id` field and save it as ENVIRONMENT_ID. **Strip any trailing slash** from ENV_URL before using it (e.g., `https://org.crm.dynamics.com/` becomes `https://org.crm.dynamics.com`). @@ -127,6 +129,14 @@ Save their answer as ENV_URL. **Strip any trailing slash** from ENV_URL before using it (e.g., `https://org.crm.dynamics.com/` becomes `https://org.crm.dynamics.com`). +Resolve the manually entered URL: + +```text +python scripts/discover.py --resolve-environment-url "{ENV_URL}" +``` + +Parse `SELECTED_ENV_JSON:` and save its `id` field as ENVIRONMENT_ID. + ## 1.2 — Write the MCP config file Build the MCP URL by appending `/api/mcp` to ENV_URL. Double-check the diff --git a/solutions/ess-maker-skills/src/skills/onboarding/step1b.md b/solutions/ess-maker-skills/src/skills/onboarding/step1b.md index e334e5dc0..9f3d25652 100644 --- a/solutions/ess-maker-skills/src/skills/onboarding/step1b.md +++ b/solutions/ess-maker-skills/src/skills/onboarding/step1b.md @@ -3,7 +3,7 @@ Every **Message** block is the exact text to show the user. Copy it verbatim. Do not rephrase, add commentary, or tell the user what tools you are calling. -You should already have ENV_URL from Step 1. +You should already have ENV_URL and ENVIRONMENT_ID from Step 1. --- diff --git a/solutions/ess-maker-skills/src/skills/onboarding/step2.md b/solutions/ess-maker-skills/src/skills/onboarding/step2.md index fa6b48fee..73de1b31b 100644 --- a/solutions/ess-maker-skills/src/skills/onboarding/step2.md +++ b/solutions/ess-maker-skills/src/skills/onboarding/step2.md @@ -3,8 +3,8 @@ Every **Message** block is the exact text to show the user. Copy it verbatim. Do not rephrase, add commentary, or tell the user what tools you are calling. -You should already have these values from Step 1: ENV_URL, BOT_ID, BOT_NAME, -SCHEMA_NAME, IS_MANAGED. +You should already have these values from Step 1: ENV_URL, ENVIRONMENT_ID, +BOT_ID, BOT_NAME, SCHEMA_NAME, IS_MANAGED. --- @@ -19,7 +19,7 @@ Extracting your agent — this usually takes 10–20 seconds... Run this single command in the terminal (substitute all values): ``` -python scripts/fetch_and_setup.py --url "{ENV_URL}" --bot-id "{BOT_ID}" --name "{BOT_NAME}" --schema "{SCHEMA_NAME}" {--managed if IS_MANAGED is true} +python scripts/fetch_and_setup.py --url "{ENV_URL}" --environment-id "{ENVIRONMENT_ID}" --bot-id "{BOT_ID}" --name "{BOT_NAME}" --schema "{SCHEMA_NAME}" {--managed if IS_MANAGED is true} ``` The script authenticates to Dataverse via the browser (the user will see an diff --git a/tests/scripts/test_fetch_and_setup.py b/tests/scripts/test_fetch_and_setup.py index f7a8f3638..7fba1ceaa 100644 --- a/tests/scripts/test_fetch_and_setup.py +++ b/tests/scripts/test_fetch_and_setup.py @@ -16,9 +16,21 @@ import fetch_and_setup -def _args(url=None, bot_id=None, name=None, schema=None, managed=False): +def _args( + url=None, + environment_id=None, + bot_id=None, + name=None, + schema=None, + managed=False, +): return SimpleNamespace( - url=url, bot_id=bot_id, name=name, schema=schema, managed=managed, + url=url, + environment_id=environment_id, + bot_id=bot_id, + name=name, + schema=schema, + managed=managed, refresh=True, ) @@ -26,6 +38,7 @@ def _args(url=None, bot_id=None, name=None, schema=None, managed=False): def _config(**overrides): cfg = { "dataverseEndpoint": "https://old-env.crm.dynamics.com", + "environmentId": "old-environment-id", "agent": { "botId": "old-bot", "name": "Old Agent", @@ -39,24 +52,30 @@ def _config(**overrides): class TestResolveRefreshTarget: def test_falls_back_to_config_when_no_overrides(self): - env, bot, name, schema, managed = \ + env, environment_id, bot, name, schema, managed = \ fetch_and_setup._resolve_refresh_target(_args(), _config()) assert env == "https://old-env.crm.dynamics.com" + assert environment_id == "old-environment-id" assert bot == "old-bot" assert name == "Old Agent" assert schema == "msdyn_oldagent" assert managed is True def test_url_override_retargets_env(self): - env, bot, *_ = fetch_and_setup._resolve_refresh_target( - _args(url="https://new-env.crm.dynamics.com/", bot_id="new-bot"), + env, environment_id, bot, *_ = fetch_and_setup._resolve_refresh_target( + _args( + url="https://new-env.crm.dynamics.com/", + environment_id="new-environment-id", + bot_id="new-bot", + ), _config(), ) assert env == "https://new-env.crm.dynamics.com" # trailing slash stripped + assert environment_id == "new-environment-id" assert bot == "new-bot" def test_partial_overrides_prefer_args_then_config(self): - env, bot, name, schema, _ = fetch_and_setup._resolve_refresh_target( + env, _, bot, name, schema, _ = fetch_and_setup._resolve_refresh_target( _args(url="https://new-env.crm.dynamics.com", name="New Name"), _config(), ) @@ -67,7 +86,7 @@ def test_partial_overrides_prefer_args_then_config(self): def test_managed_reflects_flag_only_when_retargeting(self): # Retargeting (--url given): managed reflects the flag literally. - _, _, _, _, managed = fetch_and_setup._resolve_refresh_target( + _, _, _, _, _, managed = fetch_and_setup._resolve_refresh_target( _args(url="https://new-env.crm.dynamics.com", managed=False), _config(), ) @@ -76,11 +95,22 @@ def test_managed_reflects_flag_only_when_retargeting(self): def test_managed_keeps_config_on_plain_refresh(self): # Plain refresh (no --url): managed comes from config, not the # default-False flag, so a plain refresh never downgrades managed. - _, _, _, _, managed = fetch_and_setup._resolve_refresh_target( + _, _, _, _, _, managed = fetch_and_setup._resolve_refresh_target( _args(managed=False), _config(), ) assert managed is True + def test_agent_environment_id_precedes_top_level_fallback(self): + config = _config() + config["agent"]["environmentId"] = "agent-environment-id" + + _, environment_id, *_ = fetch_and_setup._resolve_refresh_target( + _args(), + config, + ) + + assert environment_id == "agent-environment-id" + class TestFetchComponents: def test_fetches_all_components_and_preserves_description( diff --git a/tests/scripts/test_setup_evaluations.py b/tests/scripts/test_setup_evaluations.py index 4a9961157..fe962a226 100644 --- a/tests/scripts/test_setup_evaluations.py +++ b/tests/scripts/test_setup_evaluations.py @@ -70,6 +70,64 @@ def test_extract_components_groups_evaluation_cases_by_parent(tmp_path): ] +def test_write_config_persists_environment_id_per_active_agent( + tmp_path, + monkeypatch, +): + monkeypatch.chdir(tmp_path) + setup_script.write_config( + { + "name": "Mock Agent", + "botId": "bot-id", + "schema": "mspva_mock", + "managed": False, + "url": "https://example.crm.dynamics.com", + "environmentId": "environment-id", + }, + "mock-agent", + "workspace/agents/mock-agent", + False, + ) + + config = json.loads( + (tmp_path / ".local" / "config.json").read_text(encoding="utf-8") + ) + assert config["environmentId"] == "environment-id" + assert config["agent"]["environmentId"] == "environment-id" + assert config["agents"][0]["environmentId"] == "environment-id" + + +def test_write_config_preserves_existing_environment_id_when_omitted( + tmp_path, + monkeypatch, +): + monkeypatch.chdir(tmp_path) + local_dir = tmp_path / ".local" + local_dir.mkdir() + (local_dir / "config.json").write_text( + json.dumps({"environmentId": "existing-environment-id"}), + encoding="utf-8", + ) + + setup_script.write_config( + { + "name": "Mock Agent", + "botId": "bot-id", + "schema": "mspva_mock", + "managed": False, + "url": "https://example.crm.dynamics.com", + }, + "mock-agent", + "workspace/agents/mock-agent", + False, + ) + + config = json.loads( + (local_dir / "config.json").read_text(encoding="utf-8") + ) + assert config["environmentId"] == "existing-environment-id" + + def test_extract_components_writes_review_metadata_for_tagged_parent(tmp_path): parent_id = "00000000-0000-0000-0000-000000000021" components = [{ diff --git a/tests/setup/test_foundation_setup_router.py b/tests/setup/test_foundation_setup_router.py index b63a5773a..4bbf21fd8 100644 --- a/tests/setup/test_foundation_setup_router.py +++ b/tests/setup/test_foundation_setup_router.py @@ -330,6 +330,7 @@ def test_onboarding_reuses_locked_foundation_environment() -> None: assert "python scripts/setup_state.py show --view report" in onboarding assert "environment.tenant_endpoint" in onboarding + assert "environment.id" in onboarding assert "Do not list environments" in onboarding diff --git a/tests/setup/test_setup_state.py b/tests/setup/test_setup_state.py index f016d22f9..d75e9660c 100644 --- a/tests/setup/test_setup_state.py +++ b/tests/setup/test_setup_state.py @@ -18,7 +18,6 @@ ProductId, SetupState, SetupStateError, - SetupStateService, SetupWorkflow, StepMode, StepStatus, @@ -167,6 +166,7 @@ def test_current_state_view_is_compact() -> None: "connect_ready": False, "environment": { "locked": False, + "id": None, "tenant_endpoint": None, }, "completed_steps": [], From 38d261aeb25e000b62b07698fadf4e8522147195 Mon Sep 17 00:00:00 2001 From: Neha Phani Bethapudi Date: Mon, 7 Sep 2026 06:08:05 -0700 Subject: [PATCH 2/3] feat(evaluations): sync MinimalBot workspace YAML Add workspace YAML conversion for MinimalBot pushes, pull evaluation components back to YAML with component metadata, and resolve environment and bot IDs from local config. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../scripts/test_minimalbot_evaluation_poc.py | 200 +++++++++ tools/minimalbot_evaluation_poc.py | 391 +++++++++++++++++- 2 files changed, 577 insertions(+), 14 deletions(-) create mode 100644 tests/scripts/test_minimalbot_evaluation_poc.py diff --git a/tests/scripts/test_minimalbot_evaluation_poc.py b/tests/scripts/test_minimalbot_evaluation_poc.py new file mode 100644 index 000000000..bf0165ea1 --- /dev/null +++ b/tests/scripts/test_minimalbot_evaluation_poc.py @@ -0,0 +1,200 @@ +from __future__ import annotations + +import json +import sys +from pathlib import Path + +import pytest + + +ROOT = Path(__file__).resolve().parents[2] +sys.path.insert(0, str(ROOT)) + +from tools import minimalbot_evaluation_poc as poc # noqa: E402 + + +def test_wire_kinds_converts_nested_discriminators(): + result = poc._wire_kinds({ + "kind": "EvaluationSet", + "graders": [{"kind": "GeneralQualityGrader"}], + }) + + assert result == { + "$kind": "EvaluationSet", + "graders": [{"$kind": "GeneralQualityGrader"}], + } + + +def test_unwire_kinds_converts_nested_discriminators(): + result = poc._unwire_kinds({ + "$kind": "EvaluationSet", + "graders": [{"$kind": "GeneralQualityGrader"}], + }) + + assert result == { + "kind": "EvaluationSet", + "graders": [{"kind": "GeneralQualityGrader"}], + } + + +def test_workspace_payload_converts_evaluation_yaml(tmp_path): + (tmp_path / "compensation.mcs.yml").write_text( + """ +kind: EvaluationSet +displayName: Compensation +graders: + - kind: GeneralQualityGrader + - kind: CompareMeaningGrader + threshold: 0.7 +""".strip(), + encoding="utf-8", + ) + (tmp_path / "base-compensation.mcs.yml").write_text( + """ +kind: EvaluationData +rows: + - source: Imported + input: base compensation + expectedOutput: Returns base compensation +extensionData: + displayOrder: "100" +""".strip(), + encoding="utf-8", + ) + + payload, test_set_id = poc._workspace_payload(tmp_path, "change-token") + + assert payload["changeToken"] == "change-token" + assert payload["connectionReferenceChanges"] == [] + assert len(payload["botComponentChanges"]) == 2 + + parent = payload["botComponentChanges"][0] + child = payload["botComponentChanges"][1] + assert parent["$kind"] == "BotComponentInsert" + assert parent["component"]["$kind"] == "TestCaseComponent" + assert parent["component"]["id"] == test_set_id + assert parent["component"]["definition"]["$kind"] == "EvaluationSet" + assert parent["component"]["definition"]["graders"] == [ + {"$kind": "GeneralQualityGrader"}, + {"$kind": "CompareMeaningGrader", "threshold": 0.7}, + ] + assert child["component"]["parentBotComponentId"] == test_set_id + assert child["component"]["definition"]["$kind"] == "EvaluationData" + assert child["component"]["definition"]["rows"][0]["$kind"] == ( + "SimpleEvaluationCase" + ) + assert child["component"]["definition"]["extensionData"] == { + "displayOrder": "100" + } + assert parent["component"]["extensionData"] == { + "UseProvidedBotComponentId": True + } + + +def test_workspace_payload_requires_one_parent(tmp_path): + (tmp_path / "case.mcs.yml").write_text( + "kind: EvaluationData\nrows:\n - input: hello\n", + encoding="utf-8", + ) + + with pytest.raises(RuntimeError, match="exactly one EvaluationSet"): + poc._workspace_payload(tmp_path, "change-token") + + +def test_pull_writes_workspace_yaml_and_component_map(tmp_path): + response = { + "botComponentChanges": [ + { + "component": { + "id": "parent-id", + "schemaName": "mspva_compensation", + "definition": { + "$kind": "EvaluationSet", + "displayName": "Compensation", + "graders": [{"$kind": "GeneralQualityGrader"}], + }, + }, + }, + { + "component": { + "id": "case-id", + "parentBotComponentId": "parent-id", + "schemaName": "mspva_base_compensation", + "definition": { + "$kind": "EvaluationData", + "displayName": "Base compensation", + "rows": [{ + "$kind": "SimpleEvaluationCase", + "input": "base compensation", + }], + }, + }, + }, + ], + } + + counts = poc._write_workspace_evaluations(response, tmp_path) + + assert counts == {"sets": 1, "cases": 1} + parent = ( + tmp_path + / "evaluations" + / "compensation" + / "compensation.mcs.yml" + ).read_text(encoding="utf-8") + case = ( + tmp_path + / "evaluations" + / "compensation" + / "base-compensation.mcs.yml" + ).read_text(encoding="utf-8") + assert "kind: EvaluationSet" in parent + assert "kind: GeneralQualityGrader" in parent + assert "kind: EvaluationData" in case + assert "kind: SimpleEvaluationCase" in case + component_map = json.loads( + (tmp_path / ".component-map.json").read_text(encoding="utf-8") + ) + child_path = "evaluations/compensation/base-compensation.mcs.yml" + assert component_map[child_path]["parentbotcomponentid"] == "parent-id" + + +def test_pull_rejects_response_without_evaluation_sets(tmp_path): + with pytest.raises(RuntimeError, match="did not expose"): + poc._write_workspace_evaluations( + {"botComponentChanges": []}, + tmp_path, + ) + + +def test_resolve_target_uses_config_defaults(tmp_path): + config_path = tmp_path / "config.json" + config_path.write_text( + json.dumps({ + "environmentId": "environment-id", + "agent": {"botId": "bot-id"}, + }), + encoding="utf-8", + ) + + assert poc._resolve_target(None, None, config_path) == ( + "environment-id", + "bot-id", + ) + + +def test_resolve_target_cli_values_override_config(tmp_path): + config_path = tmp_path / "config.json" + config_path.write_text( + json.dumps({ + "environmentId": "old-environment-id", + "agent": {"botId": "old-bot-id"}, + }), + encoding="utf-8", + ) + + assert poc._resolve_target( + "new-environment-id", + "new-bot-id", + config_path, + ) == ("new-environment-id", "new-bot-id") diff --git a/tools/minimalbot_evaluation_poc.py b/tools/minimalbot_evaluation_poc.py index d243c91cd..da03c39c1 100644 --- a/tools/minimalbot_evaluation_poc.py +++ b/tools/minimalbot_evaluation_poc.py @@ -15,16 +15,22 @@ from datetime import datetime, timezone import json from pathlib import Path +import re from typing import Any import uuid import msal import requests +import yaml DEFAULT_CLIENT_ID = "417219b4-3a7d-42a2-bdb1-972bd8281a02" PPAPI_SCOPE = "https://api.test.powerplatform.com/.default" MCS_CONNECTOR = "shared_microsoftcopilotstudio" +DEFAULT_CONFIG_PATHS = ( + Path(".local/config.json"), + Path("solutions/ess-maker-skills/.local/config.json"), +) def _claims(token: str) -> dict[str, Any]: @@ -74,7 +80,7 @@ def _request_json( *, body: dict[str, Any] | None = None, params: dict[str, str] | None = None, -) -> tuple[requests.Response, dict[str, Any]]: +) -> tuple[requests.Response, Any]: response = requests.request( method, url, @@ -97,6 +103,315 @@ def _component_id(change: dict[str, Any]) -> str: return str((change.get("component") or {}).get("id") or "") +def _wire_kinds(value: Any) -> Any: + """Convert workspace ``kind`` discriminators to JSON ``$kind``.""" + if isinstance(value, dict): + return { + "$kind" if key == "kind" else key: _wire_kinds(child) + for key, child in value.items() + } + if isinstance(value, list): + return [_wire_kinds(child) for child in value] + return value + + +def _unwire_kinds(value: Any) -> Any: + """Convert MinimalBot JSON ``$kind`` discriminators to workspace YAML.""" + if isinstance(value, dict): + return { + "kind" if key == "$kind" else key: _unwire_kinds(child) + for key, child in value.items() + } + if isinstance(value, list): + return [_unwire_kinds(child) for child in value] + return value + + +def _safe_name(value: str, fallback: str) -> str: + name = re.sub(r"[^a-z0-9]+", "-", value.casefold()).strip("-") + return name or fallback + + +def _component_name(component: dict[str, Any]) -> str: + definition = component.get("definition") or {} + return str( + definition.get("displayName") + or component.get("displayName") + or component.get("schemaName") + or component.get("id") + or "evaluation" + ) + + +def _write_workspace_evaluations( + response: dict[str, Any], + output_dir: Path, +) -> dict[str, int]: + """Write MinimalBot evaluation components as workspace ``.mcs.yml``.""" + changes = response.get("botComponentChanges") + if not isinstance(changes, list): + raise RuntimeError( + "MinimalBot response did not contain botComponentChanges." + ) + + evaluations = [] + for change in changes: + component = change.get("component") if isinstance(change, dict) else None + definition = ( + component.get("definition") + if isinstance(component, dict) + else None + ) + if not isinstance(definition, dict): + continue + if definition.get("$kind") not in {"EvaluationSet", "EvaluationData"}: + continue + evaluations.append(component) + + parents = { + str(component.get("id")): component + for component in evaluations + if ( + (component.get("definition") or {}).get("$kind") == "EvaluationSet" + and component.get("id") + ) + } + cases = [ + component + for component in evaluations + if (component.get("definition") or {}).get("$kind") == "EvaluationData" + ] + if not parents: + raise RuntimeError( + "MinimalBot response did not expose any EvaluationSet components." + ) + + output_dir.mkdir(parents=True, exist_ok=True) + component_map: dict[str, dict[str, Any]] = {} + used_paths: set[str] = set() + + def write_component( + component: dict[str, Any], + folder_name: str, + ) -> None: + component_id = str(component.get("id") or "") + name = _component_name(component) + stem = _safe_name(name, "evaluation") + relative_path = f"evaluations/{folder_name}/{stem}.mcs.yml" + if relative_path in used_paths: + relative_path = ( + f"evaluations/{folder_name}/{stem}-" + f"{component_id.replace('-', '')[:8]}.mcs.yml" + ) + used_paths.add(relative_path) + path = output_dir.joinpath(*relative_path.split("/")) + path.parent.mkdir(parents=True, exist_ok=True) + definition = _unwire_kinds(component["definition"]) + path.write_text( + yaml.safe_dump( + definition, + sort_keys=False, + allow_unicode=True, + ), + encoding="utf-8", + ) + entry = { + "botcomponentid": component_id, + "schemaname": str(component.get("schemaName") or ""), + "componenttype": 19, + "name": name, + "description": str(component.get("description") or ""), + } + parent_id = str(component.get("parentBotComponentId") or "") + if parent_id: + entry["parentbotcomponentid"] = parent_id + component_map[relative_path] = entry + + parent_folders = {} + for parent_id, component in parents.items(): + folder_name = _safe_name(_component_name(component), "evaluation-set") + if folder_name in parent_folders.values(): + folder_name = f"{folder_name}-{parent_id.replace('-', '')[:8]}" + parent_folders[parent_id] = folder_name + write_component(component, folder_name) + + orphan_ids = [] + for component in cases: + parent_id = str(component.get("parentBotComponentId") or "") + folder_name = parent_folders.get(parent_id) + if not folder_name: + orphan_ids.append(str(component.get("id") or "unknown")) + continue + write_component(component, folder_name) + if orphan_ids: + raise RuntimeError( + "EvaluationData components referenced unavailable parents: " + + ", ".join(orphan_ids) + ) + + map_path = output_dir / ".component-map.json" + existing_map = {} + if map_path.is_file(): + try: + existing_map = json.loads(map_path.read_text(encoding="utf-8")) + except (OSError, ValueError): + existing_map = {} + if not isinstance(existing_map, dict): + existing_map = {} + existing_map.update(component_map) + map_path.write_text( + json.dumps(existing_map, indent=2) + "\n", + encoding="utf-8", + ) + return {"sets": len(parents), "cases": len(cases)} + + +def _load_config(config_path: Path | None) -> dict[str, Any]: + candidates = (config_path,) if config_path else DEFAULT_CONFIG_PATHS + for candidate in candidates: + if candidate and candidate.is_file(): + try: + config = json.loads(candidate.read_text(encoding="utf-8")) + except (OSError, ValueError) as exc: + raise RuntimeError( + f"Unable to read config {candidate}: {exc}" + ) from exc + if not isinstance(config, dict): + raise RuntimeError(f"Config must contain an object: {candidate}") + return config + if config_path: + raise RuntimeError(f"Config file does not exist: {config_path}") + return {} + + +def _resolve_target( + environment_id: str | None, + bot_id: str | None, + config_path: Path | None, +) -> tuple[str, str]: + config = ( + _load_config(config_path) + if config_path or not (environment_id and bot_id) + else {} + ) + agent = config.get("agent") if isinstance(config.get("agent"), dict) else {} + resolved_environment = str( + environment_id + or agent.get("environmentId") + or config.get("environmentId") + or "" + ) + resolved_bot = str(bot_id or agent.get("botId") or "") + if not resolved_environment: + raise RuntimeError( + "Environment ID is required. Pass --environment-id or configure " + "environmentId in .local/config.json." + ) + if not resolved_bot: + raise RuntimeError( + "Bot ID is required. Pass --bot-id or configure agent.botId in " + ".local/config.json." + ) + return resolved_environment, resolved_bot + + +def _schema_name(stem: str, component_id: str) -> str: + safe_stem = re.sub(r"[^a-z0-9]+", "_", stem.casefold()).strip("_") + safe_stem = safe_stem or "evaluation" + suffix = component_id.replace("-", "")[:8] + return f"mspva_{safe_stem[:70]}_{suffix}" + + +def _workspace_payload( + evaluation_folder: Path, + change_token: str, +) -> tuple[dict[str, Any], str]: + """Convert one workspace evaluation folder to a MinimalBot change set.""" + if not evaluation_folder.is_dir(): + raise RuntimeError( + f"Evaluation folder does not exist: {evaluation_folder}" + ) + documents = [] + for path in sorted(evaluation_folder.glob("*.mcs.yml")): + try: + document = yaml.safe_load(path.read_text(encoding="utf-8")) + except (OSError, yaml.YAMLError) as exc: + raise RuntimeError(f"Unable to read evaluation YAML {path}: {exc}") from exc + if not isinstance(document, dict): + raise RuntimeError(f"Evaluation YAML must contain an object: {path}") + documents.append((path, document)) + + parents = [ + item for item in documents + if item[1].get("kind") == "EvaluationSet" + ] + cases = [ + item for item in documents + if item[1].get("kind") == "EvaluationData" + ] + if len(parents) != 1: + raise RuntimeError( + "Evaluation folder must contain exactly one EvaluationSet." + ) + if not cases: + raise RuntimeError( + "Evaluation folder must contain at least one EvaluationData file." + ) + if len(cases) > 100: + raise RuntimeError("Copilot Studio evaluation sets support at most 100 cases.") + + parent_path, parent_document = parents[0] + parent_id = str(uuid.uuid4()) + parent_definition = _wire_kinds(parent_document) + parent_definition["displayName"] = ( + f"{parent_definition.get('displayName') or parent_path.stem} POC " + f"{datetime.now(timezone.utc):%Y-%m-%d %H:%M UTC}" + ) + changes = [{ + "$kind": "BotComponentInsert", + "component": { + "$kind": "TestCaseComponent", + "id": parent_id, + "schemaName": _schema_name(parent_path.stem, parent_id), + "definition": parent_definition, + "extensionData": {"UseProvidedBotComponentId": True}, + }, + }] + + for case_path, case_document in cases: + case_id = str(uuid.uuid4()) + case_definition = _wire_kinds(case_document) + rows = case_definition.get("rows") + if not isinstance(rows, list) or not rows: + raise RuntimeError( + f"EvaluationData must contain at least one row: {case_path}" + ) + for row in rows: + if not isinstance(row, dict): + raise RuntimeError( + f"EvaluationData rows must be objects: {case_path}" + ) + row.setdefault("$kind", "SimpleEvaluationCase") + changes.append({ + "$kind": "BotComponentInsert", + "component": { + "$kind": "TestCaseComponent", + "id": case_id, + "parentBotComponentId": parent_id, + "schemaName": _schema_name(case_path.stem, case_id), + "definition": case_definition, + "extensionData": {"UseProvidedBotComponentId": True}, + }, + }) + + return { + "changeToken": change_token, + "botComponentChanges": changes, + "connectionReferenceChanges": [], + "connectorDefinitionChanges": [], + }, parent_id + + def _clone_payload( payload_path: Path, change_token: str, @@ -242,16 +557,44 @@ def _tool_bindings( def main() -> int: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--tenant-id", required=True) - parser.add_argument("--environment-id", required=True) - parser.add_argument("--bot-id", required=True) - parser.add_argument("--payload", required=True, type=Path) + parser.add_argument("--environment-id") + parser.add_argument("--bot-id") + parser.add_argument( + "--config", + type=Path, + help=( + "Config containing environmentId and agent.botId. Defaults to " + ".local/config.json." + ), + ) + source = parser.add_mutually_exclusive_group(required=True) + source.add_argument( + "--evaluation-folder", + type=Path, + help="Folder containing one EvaluationSet and its EvaluationData YAML files.", + ) + source.add_argument( + "--payload", + type=Path, + help="Prebuilt MinimalBot JSON payload template.", + ) + source.add_argument( + "--pull-output-folder", + type=Path, + help="Pull MinimalBot evaluations into this workspace folder.", + ) parser.add_argument("--client-id", default=DEFAULT_CLIENT_ID) args = parser.parse_args() + environment_id, bot_id = _resolve_target( + args.environment_id, + args.bot_id, + args.config, + ) token, username = _authenticate(args.tenant_id, args.client_id) - host = _environment_host(args.environment_id) + host = _environment_host(environment_id) components_url = ( - f"{host}/copilotstudio/minimalBots/api/{args.bot_id}/components" + f"{host}/copilotstudio/minimalBots/api/{bot_id}/components" "?api-version=2022-03-01-preview" ) response, before = _request_json( @@ -264,10 +607,30 @@ def main() -> int: raise RuntimeError( f"MinimalBot read failed (HTTP {response.status_code})." ) - payload, test_set_id = _clone_payload( - args.payload, - str(before["changeToken"]), - ) + if args.pull_output_folder: + counts = _write_workspace_evaluations( + before, + args.pull_output_folder, + ) + print(json.dumps({ + "signedInAccount": username, + "environmentId": environment_id, + "botId": bot_id, + "outputFolder": str(args.pull_output_folder), + "evaluationSets": counts["sets"], + "evaluationCases": counts["cases"], + }, indent=2)) + return 0 + if args.evaluation_folder: + payload, test_set_id = _workspace_payload( + args.evaluation_folder, + str(before["changeToken"]), + ) + else: + payload, test_set_id = _clone_payload( + args.payload, + str(before["changeToken"]), + ) response, _ = _request_json( "PUT", components_url, @@ -300,7 +663,7 @@ def main() -> int: _connected( host, token, - args.environment_id, + environment_id, MCS_CONNECTOR, ), MCS_CONNECTOR, @@ -309,8 +672,8 @@ def main() -> int: tools_connections = _tool_bindings( host, token, - args.environment_id, - args.bot_id, + environment_id, + bot_id, str(bot.get("schemaName") or ""), components.get("connectionReferenceChanges") or [], ) @@ -324,7 +687,7 @@ def main() -> int: } run_url = ( "https://api.test.powerplatform.com/copilotstudio/environments/" - f"{args.environment_id}/bots/{args.bot_id}/api/makerevaluation/" + f"{environment_id}/bots/{bot_id}/api/makerevaluation/" f"testsets/{test_set_id}/run?api-version=2024-10-01" ) response, result = _request_json( From 7335ee70eaafe2605c314207a7b5ec50bb922e8d Mon Sep 17 00:00:00 2001 From: Neha Phani Bethapudi Date: Mon, 7 Sep 2026 06:27:28 -0700 Subject: [PATCH 3/3] fix(evaluations): prompt for ambiguous run profile Expose connected Copilot Studio profiles for explicit selection when signed-in account matching cannot safely choose one, and document the two-turn retry flow. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../scripts/evaluation_runs.py | 62 +++++++++++++++-- .../src/skills/evaluations/run/SKILL.md | 25 +++++-- tests/scripts/test_evaluation_runs.py | 67 +++++++++++++++++++ 3 files changed, 145 insertions(+), 9 deletions(-) diff --git a/solutions/ess-maker-skills/scripts/evaluation_runs.py b/solutions/ess-maker-skills/scripts/evaluation_runs.py index 366576848..b7fa19daf 100644 --- a/solutions/ess-maker-skills/scripts/evaluation_runs.py +++ b/solutions/ess-maker-skills/scripts/evaluation_runs.py @@ -252,6 +252,24 @@ def resolve_mcs_connection( signed_in_username: str | None = None, ) -> dict[str, Any]: """Discover and select the current user's Copilot Studio connection.""" + connections, effective_username = _discover_mcs_connections( + config, + environment_id, + signed_in_username, + ) + return select_mcs_connection( + connections, + effective_username, + requested_id, + ) + + +def _discover_mcs_connections( + config: dict[str, Any], + environment_id: str, + signed_in_username: str | None = None, +) -> tuple[list[dict[str, Any]], str | None]: + """Return raw Copilot Studio connections and the authenticated username.""" env_url = str(config["dataverseEndpoint"]).rstrip("/") client = PPAdminClient(discover_tenant(env_url)) client.authenticate( @@ -263,11 +281,40 @@ def resolve_mcs_connection( MCS_CONNECTOR_NAME, ) _raise_api_error(connections, "list Copilot Studio connections") - return select_mcs_connection( - connections, - signed_in_username or client.signed_in_username, - requested_id, + if not isinstance(connections, list): + raise EvaluationRunError( + "Power Platform API returned an invalid connection list." + ) + return connections, signed_in_username or client.signed_in_username + + +def list_mcs_connections( + config: dict[str, Any], + environment_id: str, + signed_in_username: str | None = None, +) -> list[dict[str, Any]]: + """List connected profiles so the user can explicitly choose one.""" + connections, effective_username = _discover_mcs_connections( + config, + environment_id, + signed_in_username, ) + username = str(effective_username or "").casefold() + return [ + { + **connection, + "matchesSignedInAccount": bool( + username + and username in { + str(connection.get("accountName") or "").casefold(), + str( + connection.get("createdByUserPrincipalName") or "" + ).casefold(), + } + ), + } + for connection in connected_mcs_connections(connections) + ] def _required_agent_connection(config: dict[str, Any]) -> dict[str, Any] | None: @@ -841,6 +888,7 @@ def main() -> int: run_parser.add_argument("--published", action="store_true") run_parser.add_argument("--mcs-connection-id") + subparsers.add_parser("list-connections") subparsers.add_parser("list-runs") results_parser = subparsers.add_parser("results") @@ -905,6 +953,12 @@ def main() -> int: run_on_published_bot=args.published, tools_connections=tools_connections, )) + elif args.command == "list-connections": + _print_json(list_mcs_connections( + config, + environment_id, + client.signed_in_username, + )) elif args.command == "list-runs": _print_json(list_runs( client, diff --git a/solutions/ess-maker-skills/src/skills/evaluations/run/SKILL.md b/solutions/ess-maker-skills/src/skills/evaluations/run/SKILL.md index c75b1c1a9..c5fd51639 100644 --- a/solutions/ess-maker-skills/src/skills/evaluations/run/SKILL.md +++ b/solutions/ess-maker-skills/src/skills/evaluations/run/SKILL.md @@ -92,12 +92,27 @@ Copilot Studio connections in the selected environment and keeps only profiles whose status is `Connected`. - If exactly one profile is connected, use it automatically. -- If multiple profiles exist, use the one that uniquely matches the signed-in +- If multiple profiles exist, use the latest profile matching the signed-in Power Apps account. -- If multiple connected profiles remain, automatically use the first profile - in deterministic name/ID order and try the run without asking the user. -- If none are connected, stop and explain that the user must create or repair - the connection in Power Apps or Power Automate. +- If automatic matching fails, run: + + ```text + python scripts/evaluation_runs.py list-connections + ``` + + Display every returned profile with its display name, account name, creator, + and connection ID. Ask the user to select one using `vscode_askQuestions`, + then **STOP**. Do not start the evaluation in the same turn. +- After the user selects a profile, retry the previously selected test set: + + ```text + python scripts/evaluation_runs.py run --test-set-id "{id}" --test-set-name "{displayName}" --mcs-connection-id "{connectionId}" + ``` + +- If no connected profile is returned, explain that the connection must be + created or repaired in Power Apps or Power Automate. Ask the user to choose + **Retry connection discovery** when ready, then stop. On their next turn, + rerun `list-connections`. Every run must include a validated `mcsConnectionId`; do not start an anonymous evaluation run. diff --git a/tests/scripts/test_evaluation_runs.py b/tests/scripts/test_evaluation_runs.py index 690bd6b41..1b51cf10e 100644 --- a/tests/scripts/test_evaluation_runs.py +++ b/tests/scripts/test_evaluation_runs.py @@ -705,6 +705,73 @@ def get_connector_connections(self, environment_id, connector_name): } +def test_list_mcs_connections_marks_signed_in_profile(monkeypatch): + monkeypatch.setattr( + evaluation_runs, + "_discover_mcs_connections", + lambda *args, **kwargs: ( + [ + _connection( + "current", + account_name="maker@example.com", + ), + _connection( + "other", + account_name="other@example.com", + ), + ], + "maker@example.com", + ), + ) + + connections = evaluation_runs.list_mcs_connections( + {"dataverseEndpoint": "https://example.crm.dynamics.com"}, + "environment-id", + ) + + assert [item["id"] for item in connections] == ["current", "other"] + assert connections[0]["matchesSignedInAccount"] is True + assert connections[1]["matchesSignedInAccount"] is False + + +def test_list_connections_command_prints_selectable_profiles( + monkeypatch, + capsys, + tmp_path, +): + client = FakeClient() + client.signed_in_username = "maker@example.com" + monkeypatch.setattr( + sys, + "argv", + ["evaluation_runs.py", "list-connections"], + ) + monkeypatch.setattr(evaluation_runs, "load_config", lambda: {}) + monkeypatch.setattr( + evaluation_runs, + "_runtime", + lambda config: ( + client, + "environment-id", + "bot-id", + tmp_path, + ), + ) + monkeypatch.setattr( + evaluation_runs, + "list_mcs_connections", + lambda *args, **kwargs: [{ + "id": "connection-id", + "displayName": "Maker profile", + "matchesSignedInAccount": True, + }], + ) + + assert evaluation_runs.main() == 0 + output = json.loads(capsys.readouterr().out) + assert output[0]["id"] == "connection-id" + + def test_resolve_tool_connections_uses_signed_in_account( monkeypatch, tmp_path,