Persist environment IDs and add MinimalBot evaluation YAML sync

solutions/ess-maker-skills/scripts/evaluation_runs.py

@@ -252,6 +252,24 @@ def resolve_mcs_connection(
     signed_in_username: str | None = None,
 ) -> dict[str, Any]:
     """Discover and select the current user's Copilot Studio connection."""
+    connections, effective_username = _discover_mcs_connections(
+        config,
+        environment_id,
+        signed_in_username,
+    )
+    return select_mcs_connection(
+        connections,
+        effective_username,
+        requested_id,
+    )
+
+
+def _discover_mcs_connections(
+    config: dict[str, Any],
+    environment_id: str,
+    signed_in_username: str | None = None,
+) -> tuple[list[dict[str, Any]], str | None]:
+    """Return raw Copilot Studio connections and the authenticated username."""
     env_url = str(config["dataverseEndpoint"]).rstrip("/")
     client = PPAdminClient(discover_tenant(env_url))
     client.authenticate(
@@ -263,11 +281,40 @@ def resolve_mcs_connection(
         MCS_CONNECTOR_NAME,
     )
     _raise_api_error(connections, "list Copilot Studio connections")
-    return select_mcs_connection(
-        connections,
-        signed_in_username or client.signed_in_username,
-        requested_id,
+    if not isinstance(connections, list):
+        raise EvaluationRunError(
+            "Power Platform API returned an invalid connection list."
+        )
+    return connections, signed_in_username or client.signed_in_username
+
+
+def list_mcs_connections(
+    config: dict[str, Any],
+    environment_id: str,
+    signed_in_username: str | None = None,
+) -> list[dict[str, Any]]:
+    """List connected profiles so the user can explicitly choose one."""
+    connections, effective_username = _discover_mcs_connections(
+        config,
+        environment_id,
+        signed_in_username,
     )
+    username = str(effective_username or "").casefold()
+    return [
+        {
+            **connection,
+            "matchesSignedInAccount": bool(
+                username
+                and username in {
+                    str(connection.get("accountName") or "").casefold(),
+                    str(
+                        connection.get("createdByUserPrincipalName") or ""
+                    ).casefold(),
+                }
+            ),
+        }
+        for connection in connected_mcs_connections(connections)
+    ]
 
 
 def _required_agent_connection(config: dict[str, Any]) -> dict[str, Any] | None:
@@ -841,6 +888,7 @@ def main() -> int:
     run_parser.add_argument("--published", action="store_true")
     run_parser.add_argument("--mcs-connection-id")
 
+    subparsers.add_parser("list-connections")
     subparsers.add_parser("list-runs")
 
     results_parser = subparsers.add_parser("results")
@@ -905,6 +953,12 @@ def main() -> int:
                 run_on_published_bot=args.published,
                 tools_connections=tools_connections,
             ))
+        elif args.command == "list-connections":
+            _print_json(list_mcs_connections(
+                config,
+                environment_id,
+                client.signed_in_username,
+            ))
         elif args.command == "list-runs":
             _print_json(list_runs(
                 client,

solutions/ess-maker-skills/scripts/fetch_and_setup.py

@@ -240,7 +240,7 @@ def save_temp_files(components, template_configs, workflows):
 
 
 def _resolve_refresh_target(args, config):
-    """Resolve the (env, bot, name, schema, managed) target for a refresh.
+    """Resolve the (env URL/ID, bot, name, schema, managed) refresh target.
 
     Explicit CLI overrides win over the stored config so ``--refresh`` can
     retarget an agent to a second environment. Without ``--url`` this is a
@@ -254,15 +254,20 @@ def _resolve_refresh_target(args, config):
     agent = config.get("agent", {})
     retargeting = bool(args.url)
     env_url = args.url.rstrip("/") if args.url else config["dataverseEndpoint"]
+    environment_id = (
+        args.environment_id
+        or agent.get("environmentId")
+        or config.get("environmentId")
+    )
     bot_id = args.bot_id or agent.get("botId")
     name = args.name or agent.get("name")
     schema = args.schema or agent.get("schemaName")
     managed = args.managed if retargeting else agent.get("isManaged", False)
-    return env_url, bot_id, name, schema, managed
+    return env_url, environment_id, bot_id, name, schema, managed
 
 
-def run_setup(env_url, args_bot_id, args_name, args_schema, args_managed,
-              paths, extra_flags=None):
+def run_setup(env_url, environment_id, args_bot_id, args_name, args_schema,
+              args_managed, paths, extra_flags=None):
     """Run setup.py with the given temp file paths."""
     print("\nRunning setup...\n")
     cmd = [
@@ -273,6 +278,8 @@ def run_setup(env_url, args_bot_id, args_name, args_schema, args_managed,
         "--schema", args_schema,
         "--components", paths["components"],
     ]
+    if environment_id:
+        cmd.extend(["--environment-id", environment_id])
     if args_managed:
         cmd.append("--managed")
     if "template_configs" in paths:
@@ -292,6 +299,10 @@ def main():
     parser.add_argument("--url",
                         help="Power Platform environment URL "
                              "(e.g. https://org.crm.dynamics.com)")
+    parser.add_argument(
+        "--environment-id",
+        help="Power Platform environment ID selected during discovery",
+    )
     parser.add_argument("--bot-id",
                         help="Bot ID (GUID) from Dataverse")
     parser.add_argument("--name",
@@ -308,7 +319,14 @@ def main():
     # retarget to a different env/bot (see _resolve_refresh_target) ---
     if args.refresh:
         config = load_config()
-        env_url, bot_id, name, schema, managed = _resolve_refresh_target(
+        (
+            env_url,
+            environment_id,
+            bot_id,
+            name,
+            schema,
+            managed,
+        ) = _resolve_refresh_target(
             args, config)
 
         if args.url:
@@ -332,8 +350,16 @@ def main():
             print(e.format_for_terminal())
             sys.exit(1)
         paths = save_temp_files(components, template_configs, workflows)
-        rc = run_setup(env_url, bot_id, name, schema, managed,
-                       paths, extra_flags=["--refresh"])
+        rc = run_setup(
+            env_url,
+            environment_id,
+            bot_id,
+            name,
+            schema,
+            managed,
+            paths,
+            extra_flags=["--refresh"],
+        )
         sys.exit(rc)
 
     # --- Normal mode: requires all arguments ---
@@ -362,8 +388,15 @@ def main():
         print(e.format_for_terminal())
         sys.exit(1)
     paths = save_temp_files(components, template_configs, workflows)
-    rc = run_setup(env_url, args.bot_id, args.name, args.schema,
-                   args.managed, paths)
+    rc = run_setup(
+        env_url,
+        args.environment_id,
+        args.bot_id,
+        args.name,
+        args.schema,
+        args.managed,
+        paths,
+    )
     sys.exit(rc)
 
 

solutions/ess-maker-skills/scripts/setup.py

@@ -584,6 +584,9 @@ def write_config(agent_info, slug, output_dir, template_configs_discovered,
         "slug": slug,
         "folder": output_dir.replace("\\", "/"),
     }
+    environment_id = str(agent_info.get("environmentId") or "").strip()
+    if environment_id:
+        agent_entry["environmentId"] = environment_id
 
     # Load existing config to preserve other agents and connections
     local_dir = ".local"
@@ -630,6 +633,10 @@ def write_config(agent_info, slug, output_dir, template_configs_discovered,
         "workflowCount": workflow_count,
         "evaluationCount": evaluation_count,
     }
+    if environment_id:
+        config["environmentId"] = environment_id
+    elif existing.get("environmentId"):
+        config["environmentId"] = existing["environmentId"]
 
     # Preserve existing connections and other user-set fields
     for key in ("connections", "workdayTestEmployeeId", "referenceSource", "environmentSku"):
@@ -753,6 +760,10 @@ def main():
         description="ESS Maker Kit — one-shot setup")
     parser.add_argument("--url", required=True,
                         help="Power Platform environment URL")
+    parser.add_argument(
+        "--environment-id",
+        help="Power Platform environment ID selected during discovery",
+    )
     parser.add_argument("--bot-id", required=True,
                         help="Selected bot ID from Dataverse")
     parser.add_argument("--name", required=True,
@@ -781,6 +792,7 @@ def main():
         "schema": args.schema,
         "managed": args.managed,
         "url": args.url,
+        "environmentId": args.environment_id,
     }
 
     # --- Idempotency gate: refuse silent overwrite of existing agent dir ---

solutions/ess-maker-skills/scripts/setup_state.py

@@ -1187,6 +1187,7 @@ def _state_view(state: SetupState, view: str) -> dict[str, Any]:
         "connect_ready": state.connect_ready,
         "environment": {
             "locked": bool(state.environment.get("locked")),
+            "id": state.environment.get("id"),
             "tenant_endpoint": state.environment.get("tenant_endpoint"),
         },
         "completed_steps": [

solutions/ess-maker-skills/src/skills/evaluations/run/SKILL.md

@@ -92,12 +92,27 @@ Copilot Studio connections in the selected environment and keeps only profiles
 whose status is `Connected`.
 
 - If exactly one profile is connected, use it automatically.
-- If multiple profiles exist, use the one that uniquely matches the signed-in
+- If multiple profiles exist, use the latest profile matching the signed-in
   Power Apps account.
-- If multiple connected profiles remain, automatically use the first profile
-  in deterministic name/ID order and try the run without asking the user.
-- If none are connected, stop and explain that the user must create or repair
-  the connection in Power Apps or Power Automate.
+- If automatic matching fails, run:
+
+  ```text
+  python scripts/evaluation_runs.py list-connections
+  ```
+
+  Display every returned profile with its display name, account name, creator,
+  and connection ID. Ask the user to select one using `vscode_askQuestions`,
+  then **STOP**. Do not start the evaluation in the same turn.
+- After the user selects a profile, retry the previously selected test set:
+
+  ```text
+  python scripts/evaluation_runs.py run --test-set-id "{id}" --test-set-name "{displayName}" --mcs-connection-id "{connectionId}"
+  ```
+
+- If no connected profile is returned, explain that the connection must be
+  created or repaired in Power Apps or Power Automate. Ask the user to choose
+  **Retry connection discovery** when ready, then stop. On their next turn,
+  rerun `list-connections`.
 
 Every run must include a validated `mcsConnectionId`; do not start an
 anonymous evaluation run.

solutions/ess-maker-skills/src/skills/onboarding/step1.md

@@ -17,10 +17,11 @@ Parse the printed state. If `connect_ready` is true and `environment.locked` is
 true:
 
 1. Set ENV_URL to `environment.tenant_endpoint`, stripping any trailing slash.
-2. Do not list environments, ask how to provide an environment, or ask the
+2. Set ENVIRONMENT_ID to `environment.id`.
+3. Do not list environments, ask how to provide an environment, or ask the
    maker to select it again.
-3. Set FOUNDATION_REUSED to true.
-4. Continue directly to section 1.2.
+4. Set FOUNDATION_REUSED to true.
+5. Continue directly to section 1.2.
 
 Only continue to section 1.0 when no completed foundation state with a locked
 environment exists.
@@ -102,7 +103,8 @@ Map the selected URL to the unique matching `instanceUrl` in
 
 ## 1.1b — Use selection
 
-Read the selected object's `instanceUrl` field. Save it as ENV_URL.
+Read the selected object's `instanceUrl` field. Save it as ENV_URL. Read its
+`id` field and save it as ENVIRONMENT_ID.
 **Strip any trailing slash** from ENV_URL before using it (e.g.,
 `https://org.crm.dynamics.com/` becomes `https://org.crm.dynamics.com`).
 
@@ -127,6 +129,14 @@ Save their answer as ENV_URL. **Strip any trailing
 slash** from ENV_URL before using it (e.g., `https://org.crm.dynamics.com/`
 becomes `https://org.crm.dynamics.com`).
 
+Resolve the manually entered URL:
+
+```text
+python scripts/discover.py --resolve-environment-url "{ENV_URL}"
+```
+
+Parse `SELECTED_ENV_JSON:` and save its `id` field as ENVIRONMENT_ID.
+
 ## 1.2 — Write the MCP config file
 
 Build the MCP URL by appending `/api/mcp` to ENV_URL. Double-check the

solutions/ess-maker-skills/src/skills/onboarding/step1b.md

@@ -3,7 +3,7 @@
 Every **Message** block is the exact text to show the user. Copy it verbatim.
 Do not rephrase, add commentary, or tell the user what tools you are calling.
 
-You should already have ENV_URL from Step 1.
+You should already have ENV_URL and ENVIRONMENT_ID from Step 1.
 
 ---
 

solutions/ess-maker-skills/src/skills/onboarding/step2.md

@@ -3,8 +3,8 @@
 Every **Message** block is the exact text to show the user. Copy it verbatim.
 Do not rephrase, add commentary, or tell the user what tools you are calling.
 
-You should already have these values from Step 1: ENV_URL, BOT_ID, BOT_NAME,
-SCHEMA_NAME, IS_MANAGED.
+You should already have these values from Step 1: ENV_URL, ENVIRONMENT_ID,
+BOT_ID, BOT_NAME, SCHEMA_NAME, IS_MANAGED.
 
 ---
 
@@ -19,7 +19,7 @@ Extracting your agent — this usually takes 10–20 seconds...
 Run this single command in the terminal (substitute all values):
 
 ```
-python scripts/fetch_and_setup.py --url "{ENV_URL}" --bot-id "{BOT_ID}" --name "{BOT_NAME}" --schema "{SCHEMA_NAME}" {--managed if IS_MANAGED is true}
+python scripts/fetch_and_setup.py --url "{ENV_URL}" --environment-id "{ENVIRONMENT_ID}" --bot-id "{BOT_ID}" --name "{BOT_NAME}" --schema "{SCHEMA_NAME}" {--managed if IS_MANAGED is true}
 ```
 
 The script authenticates to Dataverse via the browser (the user will see an