Allow a read-only MCP tool as a connection health check

.changeset/mcp-health-tool-probes.md

@@ -0,0 +1,5 @@
+---
+"executor": minor
+---
+
+Allow an explicitly selected read-only MCP tool to check connection health with JSON arguments and an optional account identity field. Keep discovery-only checks when no tool is configured, and report tool errors, rejected credentials, and insufficient scope without retaining upstream error text.

e2e/scenarios/health-checks-mcp-probe-ui.test.ts

@@ -0,0 +1,124 @@
+// Opt-in MCP health probes. Without a configured probe an MCP connection's
+// health is discovery only (dial + tools/list), and the editor offers only
+// tools the server declares read-only on every connection.
+//
+// The upstream is a per-run instance of the published MCP emulator. Its tools
+// carry no annotations, so none of them may serve as a probe: the editor stays
+// hidden, nothing is saved, and no check ever calls a tool. The positive path
+// (picking and probing a read-only tool) needs an upstream that advertises
+// `readOnlyHint`, which the published emulator does not; plugin tests cover it.
+import { randomBytes } from "node:crypto";
+
+import { Effect } from "effect";
+import { expect } from "@effect/vitest";
+import { composePluginApi } from "@executor-js/api/server";
+import { connectEmulator } from "@executor-js/emulate";
+import { mcpHttpPlugin } from "@executor-js/plugin-mcp/api";
+import { variable } from "@executor-js/sdk/http-auth";
+import { AuthTemplateSlug, ConnectionName, IntegrationSlug } from "@executor-js/sdk/shared";
+
+import { createEmulatorInstance } from "../src/emulator-instance";
+import { scenario } from "../src/scenario";
+import { Api, Browser, Target } from "../src/services";
+import { visit } from "../src/surfaces/browser";
+
+const api = composePluginApi([mcpHttpPlugin()] as const);
+const template = AuthTemplateSlug.make("bearer");
+const name = ConnectionName.make("main");
+
+scenario(
+  "Health checks (UI) · MCP offers no unannotated tool and checks discovery only",
+  {},
+  Effect.scoped(
+    Effect.gen(function* () {
+      const target = yield* Target;
+      const browser = yield* Browser;
+      const { client: makeClient } = yield* Api;
+      const identity = yield* target.newIdentity();
+      const client = yield* makeClient(api, identity);
+      const slug = IntegrationSlug.make(`hc-mcp-probe-${randomBytes(4).toString("hex")}`);
+      const baseUrl = yield* createEmulatorInstance("mcp", "mcp-health-probe");
+      const emulator = yield* Effect.promise(() => connectEmulator({ baseUrl }));
+      const credential = yield* Effect.promise(() =>
+        emulator.credentials.mint({ type: "bearer-token" }),
+      );
+      const token = credential.token;
+      if (!token) return yield* Effect.die("Emulator did not mint a bearer token");
+
+      yield* Effect.ensuring(
+        Effect.gen(function* () {
+          yield* client.mcp.addServer({
+            payload: {
+              transport: "remote",
+              name: "Emulated MCP",
+              endpoint: `${baseUrl}/mcp`,
+              slug: String(slug),
+              authenticationTemplate: [
+                {
+                  slug: "bearer",
+                  type: "apiKey",
+                  headers: { Authorization: ["Bearer ", variable("token")] },
+                },
+              ],
+            },
+          });
+          yield* client.connections.create({
+            payload: { owner: "org", name, integration: slug, template, value: token },
+          });
+
+          const candidates = yield* client.integrations.healthCheckCandidates({
+            params: { slug },
+          });
+          expect(candidates, "an unannotated tool is never a probe candidate").toEqual([]);
+
+          yield* browser.session(identity, async ({ page, step }) => {
+            await step("The integration offers no MCP health check to set up", async () => {
+              await visit(page, `/integrations/${slug}`);
+              await page.getByText("main", { exact: true }).first().waitFor();
+              expect(
+                await page.getByRole("heading", { name: "Health check", exact: true }).count(),
+              ).toBe(0);
+              expect(await page.getByRole("button", { name: "Set up", exact: true }).count()).toBe(
+                0,
+              );
+            });
+          });
+
+          expect(yield* client.integrations.healthCheckGet({ params: { slug } })).toBeNull();
+          const discoveryOnly = yield* client.connections.checkHealth({
+            params: { owner: "org", integration: slug, name },
+            query: {},
+          });
+          expect(discoveryOnly.status, "no probe configured: discovery health").toBe("healthy");
+          expect(discoveryOnly.detail).toContain("discovery only");
+
+          const rejected = yield* client.connections.validate({
+            payload: { owner: "org", integration: slug, template, value: "wrong-token" },
+          });
+          expect(rejected.status, "a rejected key is expired, not healthy").toBe("expired");
+          expect(JSON.stringify(rejected)).not.toContain("Requires authentication");
+
+          const requests = yield* Effect.promise(() => emulator.ledger.list());
+          expect(
+            requests.filter((request) =>
+              JSON.stringify(request.request.body ?? null).includes('"tools/call"'),
+            ),
+            "no health check called a tool",
+          ).toEqual([]);
+          expect(
+            requests.some((request) =>
+              JSON.stringify(request.request.body ?? null).includes('"tools/list"'),
+            ),
+            "discovery reached the emulator",
+          ).toBe(true);
+        }),
+        Effect.gen(function* () {
+          yield* client.connections
+            .remove({ params: { owner: "org", integration: slug, name } })
+            .pipe(Effect.ignore);
+          yield* client.mcp.removeServer({ params: { slug } }).pipe(Effect.ignore);
+        }),
+      );
+    }),
+  ),
+);

packages/core/sdk/src/executor.ts

@@ -177,6 +177,7 @@ import type {
   OwnerBinding,
   PluginCtx,
   PluginExtensions,
+  RefreshedToolCatalog,
   ResolveToolsResult,
   StaticIntegrationDecl,
   StaticToolDecl,
@@ -3518,7 +3519,7 @@ export const createExecutor = <const TPlugins extends readonly AnyPlugin[] = rea
       integrationRow: IntegrationRow,
       ref: ConnectionRef,
       mode: () => "explicit" | "background",
-    ): Effect.Effect<readonly Tool[], IntegrationNotFoundError | StorageFailure> =>
+    ): Effect.Effect<RefreshedToolCatalog, IntegrationNotFoundError | StorageFailure> =>
       Effect.gen(function* () {
         const runtime = runtimes.get(integrationRow.plugin_id);
         const keys = yield* Effect.try({
@@ -3638,7 +3639,7 @@ export const createExecutor = <const TPlugins extends readonly AnyPlugin[] = rea
               yield* stampSynced(existingRow);
             }),
           );
-          return [];
+          return { tools: [], complete: true };
         }
 
         if (!runtime?.plugin.resolveTools) {
@@ -3650,7 +3651,7 @@ export const createExecutor = <const TPlugins extends readonly AnyPlugin[] = rea
               yield* stampSynced(existingRow);
             }),
           );
-          return [];
+          return { tools: [], complete: true };
         }
 
         const result: ResolveToolsResult = yield* runtime.plugin
@@ -3685,7 +3686,10 @@ export const createExecutor = <const TPlugins extends readonly AnyPlugin[] = rea
             connection: String(ref.name),
           });
           const keptRows = yield* core.findMany("tool", { where });
-          return keptRows.map((row) => rowToTool(row as ConnectionToolRow));
+          return {
+            tools: keptRows.map((row) => rowToTool(row as ConnectionToolRow)),
+            complete: false,
+          };
         }
 
         if (
@@ -3704,7 +3708,10 @@ export const createExecutor = <const TPlugins extends readonly AnyPlugin[] = rea
               connection: String(ref.name),
               existingToolCount: keptRows.length,
             });
-            return keptRows.map((row) => rowToTool(row as ConnectionToolRow));
+            return {
+              tools: keptRows.map((row) => rowToTool(row as ConnectionToolRow)),
+              complete: false,
+            };
           }
         }
 
@@ -3747,7 +3754,7 @@ export const createExecutor = <const TPlugins extends readonly AnyPlugin[] = rea
           }),
         );
 
-        return result.tools.map((tool: ToolDef) =>
+        const tools = result.tools.map((tool: ToolDef) =>
           rowToTool(
             {
               tenant: keys.tenant,
@@ -3767,19 +3774,20 @@ export const createExecutor = <const TPlugins extends readonly AnyPlugin[] = rea
             tool.annotations,
           ),
         );
+        return { tools, complete: true };
       });
 
     type ToolProductionError = IntegrationNotFoundError | StorageFailure;
     interface ToolProductionInFlight {
-      readonly deferred: Deferred.Deferred<readonly Tool[], ToolProductionError>;
+      readonly deferred: Deferred.Deferred<RefreshedToolCatalog, ToolProductionError>;
       mode: "explicit" | "background";
     }
     const toolProductionInFlight = new Map<string, ToolProductionInFlight>();
     const produceConnectionTools = (
       integrationRow: IntegrationRow,
       ref: ConnectionRef,
       requestedMode: "explicit" | "background" = "explicit",
-    ): Effect.Effect<readonly Tool[], ToolProductionError> =>
+    ): Effect.Effect<RefreshedToolCatalog, ToolProductionError> =>
       Effect.suspend(() => {
         const key = `${ref.owner}:${String(ref.integration)}:${String(ref.name)}`;
         const existing = toolProductionInFlight.get(key);
@@ -3789,7 +3797,7 @@ export const createExecutor = <const TPlugins extends readonly AnyPlugin[] = rea
         }
 
         const entry: ToolProductionInFlight = {
-          deferred: Deferred.makeUnsafe<readonly Tool[], ToolProductionError>(),
+          deferred: Deferred.makeUnsafe<RefreshedToolCatalog, ToolProductionError>(),
           mode: requestedMode,
         };
         toolProductionInFlight.set(key, entry);
@@ -4863,10 +4871,10 @@ export const createExecutor = <const TPlugins extends readonly AnyPlugin[] = rea
         }),
       );
 
-    const connectionsRefresh = (
+    const connectionsRefreshCatalog = (
       ref: ConnectionRef,
     ): Effect.Effect<
-      readonly Tool[],
+      RefreshedToolCatalog,
       ConnectionNotFoundError | IntegrationNotFoundError | OrgWriteDeniedError | StorageFailure
     > =>
       Effect.gen(function* () {
@@ -4886,6 +4894,9 @@ export const createExecutor = <const TPlugins extends readonly AnyPlugin[] = rea
         return yield* produceConnectionTools(integrationRow, ref);
       });
 
+    const connectionsRefresh = (ref: ConnectionRef) =>
+      connectionsRefreshCatalog(ref).pipe(Effect.map((catalog) => catalog.tools));
+
     // No health-check capability ⇒ "unknown" rather than an error: the caller
     // can still render the connection, just without a liveness verdict.
     const unknownHealth = (): HealthCheckResult => ({
@@ -5603,8 +5614,8 @@ export const createExecutor = <const TPlugins extends readonly AnyPlugin[] = rea
         // runs entirely in the background and the read answers from the
         // persisted rows at once. Without that split every read after the TTL
         // paid the grace budget for MCP listings it had no reason to wait on.
-        const urgent: Effect.Effect<readonly Tool[]>[] = [];
-        const deferred: Effect.Effect<readonly Tool[]>[] = [];
+        const urgent: Effect.Effect<void>[] = [];
+        const deferred: Effect.Effect<void>[] = [];
         for (const connection of connections) {
           const integrationRow = integrationBySlug.get(connection.integration);
           if (!integrationRow) continue;
@@ -5641,6 +5652,7 @@ export const createExecutor = <const TPlugins extends readonly AnyPlugin[] = rea
               },
               "background",
             ).pipe(
+              Effect.asVoid,
               // Best-effort, but never silent: the read still succeeds on the
               // stale-but-working catalog and the peer rebuilds still finish,
               // while the operator gets the connection that failed and why.
@@ -5651,7 +5663,7 @@ export const createExecutor = <const TPlugins extends readonly AnyPlugin[] = rea
                   integration: connection.integration,
                   connection: connection.name,
                   errorTag: Predicate.isTagged(error, "StorageError") ? "StorageError" : "Unknown",
-                }).pipe(Effect.as([] as readonly Tool[])),
+                }),
               ),
               Effect.withSpan("executor.tools.sync_stale", {
                 attributes: {
@@ -6969,6 +6981,7 @@ export const createExecutor = <const TPlugins extends readonly AnyPlugin[] = rea
           update: (ref, input) => connectionsUpdate(ref, input),
           remove: (ref) => connectionsRemove(ref),
           refresh: (ref) => connectionsRefresh(ref),
+          refreshCatalog: (ref) => connectionsRefreshCatalog(ref),
           checkHealth: (ref, options) => connectionCheckHealth(ref, options),
           markToolsStale: (ref) => connectionsMarkToolsStale(ref),
           resolveValue: (ref) => resolveConnectionValueByRef(ref),

packages/core/sdk/src/health-check.ts

@@ -187,6 +187,14 @@ export const HealthCheckCandidate = Schema.Struct({
   /** True for mutating methods (post/put/patch/delete), ranked last and shown
    *  with a warning, since a health check should be safe to run repeatedly. */
   destructive: Schema.Boolean,
+  /** The user must pick this candidate themselves: surfaces never pre-select
+   *  it, run it, or save it as the check on their own. Set when "read-only"
+   *  is the upstream's own claim (MCP tool annotations are hints) rather than
+   *  a protocol guarantee such as HTTP GET. */
+  explicitSelectionOnly: Schema.optional(Schema.Boolean),
+  /** The arguments are one typed JSON object (MCP tool arguments), edited
+   *  whole rather than as one text field per parameter. */
+  jsonArguments: Schema.optional(Schema.Boolean),
   /** Operation summary / description for display, when known. */
   summary: Schema.optional(Schema.String),
   /** The operation's parameters, so the editor can offer pinned-arg inputs. */

packages/core/sdk/src/index.ts

@@ -411,6 +411,7 @@ export {
   type IntegrationPresetAuthentication,
   type IntegrationPresetCatalogEntry,
   type ResolveToolsInput,
+  type RefreshedToolCatalog,
   type ResolveToolsResult,
   type ToolInvocationCredential,
   type HealthCheckInput,

packages/core/sdk/src/plugin.ts

@@ -265,6 +265,16 @@ export interface PluginCtx<TStore = unknown> {
       readonly Tool[],
       ConnectionNotFoundError | IntegrationNotFoundError | OrgWriteDeniedError | StorageFailure
     >;
+    /** `refresh`, also reporting whether the listing was authoritative. When
+     *  discovery came back incomplete, `tools` is the retained prior catalog
+     *  and `complete` is false — for callers that must not treat a stale
+     *  catalog as the live one. */
+    readonly refreshCatalog: (
+      ref: ConnectionRef,
+    ) => Effect.Effect<
+      RefreshedToolCatalog,
+      ConnectionNotFoundError | IntegrationNotFoundError | OrgWriteDeniedError | StorageFailure
+    >;
     /** Run the integration's declared health check against a saved connection
      *  and persist the verdict. `ifStaleMs` serves the persisted verdict when
      *  younger than that window, so concurrent readers collapse to one probe;
@@ -405,6 +415,13 @@ export interface ResolveToolsResult {
   readonly health?: HealthCheckResult;
 }
 
+/** A connection's catalog after a refresh. `complete` is false when the
+ *  listing was non-authoritative and core kept the prior persisted catalog. */
+export interface RefreshedToolCatalog {
+  readonly tools: readonly Tool[];
+  readonly complete: boolean;
+}
+
 export interface ProjectToolSchemaInput<TStore = unknown> {
   readonly ctx: PluginCtx<TStore>;
   readonly toolRow: ToolInvocationRow;

packages/plugins/mcp/src/react/McpAccountsPanel.tsx

@@ -5,6 +5,7 @@ import * as Exit from "effect/Exit";
 
 import { IntegrationSlug } from "@executor-js/sdk/shared";
 import type { IntegrationAccountHandoff } from "@executor-js/sdk/client";
+import { HealthCheckEditor } from "@executor-js/react/components/health-check-editor";
 import { AccountsSection } from "@executor-js/react/components/accounts-section";
 import type { AuthMethod, Placement } from "@executor-js/react/lib/auth-placements";
 import {
@@ -108,6 +109,7 @@ export default function McpAccountsPanel(props: {
         createCustomMethod={canConfigureAuth ? createCustomMethod : undefined}
         removeCustomMethod={canConfigureAuth ? removeCustomMethod : undefined}
       />
+      <HealthCheckEditor integration={slug} />
     </div>
   );
 }