From 846cfa31028ea949a1f34d767e9ff3180f4dd66b Mon Sep 17 00:00:00 2001 From: zak <36251971+zrm625@users.noreply.github.com> Date: Wed, 14 Oct 2026 00:07:09 -0400 Subject: [PATCH] Repair missing provider service catalogs and toolkit connections --- .changeset/repair-provider-catalogs.md | 6 + .../src/db/data-migrations.test.ts | 3 + .../host-cloudflare/src/db/data-migrations.ts | 2 + apps/host-selfhost/src/db/data-migrations.ts | 6 +- apps/local/src/db/data-migrations.ts | 6 +- e2e/selfhost/toolkits-ui.test.ts | 89 ++++++++++++ .../plugins/openapi/src/sdk/plugin.test.ts | 124 +++++++++++++++++ packages/plugins/openapi/src/sdk/plugin.ts | 120 ++++++++++++++++- .../src/catalog-repair.test.ts | 127 ++++++++++++++++++ .../src/catalog-repair.ts | 81 +++++++++++ .../provider-service-split/src/index.ts | 1 + packages/plugins/toolkits/src/page.tsx | 63 +++++++-- 12 files changed, 616 insertions(+), 12 deletions(-) create mode 100644 .changeset/repair-provider-catalogs.md create mode 100644 packages/plugins/provider-service-split/src/catalog-repair.test.ts create mode 100644 packages/plugins/provider-service-split/src/catalog-repair.ts diff --git a/.changeset/repair-provider-catalogs.md b/.changeset/repair-provider-catalogs.md new file mode 100644 index 0000000000..047546900a --- /dev/null +++ b/.changeset/repair-provider-catalogs.md @@ -0,0 +1,6 @@ +--- +"@executor-js/plugin-openapi": patch +"@executor-js/plugin-toolkits": patch +--- + +Repair missing Google and Microsoft operation catalogs through the stored service URL when an integration administrator reads tools. Preserve existing catalogs when repair cannot complete, and include connected accounts with zero tools in the toolkit connection picker. diff --git a/apps/host-cloudflare/src/db/data-migrations.test.ts b/apps/host-cloudflare/src/db/data-migrations.test.ts index 254c023776..ba6ee5a85d 100644 --- a/apps/host-cloudflare/src/db/data-migrations.test.ts +++ b/apps/host-cloudflare/src/db/data-migrations.test.ts @@ -240,6 +240,7 @@ describe("runCloudflareDataMigrations", () => { expect(yield* Effect.promise(() => runCloudflareDataMigrations(d1, bucket))).toEqual([ "2026-06-20-google-openapi-ownership", "2026-07-08-provider-service-split", + "2026-07-10-provider-service-empty-catalog-repair", "2026-07-09-openapi-ndjson-output-arrays", "2026-07-27-encrypted-secrets-owner-repartition", ]); @@ -285,6 +286,7 @@ describe("runCloudflareDataMigrations", () => { expect(yield* Effect.promise(() => runCloudflareDataMigrations(d1, bucket))).toEqual([ "2026-06-20-google-openapi-ownership", "2026-07-08-provider-service-split", + "2026-07-10-provider-service-empty-catalog-repair", "2026-07-09-openapi-ndjson-output-arrays", "2026-07-27-encrypted-secrets-owner-repartition", ]); @@ -337,6 +339,7 @@ describe("runCloudflareDataMigrations", () => { expect(yield* Effect.promise(() => runCloudflareDataMigrations(d1, bucket))).toEqual([ "2026-06-20-google-openapi-ownership", "2026-07-08-provider-service-split", + "2026-07-10-provider-service-empty-catalog-repair", "2026-07-09-openapi-ndjson-output-arrays", "2026-07-27-encrypted-secrets-owner-repartition", ]); diff --git a/apps/host-cloudflare/src/db/data-migrations.ts b/apps/host-cloudflare/src/db/data-migrations.ts index 1efb4340dd..c14020af1a 100644 --- a/apps/host-cloudflare/src/db/data-migrations.ts +++ b/apps/host-cloudflare/src/db/data-migrations.ts @@ -13,6 +13,7 @@ import { encryptedSecretsRepartitionDataMigration } from "@executor-js/plugin-en import { providerServiceSplitDataMigration, + providerServiceCatalogRepairDataMigration, runSqliteProviderServiceSplitMigration, type BlobRow, } from "@executor-js/plugin-provider-service-split"; @@ -248,6 +249,7 @@ const cloudflareDataMigrations = (bucket: R2Bucket | undefined): readonly Sqlite }).pipe(Effect.asVoid); }), }, + providerServiceCatalogRepairDataMigration, // Stale-mark connections whose operations return NDJSON so their tool rows // rebuild with array-wrapped output schemas (mirrors cloud's drizzle 0010). openApiNdjsonOutputDataMigration, diff --git a/apps/host-selfhost/src/db/data-migrations.ts b/apps/host-selfhost/src/db/data-migrations.ts index aaf3b2295d..153a07914f 100644 --- a/apps/host-selfhost/src/db/data-migrations.ts +++ b/apps/host-selfhost/src/db/data-migrations.ts @@ -19,7 +19,10 @@ import { import { graphqlIntrospectionBlobDataMigration } from "@executor-js/plugin-graphql"; import { googleOpenApiOwnershipDataMigration } from "@executor-js/plugin-openapi/providers/google"; -import { providerServiceSplitDataMigration } from "@executor-js/plugin-provider-service-split"; +import { + providerServiceSplitDataMigration, + providerServiceCatalogRepairDataMigration, +} from "@executor-js/plugin-provider-service-split"; import { encryptedSecretsRepartitionDataMigration } from "@executor-js/plugin-encrypted-secrets"; import { authConfigTransforms } from "./auth-config-migration"; @@ -44,6 +47,7 @@ export const selfHostDataMigrations: readonly SqliteDataMigration[] = [ graphqlIntrospectionBlobDataMigration, googleOpenApiOwnershipDataMigration, providerServiceSplitDataMigration, + providerServiceCatalogRepairDataMigration, // Stale-mark connections whose operations return NDJSON so their tool rows // rebuild with array-wrapped output schemas (mirrors cloud's drizzle 0010). openApiNdjsonOutputDataMigration, diff --git a/apps/local/src/db/data-migrations.ts b/apps/local/src/db/data-migrations.ts index 0534b26f44..2c8ad952d8 100644 --- a/apps/local/src/db/data-migrations.ts +++ b/apps/local/src/db/data-migrations.ts @@ -21,7 +21,10 @@ import { import { graphqlIntrospectionBlobDataMigration } from "@executor-js/plugin-graphql"; import { googleOpenApiOwnershipDataMigration } from "@executor-js/plugin-openapi/providers/google"; -import { providerServiceSplitDataMigration } from "@executor-js/plugin-provider-service-split"; +import { + providerServiceSplitDataMigration, + providerServiceCatalogRepairDataMigration, +} from "@executor-js/plugin-provider-service-split"; import { authConfigTransforms } from "./auth-config-migration"; import { LOCAL_V1_V2_LEDGER_NAME } from "./v1-v2-migration"; @@ -51,6 +54,7 @@ export const localDataMigrations: readonly SqliteDataMigration[] = [ graphqlIntrospectionBlobDataMigration, googleOpenApiOwnershipDataMigration, providerServiceSplitDataMigration, + providerServiceCatalogRepairDataMigration, // GC dead DCR oauth_client rows (old always-register duplicates) and backfill // the surviving DCR rows' origin_issuer from token_url (issue #1120, Part C). oauthClientGcSqliteMigration, diff --git a/e2e/selfhost/toolkits-ui.test.ts b/e2e/selfhost/toolkits-ui.test.ts index aac0e96dd6..377afca67b 100644 --- a/e2e/selfhost/toolkits-ui.test.ts +++ b/e2e/selfhost/toolkits-ui.test.ts @@ -27,6 +27,27 @@ const hiddenPersonalSpec = (baseUrl: string): string => }, }); +const connectedGoogleSpec = (): string => + JSON.stringify({ + openapi: "3.0.3", + info: { title: "Gmail", version: "1.0.0" }, + paths: { + "/messages": { + get: { + operationId: "gmail.users.messages.list", + responses: { "200": { description: "Messages" } }, + }, + }, + }, + }); + +const connectedWithoutToolsSpec = (): string => + JSON.stringify({ + openapi: "3.0.3", + info: { title: "Gmail", version: "1.0.0" }, + paths: {}, + }); + scenario( "Toolkits · self-host UI creates a toolkit and configures tools", { timeout: 180_000 }, @@ -43,6 +64,8 @@ scenario( const slug = name; const hiddenPersonalIntegration = `${prefix}-personal-api`; const hiddenPersonalConnection = "mine"; + const connectedWithoutToolsIntegration = `${prefix}-gmail`; + const connectedWithoutToolsConnection = "waitingAccount"; const seededToolkits = [ { owner: "org" as const, name: `${prefix}-workspace-a` }, { owner: "org" as const, name: `${prefix}-workspace-b` }, @@ -63,9 +86,21 @@ scenario( }, }) .pipe(Effect.ignore); + yield* client.connections + .remove({ + params: { + owner: "org", + integration: IntegrationSlug.make(connectedWithoutToolsIntegration), + name: ConnectionName.make(connectedWithoutToolsConnection), + }, + }) + .pipe(Effect.ignore); yield* client.openapi .removeSpec({ params: { slug: hiddenPersonalIntegration } }) .pipe(Effect.ignore); + yield* client.openapi + .removeSpec({ params: { slug: connectedWithoutToolsIntegration } }) + .pipe(Effect.ignore); const listed = yield* client.toolkits.list(); yield* Effect.forEach( listed.toolkits.filter((row) => row.slug.startsWith(prefix)), @@ -103,6 +138,42 @@ scenario( value: "unused-token", }, }); + yield* client.openapi.addSpec({ + payload: { + spec: { kind: "blob", value: connectedGoogleSpec() }, + slug: IntegrationSlug.make(connectedWithoutToolsIntegration), + authenticationTemplate: [ + { + slug: "apiKey", + type: "apiKey", + headers: { "x-api-key": [{ type: "variable", name: "token" }] }, + }, + ], + }, + }); + yield* client.connections.create({ + payload: { + owner: "org", + name: ConnectionName.make(connectedWithoutToolsConnection), + integration: IntegrationSlug.make(connectedWithoutToolsIntegration), + template: AuthTemplateSlug.make("apiKey"), + value: "unused-token", + }, + }); + yield* client.openapi.updateSpec({ + params: { slug: connectedWithoutToolsIntegration }, + payload: { spec: { kind: "blob", value: connectedWithoutToolsSpec() } }, + }); + const connectedAccounts = yield* client.connections.list({ + query: { + owner: "org", + integration: IntegrationSlug.make(connectedWithoutToolsIntegration), + }, + }); + expect( + connectedAccounts.map((connection) => connection.name), + "the Google account remains connected while its refreshed tool catalog is empty", + ).toContain(connectedWithoutToolsConnection); yield* browser.session(identity, async ({ page, step }) => { await step("Open the Toolkits plugin page", async () => { @@ -213,6 +284,24 @@ scenario( .waitFor(); }); + await step( + "A connected Google account remains addable before its tools are available", + async () => { + const dialog = page.getByRole("dialog", { name: "Manage connections" }); + await dialog + .getByLabel("Search connections and tools") + .fill(connectedWithoutToolsConnection); + await dialog.getByText(connectedWithoutToolsConnection, { exact: true }).waitFor(); + await dialog.getByText("Gmail · 0 tools", { exact: true }).waitFor(); + const add = dialog.getByRole("button", { name: /^Add connection / }); + await add.click(); + const remove = dialog.getByRole("button", { name: /^Remove connection / }); + await remove.waitFor(); + await remove.click(); + await add.waitFor(); + }, + ); + await step("Add a connection to the toolkit", async () => { const dialog = page.getByRole("dialog", { name: "Manage connections" }); await dialog.waitFor(); diff --git a/packages/plugins/openapi/src/sdk/plugin.test.ts b/packages/plugins/openapi/src/sdk/plugin.test.ts index 1e125e3b98..31fd505234 100644 --- a/packages/plugins/openapi/src/sdk/plugin.test.ts +++ b/packages/plugins/openapi/src/sdk/plugin.test.ts @@ -1394,6 +1394,130 @@ paths: }), ); + it.effect("repairs a migrated provider connection whose operation catalog is missing", () => + Effect.scoped( + Effect.gen(function* () { + const specServer = yield* serveMutableOpenApiSpecTestServer({ initialApi: TestApi }); + const config = makeTestConfig({ plugins: testPlugins() }); + const executor = yield* createExecutor(config); + const integration = IntegrationSlug.make("google_gmail"); + + yield* executor.openapi.addSpec({ + spec: { kind: "url", url: specServer.specUrl }, + slug: integration, + family: "google", + baseUrl: specServer.baseUrl, + authenticationTemplate: [apiKeyTemplate], + }); + yield* executor.connections.create({ + owner: "org", + name: ConnectionName.make("main"), + integration, + template: AuthTemplateSlug.make("apiKey"), + value: "secret-key-123", + }); + expect((yield* executor.tools.list({ integration })).length).toBeGreaterThan(0); + + // The migrated hash can still refer to a monolith. Re-fetch the + // selected service instead of rebuilding tools from that old blob. + yield* specServer.setApi( + HttpApi.make("selectedService").add( + HttpApiGroup.make("items").add( + HttpApiEndpoint.get("listItems", "/items", { success: Schema.Array(Item) }), + ), + ), + ); + + // Reproduce the bad service-split state: the integration + source URL + // and connection survived, but operation storage and tool rows did not, + // while the follow-up migration has marked the connection stale. + yield* Effect.promise(() => + config.db.deleteMany("plugin_storage", { + where: (b) => b.and(b("plugin_id", "=", "openapi"), b("collection", "=", "operation")), + }), + ); + yield* Effect.promise(() => + config.db.deleteMany("tool", { + where: (b) => b("integration", "=", String(integration)), + }), + ); + yield* Effect.promise(() => + config.db.updateMany("connection", { + where: (b) => b("integration", "=", String(integration)), + set: { tools_synced_at: null }, + }), + ); + + const repaired = yield* executor.tools.list({ integration }); + expect( + repaired.map((tool) => String(tool.name)), + "the next catalog read refetches the provider spec and restores tools", + ).toContain("items.listItems"); + expect(repaired.map((tool) => String(tool.name))).not.toContain("items.queryRows"); + const operations = yield* Effect.promise(() => + config.db.findMany("plugin_storage", { + where: (b) => b.and(b("plugin_id", "=", "openapi"), b("collection", "=", "operation")), + }), + ); + expect(operations.length, "the repaired operation catalog is persisted").toBeGreaterThan(0); + }), + ), + ); + + it.effect("member reads preserve the prior provider tools without repairing org storage", () => + Effect.scoped( + Effect.gen(function* () { + const specServer = yield* serveMutableOpenApiSpecTestServer({ initialApi: TestApi }); + const config = makeTestConfig({ plugins: testPlugins() }); + const blobs = recordingBlobStore(); + const admin = yield* createExecutor({ ...config, blobs: blobs.store }); + const integration = IntegrationSlug.make("microsoft_calendar"); + yield* admin.openapi.addSpec({ + spec: { kind: "url", url: specServer.specUrl }, + slug: integration, + family: "microsoft", + authenticationTemplate: [apiKeyTemplate], + }); + yield* admin.connections.create({ + owner: "org", + name: ConnectionName.make("main"), + integration, + template: AuthTemplateSlug.make("apiKey"), + value: "synthetic-api-key", + }); + const before = yield* admin.tools.list({ integration }); + yield* Effect.promise(() => + config.db.deleteMany("plugin_storage", { + where: (b) => b.and(b("plugin_id", "=", "openapi"), b("collection", "=", "operation")), + }), + ); + yield* Effect.promise(() => + config.db.updateMany("connection", { + where: (b) => b("integration", "=", String(integration)), + set: { tools_synced_at: null }, + }), + ); + const writes = blobs.writeCount(); + const member = yield* createExecutor({ + ...config, + blobs: blobs.store, + orgWrites: "denied", + }); + const after = yield* member.tools.list({ integration }); + expect(after.map((tool) => String(tool.name))).toEqual( + before.map((tool) => String(tool.name)), + ); + expect(blobs.writeCount()).toBe(writes); + const operations = yield* Effect.promise(() => + config.db.findMany("plugin_storage", { + where: (b) => b.and(b("plugin_id", "=", "openapi"), b("collection", "=", "operation")), + }), + ); + expect(operations).toEqual([]); + }), + ), + ); + it.effect("updateSpec accepts new inline content for blob-sourced integrations", () => Effect.scoped( Effect.gen(function* () { diff --git a/packages/plugins/openapi/src/sdk/plugin.ts b/packages/plugins/openapi/src/sdk/plugin.ts index b41816c1c9..60936a40bf 100644 --- a/packages/plugins/openapi/src/sdk/plugin.ts +++ b/packages/plugins/openapi/src/sdk/plugin.ts @@ -734,6 +734,89 @@ export const openApiPlugin = definePlugin< ); }); + const migratedProviderFamilies = new Set(["google", "microsoft"]); + + const repairMissingMigratedProviderCatalog = (input: { + readonly ctx: PluginCtx; + readonly integration: Integration; + readonly current: OpenApiIntegrationConfig; + readonly storage: OpenapiStore; + readonly httpClientLayer: Layer.Layer; + }) => + Effect.gen(function* () { + const specUrl = input.current.specUrl; + if ( + specUrl === undefined || + input.current.family === undefined || + !migratedProviderFamilies.has(input.current.family) + ) { + return undefined; + } + + const slug = String(input.integration.slug); + const operations = yield* input.storage.listOperations(slug); + if (operations.length > 0) return undefined; + + yield* input.ctx.core.integrations.authorizeWrite(); + const resolved = yield* resolveSpecForInput( + { + spec: { kind: "url", url: specUrl }, + specFormat: input.current.specFormat, + specOverrides: input.current.specOverrides, + authenticationTemplate: input.current.authenticationTemplate, + headers: input.current.headers, + queryParams: input.current.queryParams, + baseUrl: input.current.baseUrl, + }, + input.httpClientLayer, + ); + const compiled = resolved.keepPathItem + ? undefined + : yield* compileOpenApiSpec(resolved.specText); + const specHash = yield* sha256Hex(resolved.specText); + const sourceSpecHash = input.current.specOverrides?.length + ? yield* sha256Hex(resolved.sourceSpecText) + : undefined; + yield* input.storage.putSpec(specHash, resolved.specText); + if (sourceSpecHash) yield* input.storage.putSpec(sourceSpecHash, resolved.sourceSpecText); + if (compiled) { + yield* input.storage.putDefs(specHash, encodeJsonText(compiled.hoistedDefs)); + } + + const nextConfig: OpenApiIntegrationConfig = { + ...input.current, + specHash, + ...(sourceSpecHash ? { sourceSpecHash } : {}), + specUrl: resolved.specUrl ?? specUrl, + }; + yield* input.ctx.transaction( + Effect.gen(function* () { + yield* input.ctx.core.integrations.update(input.integration.slug, { + config: nextConfig, + }); + if (compiled) { + yield* input.storage.putOperations( + slug, + openApiStoredOperationsFromCompiled(slug, compiled), + ); + } else { + yield* compileAndPersistOpenApiSpecStreaming({ + specText: resolved.specText, + integration: slug, + storage: input.storage, + specHash, + keepPathItem: resolved.keepPathItem, + }); + } + }), + ); + return nextConfig; + }).pipe( + Effect.withSpan("openapi.plugin.repair_missing_migrated_provider_catalog", { + attributes: { "openapi.integration.slug": String(input.integration.slug) }, + }), + ); + return { id: "openapi" as const, packageName: "@executor-js/plugin-openapi", @@ -1391,8 +1474,41 @@ export const openApiPlugin = definePlugin< // operation bindings invokeTool needs are persisted at addSpec time; this // hook only shapes the per-connection ToolDefs from the spec blob the // catalog config points at. - resolveTools: ({ integration, config, storage }) => - resolveOpenApiBackedTools({ integration, config, storage }), + resolveTools: (input) => + Effect.gen(function* () { + const stored = yield* resolveOpenApiBackedTools(input); + if (input.ctx === undefined) return stored; + + const current = decodeOpenApiIntegrationConfig(input.config); + if ( + current?.family === undefined || + !migratedProviderFamilies.has(current.family) || + current.specUrl === undefined + ) { + return stored; + } + + const repaired = yield* repairMissingMigratedProviderCatalog({ + ctx: input.ctx, + integration: input.integration, + current, + storage: input.storage, + httpClientLayer: options?.httpClientLayer ?? input.httpClientLayer, + }).pipe(Effect.catch(() => Effect.succeed(undefined))); + if (repaired) { + return yield* resolveOpenApiBackedTools({ ...input, config: repaired }); + } + const operations = yield* input.storage.listOperations(String(input.integration.slug)); + return operations.length > 0 + ? stored + : { + tools: [], + definitions: {}, + incomplete: true, + incompleteReason: + "Provider catalog is missing. An integration administrator must refresh the spec.", + }; + }), invokeTool: ({ ctx: invokeCtx, toolRow, credential, args }) => { const httpClientLayer = options?.httpClientLayer ?? invokeCtx.httpClientLayer; diff --git a/packages/plugins/provider-service-split/src/catalog-repair.test.ts b/packages/plugins/provider-service-split/src/catalog-repair.test.ts new file mode 100644 index 0000000000..ff67c3a24a --- /dev/null +++ b/packages/plugins/provider-service-split/src/catalog-repair.test.ts @@ -0,0 +1,127 @@ +import { describe, expect, it } from "@effect/vitest"; +import { Effect } from "effect"; +import { collectTables, runSqliteDataMigrations } from "@executor-js/sdk"; +import { createSqliteTestFumaDb } from "@executor-js/sdk/testing"; + +import { + providerServiceCatalogRepairDataMigration, + runSqliteProviderServiceCatalogRepair, +} from "./catalog-repair"; + +describe("provider service catalog repair", () => { + it.effect("stale-marks only empty provider catalogs in the matching identity partition", () => + Effect.gen(function* () { + const { client } = yield* Effect.promise(() => + createSqliteTestFumaDb({ tables: collectTables() }), + ); + const rows = [ + { + id: "empty-google", + family: "google", + tenant: "one", + owner: "org", + subject: "", + tools: false, + }, + { + id: "empty-microsoft", + family: "microsoft", + tenant: "one", + owner: "org", + subject: "", + tools: false, + }, + { + id: "populated", + family: "google", + tenant: "one", + owner: "user", + subject: "alice", + tools: true, + }, + { + id: "other-subject", + family: "google", + tenant: "one", + owner: "user", + subject: "bob", + tools: false, + }, + { + id: "other-tenant", + family: "google", + tenant: "two", + owner: "user", + subject: "alice", + tools: false, + }, + { + id: "ordinary", + family: "custom", + tenant: "one", + owner: "org", + subject: "", + tools: false, + }, + ]; + for (const tenant of ["one", "two"]) { + for (const family of ["google", "microsoft", "custom"]) { + yield* Effect.promise(() => + client.execute({ + sql: `INSERT INTO integration + (row_id, tenant, slug, plugin_id, name, config, can_remove, can_refresh, created_at, updated_at) + VALUES (?, ?, ?, 'openapi', ?, ?, 1, 1, 1, 1)`, + args: [ + `${tenant}-${family}`, + tenant, + family, + family, + JSON.stringify({ family, specUrl: "https://example.test/spec" }), + ], + }), + ); + } + } + for (const row of rows) { + yield* Effect.promise(() => + client.execute({ + sql: `INSERT INTO connection + (row_id, tenant, owner, subject, integration, name, template, provider, item_ids, + tools_synced_at, created_at, updated_at) + VALUES (?, ?, ?, ?, ?, 'main', 'apiKey', 'vault', '{}', 42, 1, 1)`, + args: [row.id, row.tenant, row.owner, row.subject, row.family], + }), + ); + if (row.tools) { + yield* Effect.promise(() => + client.execute({ + sql: `INSERT INTO tool + (row_id, tenant, owner, subject, integration, connection, plugin_id, name, description, + input_schema, created_at, updated_at) + VALUES (?, ?, ?, ?, ?, 'main', 'openapi', 'list', 'List records', '{}', 1, 1)`, + args: [`tool-${row.id}`, row.tenant, row.owner, row.subject, row.family], + }), + ); + } + } + expect(yield* runSqliteProviderServiceCatalogRepair(client)).toBe(4); + const result = yield* Effect.promise(() => + client.execute("SELECT row_id, tools_synced_at FROM connection ORDER BY row_id"), + ); + expect(result.rows).toEqual([ + { row_id: "empty-google", tools_synced_at: null }, + { row_id: "empty-microsoft", tools_synced_at: null }, + { row_id: "ordinary", tools_synced_at: 42 }, + { row_id: "other-subject", tools_synced_at: null }, + { row_id: "other-tenant", tools_synced_at: null }, + { row_id: "populated", tools_synced_at: 42 }, + ]); + expect( + yield* runSqliteDataMigrations(client, [providerServiceCatalogRepairDataMigration]), + ).toEqual([providerServiceCatalogRepairDataMigration.name]); + expect( + yield* runSqliteDataMigrations(client, [providerServiceCatalogRepairDataMigration]), + ).toEqual([]); + }), + ); +}); diff --git a/packages/plugins/provider-service-split/src/catalog-repair.ts b/packages/plugins/provider-service-split/src/catalog-repair.ts new file mode 100644 index 0000000000..1a905b22f9 --- /dev/null +++ b/packages/plugins/provider-service-split/src/catalog-repair.ts @@ -0,0 +1,81 @@ +// --------------------------------------------------------------------------- +// Follow-up repair for provider-service splits whose connection rows were +// cloned as "synced" even though no per-service tool rows were persisted. +// +// The OpenAPI resolver now repairs a missing migrated Google/Microsoft catalog +// from its stored spec URL. Stale-mark only affected connections so their next +// normal tools read enters that resolver. Connections that already have tools, +// non-provider OpenAPI integrations, and integrations owned by other plugins +// are left untouched. +// --------------------------------------------------------------------------- + +import { Effect } from "effect"; +import { DataMigrationError, type SqliteDataMigrationClient } from "@executor-js/sdk"; + +const MIGRATION_NAME = "2026-07-10-provider-service-empty-catalog-repair"; + +const execute = ( + client: SqliteDataMigrationClient, + stmt: string | { readonly sql: string; readonly args: readonly unknown[] }, +) => + Effect.tryPromise({ + try: () => client.execute(stmt), + catch: (cause) => new DataMigrationError({ migration: MIGRATION_NAME, cause }), + }); + +const tableExists = (client: SqliteDataMigrationClient, table: string) => + Effect.map( + execute(client, { + sql: "SELECT name FROM sqlite_master WHERE type = 'table' AND name = ?", + args: [table], + }), + (result) => result.rows.length > 0, + ); + +export const runSqliteProviderServiceCatalogRepair = ( + client: SqliteDataMigrationClient, +): Effect.Effect => + Effect.gen(function* () { + for (const table of ["connection", "integration", "tool"]) { + if (!(yield* tableExists(client, table))) return 0; + } + + const affected = yield* execute(client, { + sql: `SELECT c.row_id AS row_id + FROM connection c + INNER JOIN integration i + ON i.tenant = c.tenant AND i.slug = c.integration + WHERE i.plugin_id = 'openapi' + AND i.config IS NOT NULL + AND json_valid(i.config) + AND json_extract(i.config, '$.family') IN ('google', 'microsoft') + AND json_extract(i.config, '$.specUrl') IS NOT NULL + AND NOT EXISTS ( + SELECT 1 + FROM tool t + WHERE t.tenant = c.tenant + AND t.owner = c.owner + AND t.subject = c.subject + AND t.integration = c.integration + AND t.connection = c.name + )`, + args: [], + }); + + const rowIds = affected.rows.flatMap((row) => + typeof row.row_id === "string" ? [row.row_id] : [], + ); + for (const rowId of rowIds) { + yield* execute(client, { + sql: "UPDATE connection SET tools_synced_at = NULL WHERE row_id = ?", + args: [rowId], + }); + } + return rowIds.length; + }); + +export const providerServiceCatalogRepairDataMigration = { + name: MIGRATION_NAME, + run: (client: SqliteDataMigrationClient) => + runSqliteProviderServiceCatalogRepair(client).pipe(Effect.asVoid), +}; diff --git a/packages/plugins/provider-service-split/src/index.ts b/packages/plugins/provider-service-split/src/index.ts index 26d727a3a6..56105782ea 100644 --- a/packages/plugins/provider-service-split/src/index.ts +++ b/packages/plugins/provider-service-split/src/index.ts @@ -1,2 +1,3 @@ export * from "./planner"; export * from "./sqlite"; +export * from "./catalog-repair"; diff --git a/packages/plugins/toolkits/src/page.tsx b/packages/plugins/toolkits/src/page.tsx index 5d1a9f2a3e..e2f0e9d726 100644 --- a/packages/plugins/toolkits/src/page.tsx +++ b/packages/plugins/toolkits/src/page.tsx @@ -12,13 +12,18 @@ import { } from "@executor-js/sdk/client"; import { matchPattern, + type Connection, type EffectivePolicy, type Integration, type Owner, type ToolAddress, type ToolPolicyAction, } from "@executor-js/sdk/shared"; -import { integrationsOptimisticAtom, toolsAllAtom } from "@executor-js/react/api/atoms"; +import { + connectionsAllAtom, + integrationsOptimisticAtom, + toolsAllAtom, +} from "@executor-js/react/api/atoms"; import { ReactivityKey } from "@executor-js/react/api/reactivity-keys"; import { useOrganizationSlug } from "@executor-js/react/api/organization-context"; import { @@ -159,6 +164,9 @@ const toolMatchId = (tool: ToolRow): string => const toolCanAppearInToolkit = (toolkit: ToolkitResponse, tool: ToolRow): boolean => toolkit.owner === "user" || tool.static === true || tool.owner !== "user"; +const connectionCanAppearInToolkit = (toolkit: ToolkitResponse, connection: Connection): boolean => + toolkit.owner === "user" || connection.owner !== "user"; + const toolkitUrlFor = (orgSlug: string | undefined, slug: string): string => { const path = orgSlug ? `/${orgSlug}/mcp/toolkits/${slug}` : `/mcp/toolkits/${slug}`; if (typeof window === "undefined") return path; @@ -260,8 +268,24 @@ const reducePolicyPatterns = (patterns: readonly string[]): readonly string[] => ); }; -const buildConnectionGroups = (tools: readonly ToolRow[]): readonly ToolkitConnectionGroup[] => { +const buildConnectionGroups = ( + tools: readonly ToolRow[], + connections: readonly Connection[] = [], +): readonly ToolkitConnectionGroup[] => { const groups = new Map(); + for (const connection of connections) { + const integration = String(connection.integration); + const name = String(connection.name); + const key = `${connection.owner}:${integration}:${name}`; + groups.set(key, { + id: key, + owner: connection.owner, + integration, + connection: name, + patterns: [`${integration}.${connection.owner}.${name}.*`], + tools: [], + }); + } for (const tool of tools) { const owner = toolOwner(tool); const key = `${owner}:${tool.integration}:${toolConnectionName(tool)}`; @@ -284,7 +308,10 @@ const buildConnectionGroups = (tools: readonly ToolRow[]): readonly ToolkitConne const sortedTools = [...group.tools].sort(compareTools); return { ...group, - patterns: reducePolicyPatterns(sortedTools.map(connectionPatternForTool)), + patterns: reducePolicyPatterns([ + ...group.patterns, + ...sortedTools.map(connectionPatternForTool), + ]), tools: sortedTools, }; }) @@ -1032,6 +1059,7 @@ function ToolkitWorkspace(props: { showOwnerLabels: boolean; policies: readonly ToolkitPolicyResponse[]; connections: readonly ToolkitConnectionResponse[]; + availableConnections: readonly Connection[]; tools: readonly ToolRow[]; integrations: readonly Integration[]; integrationPlugins: readonly IntegrationPlugin[]; @@ -1049,11 +1077,21 @@ function ToolkitWorkspace(props: { () => props.tools.filter((tool) => toolCanAppearInToolkit(props.toolkit, tool)), [props.toolkit, props.tools], ); - const connectionGroups = useMemo(() => buildConnectionGroups(visibleTools), [visibleTools]); + const visibleConnections = useMemo( + () => + props.availableConnections.filter((connection) => + connectionCanAppearInToolkit(props.toolkit, connection), + ), + [props.availableConnections, props.toolkit], + ); + const connectionGroups = useMemo( + () => buildConnectionGroups(visibleTools, visibleConnections), + [visibleConnections, visibleTools], + ); const hiddenPersonalConnectionCount = useMemo(() => { if (props.toolkit.owner !== "org") return 0; - return buildConnectionGroups(props.tools.filter((tool) => toolOwner(tool) === "user")).length; - }, [props.toolkit.owner, props.tools]); + return props.availableConnections.filter((connection) => connection.owner === "user").length; + }, [props.availableConnections, props.toolkit.owner]); const configuredConnections = useMemo( () => configuredConnectionViews( @@ -1283,6 +1321,7 @@ function ToolkitDetailSkeleton() { function ToolkitDetailView(props: { toolkit: ToolkitResponse; showOwnerLabels: boolean; + availableConnections: readonly Connection[]; tools: readonly ToolRow[]; integrations: readonly Integration[]; integrationPlugins: readonly IntegrationPlugin[]; @@ -1354,6 +1393,7 @@ function ToolkitDetailView(props: { showOwnerLabels={props.showOwnerLabels} policies={policyRows} connections={connectionRows} + availableConnections={props.availableConnections} tools={props.tools} integrations={props.integrations} integrationPlugins={props.integrationPlugins} @@ -1375,6 +1415,7 @@ export function ToolkitsPage(props: PluginPageProps) { const integrationPlugins = useIntegrationPlugins(); const toolkits = useAtomValue(toolkitsAtom); const tools = useAtomValue(toolsAllAtom); + const availableConnections = useAtomValue(connectionsAllAtom); const integrations = useAtomValue(integrationsOptimisticAtom); const doCreateToolkit = useAtomSet(createToolkit, { mode: "promiseExit" }); const doRemoveToolkit = useAtomSet(removeToolkit, { mode: "promiseExit" }); @@ -1385,6 +1426,9 @@ export function ToolkitsPage(props: PluginPageProps) { selectedToolkitSlug === null ? null : toolkitByRouteSlug(toolkitRows, selectedToolkitSlug); const toolRows = AsyncResult.isSuccess(tools) ? (tools.value as readonly ToolRow[]) : []; + const availableConnectionRows = AsyncResult.isSuccess(availableConnections) + ? availableConnections.value + : []; const integrationRows = AsyncResult.isSuccess(integrations) ? (integrations.value as readonly Integration[]) : []; @@ -1392,6 +1436,8 @@ export function ToolkitsPage(props: PluginPageProps) { const toolkitsFailed = AsyncResult.isFailure(toolkits); const toolsReady = AsyncResult.isSuccess(tools); const toolsFailed = AsyncResult.isFailure(tools); + const connectionsReady = AsyncResult.isSuccess(availableConnections); + const connectionsFailed = AsyncResult.isFailure(availableConnections); const navigateToIndex = () => navigate({ to: "/{-$orgSlug}/toolkits", @@ -1440,14 +1486,15 @@ export function ToolkitsPage(props: PluginPageProps) { ) ) : selectedToolkit ? ( - toolsFailed ? ( + toolsFailed || connectionsFailed ? (
Failed to load toolkit tools
- ) : !toolsReady ? ( + ) : !toolsReady || !connectionsReady ? ( ) : (