From 5882a4f3e30924218a0662f763e7251c1e425a54 Mon Sep 17 00:00:00 2001 From: Martin Helmich Date: Mon, 9 Sep 2024 13:12:00 +0200 Subject: [PATCH 01/14] Re-enable integration tests --- src/commands/conversation/show.test.ts | 29 +--------------------- src/commands/database/mysql/create.test.ts | 5 ++-- 2 files changed, 3 insertions(+), 31 deletions(-) diff --git a/src/commands/conversation/show.test.ts b/src/commands/conversation/show.test.ts index f292efbc1..10dbae38b 100644 --- a/src/commands/conversation/show.test.ts +++ b/src/commands/conversation/show.test.ts @@ -35,8 +35,7 @@ describe("conversation:show", () => { expect(true).toBeTruthy(); }); - // skipped, to be fixed later - it.skip("shows a conversation and its messages", async () => { + it("shows a conversation and its messages", async () => { const scope = nock("https://api.mittwald.de") .get(`/v2/conversations/${conversationId}`) .reply(200, { @@ -90,29 +89,3 @@ describe("conversation:show", () => { expect(error).toBeUndefined(); }); }); - -/* - - api - .env({ MITTWALD_API_TOKEN: "foo" }) - .stdout() - .command(["conversation show", conversationId]) - .it("shows a conversation and its messages", (ctx) => { - expect(ctx.stdout.trim()).to.equal(`Conversation metadata -───────────────────── - -Title Test conversation -ID CONV-ID -Opened less than a minute ago by Unknown User -Status open - -Messages -──────── - -CREATED, less than a minute ago - -John Doe, less than a minute ago -Hello, World! - -CLOSED, less than a minute ago`); - });*/ diff --git a/src/commands/database/mysql/create.test.ts b/src/commands/database/mysql/create.test.ts index 0f9d81283..6fa6da495 100644 --- a/src/commands/database/mysql/create.test.ts +++ b/src/commands/database/mysql/create.test.ts @@ -34,8 +34,7 @@ describe("database:mysql:create", () => { nock.cleanAll(); }); - // Skipped, to be fixed later - it.skip("creates a database and prints database and user name", async () => { + it("creates a database and prints database and user name", async () => { const scope = nock("https://api.mittwald.de"); scope.get(`/v2/projects/${projectId}`).reply(200, { @@ -86,7 +85,7 @@ describe("database:mysql:create", () => { }); // Skipped, to be fixed later - it.skip("retries fetching user until successful", async () => { + it("retries fetching user until successful", async () => { const scope = nock("https://api.mittwald.de"); scope.get(`/v2/projects/${projectId}`).reply(200, { From ce5e1721d8d900dde756456f794a45809998c09a Mon Sep 17 00:00:00 2001 From: Lars Bergmann Date: Mon, 3 Aug 2026 10:13:48 +0200 Subject: [PATCH 02/14] Integration test reactivation, engine and tools --- .gitignore | 6 + package.json | 4 +- src/commands/app/database/link.tsx | 2 +- src/commands/backup/download.tsx | 2 +- src/commands/conversation/show.test.ts | 105 +- .../integration/classification-catalog.ts | 227 ++++ src/test/integration/command-discovery.ts | 156 +++ .../integration/command-discovery/config.ts | 194 +++ .../integration/command-discovery/parsing.ts | 930 ++++++++++++++ .../command-discovery/synthesis.ts | 686 ++++++++++ .../integration/command-discovery/types.ts | 107 ++ src/test/integration/command.ts | 127 ++ .../config/command-classifications.json | 345 +++++ .../integration/config/command-waivers.json | 464 +++++++ .../config/invocation-profiles.json | 218 ++++ src/test/integration/config/loader.ts | 276 ++++ src/test/integration/env.ts | 32 + src/test/integration/run-all-commands.test.ts | 594 +++++++++ .../tools/generate-command-endpoint-map.ts | 1137 +++++++++++++++++ 19 files changed, 5537 insertions(+), 75 deletions(-) create mode 100644 src/test/integration/classification-catalog.ts create mode 100644 src/test/integration/command-discovery.ts create mode 100644 src/test/integration/command-discovery/config.ts create mode 100644 src/test/integration/command-discovery/parsing.ts create mode 100644 src/test/integration/command-discovery/synthesis.ts create mode 100644 src/test/integration/command-discovery/types.ts create mode 100644 src/test/integration/command.ts create mode 100644 src/test/integration/config/command-classifications.json create mode 100644 src/test/integration/config/command-waivers.json create mode 100644 src/test/integration/config/invocation-profiles.json create mode 100644 src/test/integration/config/loader.ts create mode 100644 src/test/integration/env.ts create mode 100644 src/test/integration/run-all-commands.test.ts create mode 100644 src/test/integration/tools/generate-command-endpoint-map.ts diff --git a/.gitignore b/.gitignore index 3ea8a77ec..a38b16f16 100644 --- a/.gitignore +++ b/.gitignore @@ -65,3 +65,9 @@ atlassian-ide-plugin.xml # Editor-based Rest Client .idea/httpRequests + +# integration testing artifacts +openapi.json +run-all-commands.ndjson +command-endpoint-map.json +command-endpoint-map.md diff --git a/package.json b/package.json index ad8dde446..f1eef1f36 100644 --- a/package.json +++ b/package.json @@ -38,7 +38,9 @@ "test:format": "yarn lint && yarn format:prettier --check", "test:licenses": "yarn license-check --summary --unknown --failOn 'UNLICENSED;UNKNOWN'", "test:readme": "yarn generate:readme && git diff --exit-code README.md docs/*.md", - "test:unit": "NODE_NO_WARNINGS=1 yarn node --experimental-vm-modules $(yarn bin jest) ./src" + "test:unit": "NODE_NO_WARNINGS=1 yarn node --experimental-vm-modules $(yarn bin jest) ./src", + "tool:integration:generate-command-endpoint-map": "yarn compile && node dist/test/integration/tools/generate-command-endpoint-map.js", + "tool:integration:generate-resource-precondition-map": "yarn compile && node dist/test/integration/tools/generate-command-endpoint-map.js --category RESOURCE_PRECONDITION" }, "files": [ ".deps", diff --git a/src/commands/app/database/link.tsx b/src/commands/app/database/link.tsx index 927c623cf..826eaeb36 100644 --- a/src/commands/app/database/link.tsx +++ b/src/commands/app/database/link.tsx @@ -71,7 +71,7 @@ export default class Link extends ExecRenderBaseCommand { }); await process.runStep("linking database", async () => { - const response = await this.apiClient.app.linkDatabase({ + const response = await this.apiClient.app.linkDatabase({ // XXX: deprecated?! Should use UPDATE on app installation instead! appInstallationId, data: { databaseId, diff --git a/src/commands/backup/download.tsx b/src/commands/backup/download.tsx index 3fd1e318a..b6e2ef750 100644 --- a/src/commands/backup/download.tsx +++ b/src/commands/backup/download.tsx @@ -135,7 +135,7 @@ export class Download extends ExecRenderBaseCommand { } return null; - }, Duration.fromString("1h")); + }, Duration.fromString("1h")); // XXX: may i have a word here, too?! }, ); diff --git a/src/commands/conversation/show.test.ts b/src/commands/conversation/show.test.ts index 10dbae38b..2f28d3d59 100644 --- a/src/commands/conversation/show.test.ts +++ b/src/commands/conversation/show.test.ts @@ -1,91 +1,52 @@ -import { runCommand } from "@oclif/test"; -import { MittwaldAPIV2 } from "@mittwald/api-client"; -import nock from "nock"; import { afterEach, beforeEach, describe, expect, it } from "@jest/globals"; - -type Conversation = MittwaldAPIV2.Components.Schemas.ConversationConversation; -type Message = MittwaldAPIV2.Components.Schemas.ConversationMessage; -type StatusUpdate = MittwaldAPIV2.Components.Schemas.ConversationStatusUpdate; +import { runDevCommand } from "../../test/integration/command.js"; +import { + configureIntegrationEnv, + restoreEnv, + snapshotEnv, +} from "../../test/integration/env.js"; + +function normalizeOutput(output: string): string { + return output + .replace(/\u001b\[[0-9;]*m/g, "") + .replace(/\r/g, "") + .trim(); +} describe("conversation:show", () => { - const conversationId = "186f8f22-aa0f-42bf-909d-757cb9d27b04"; - const userId = "6dbd84b5-74e0-43ed-8a81-b0b8a0405a47"; - const messageId = "10a59409-ff2d-478e-b07f-72c8f9f5b63f"; - const now = new Date(); - const user = { - userId, - clearName: "John Doe", - }; + const fallbackConversationId = "186f8f22-aa0f-42bf-909d-757cb9d27b04"; let originalEnv: NodeJS.ProcessEnv; beforeEach(() => { - originalEnv = { ...process.env }; - process.env["MITTWALD_API_TOKEN"] = "foo"; - - nock.disableNetConnect(); + originalEnv = snapshotEnv(); + configureIntegrationEnv("conversation:show"); }); afterEach(() => { - process.env = originalEnv; - nock.cleanAll(); - }); - - it("should test", () => { - expect(true).toBeTruthy(); + restoreEnv(originalEnv); }); it("shows a conversation and its messages", async () => { - const scope = nock("https://api.mittwald.de") - .get(`/v2/conversations/${conversationId}`) - .reply(200, { - conversationId, - shortId: "CONV-ID", - createdAt: now.toJSON(), - title: "Test conversation", - status: "open", - visibility: "shared", - mainUser: user, - } satisfies Conversation) - .get(`/v2/conversations/${conversationId}/messages`) - .reply(200, [ - { - conversationId, - type: "STATUS_UPDATE", - createdAt: now.toJSON(), - meta: { user }, - messageContent: "CONVERSATION_CREATED", - }, - { - messageId, - conversationId, - type: "MESSAGE", - createdAt: now.toJSON(), - createdBy: user, - messageContent: "Hello, World!", - }, - { - conversationId, - type: "STATUS_UPDATE", - createdAt: now.toJSON(), - meta: { user }, - messageContent: "STATUS_CLOSED", - }, - ] satisfies Array); + const conversationId = + process.env["MW_TEST_CONVERSATION_ID"] ?? fallbackConversationId; - console.log("foo"); + const { stdout, stderr, error, timedOut } = await runDevCommand( + ["conversation", "show", conversationId], + { + timeoutMs: 25_000, + }, + ); - const { stdout, stderr, error } = await runCommand([ - "conversation:show", - conversationId, - ]); + expect(timedOut).toBeFalsy(); - console.log("foo"); + expect(error).toBeUndefined(); - setTimeout(() => scope.done(), 5000); + const output = normalizeOutput(`${stdout}\n${stderr}`); - expect(stdout).toEqual(""); - expect(stderr).toEqual(""); - expect(error).toBeUndefined(); - }); + expect(output).toContain("Conversation metadata"); + expect(output).toContain("Messages"); + expect(output).toMatch(/ID\s+\S+/); + expect(output).toMatch(/Status\s+\S+/i); + }, 30_000); }); diff --git a/src/test/integration/classification-catalog.ts b/src/test/integration/classification-catalog.ts new file mode 100644 index 000000000..cebdcf031 --- /dev/null +++ b/src/test/integration/classification-catalog.ts @@ -0,0 +1,227 @@ +import { readFile, writeFile } from "node:fs/promises"; +import path from "node:path"; +import type { WaiverCategory } from "./command-discovery/types.js"; + +export type FailureCategory = WaiverCategory; + +export type ClassificationEntrySource = "failure" | "waiver" | "skip"; + +export type CommandClassificationEntry = { + commandId: string; + category: FailureCategory; + source: ClassificationEntrySource; +}; + +export type CommandClassificationCatalog = { + schemaVersion: 1; + generatedAt: string; + source: { + kind: "run-all-summary" | "log-extract"; + path?: string; + }; + statistics: { + successful: number; + failed: number; + waivedSkipped: number; + total: number; + }; + entries: CommandClassificationEntry[]; +}; + +export const FAILURE_CATEGORIES: FailureCategory[] = [ + "ARG_MISUSE", + "INTERACTIVE_REQUIRED", + "RESOURCE_PRECONDITION", + "CONTRACT_SHAPE", + "COMMAND_BUG", + "DEPRECATED_ENDPOINT", +]; + +export function isFailureCategory(value: string): value is FailureCategory { + return FAILURE_CATEGORIES.includes(value as FailureCategory); +} + +export function parseFailureCategory(value: string): FailureCategory { + if (!isFailureCategory(value)) { + throw new Error( + `Invalid category '${value}'. Allowed categories: ${FAILURE_CATEGORIES.join(", ")}`, + ); + } + + return value; +} + +export function createFailureBuckets(): Record { + return { + ARG_MISUSE: [], + INTERACTIVE_REQUIRED: [], + RESOURCE_PRECONDITION: [], + CONTRACT_SHAPE: [], + COMMAND_BUG: [], + DEPRECATED_ENDPOINT: [], + }; +} + +export function getDefaultClassificationCatalogPath(): string { + return path.resolve( + process.cwd(), + "src/test/integration/config/command-classifications.json", + ); +} + +export async function loadClassificationCatalog( + catalogPath = getDefaultClassificationCatalogPath(), +): Promise { + const raw = await readFile(catalogPath, "utf8"); + const parsed = JSON.parse(raw) as CommandClassificationCatalog; + return parsed; +} + +export async function saveClassificationCatalog( + catalog: CommandClassificationCatalog, + catalogPath = getDefaultClassificationCatalogPath(), +): Promise { + await writeFile(catalogPath, `${JSON.stringify(catalog, null, 2)}\n`, "utf8"); +} + +export function buildClassificationCatalogFromBuckets(input: { + failuresByCategory: Record; + waivedByCategory: Record; + statistics: { + successful: number; + failed: number; + waivedSkipped: number; + total: number; + }; + generatedAt?: string; +}): CommandClassificationCatalog { + const entryMap = new Map(); + + for (const category of FAILURE_CATEGORIES) { + for (const commandId of input.failuresByCategory[category]) { + entryMap.set(commandId, { + commandId, + category, + source: "failure", + }); + } + } + + for (const category of FAILURE_CATEGORIES) { + for (const commandId of input.waivedByCategory[category]) { + if (entryMap.has(commandId)) { + continue; + } + + entryMap.set(commandId, { + commandId, + category, + source: "waiver", + }); + } + } + + return { + schemaVersion: 1, + generatedAt: input.generatedAt ?? new Date().toISOString(), + source: { + kind: "run-all-summary", + }, + statistics: input.statistics, + entries: [...entryMap.values()].sort((a, b) => + a.commandId.localeCompare(b.commandId), + ), + }; +} + +export function extractClassificationCatalogFromRunLog(input: { + logContent: string; + logPath?: string; +}): CommandClassificationCatalog { + const entryMap = new Map(); + + const classifiedRegex = + /^\[(\d+)\/(\d+)\] classified (.+) as (ARG_MISUSE|INTERACTIVE_REQUIRED|RESOURCE_PRECONDITION|CONTRACT_SHAPE|COMMAND_BUG|DEPRECATED_ENDPOINT)$/m; + const waivedRegex = + /^\[(\d+)\/(\d+)\] waived (.+) \(category=(ARG_MISUSE|INTERACTIVE_REQUIRED|RESOURCE_PRECONDITION|CONTRACT_SHAPE|COMMAND_BUG|DEPRECATED_ENDPOINT)(?:;|\))/m; + const skippedInteractiveRegex = + /^\[(\d+)\/(\d+)\] skipped (.+) \(interactive required\)$/m; + + for (const line of input.logContent.split(/\r?\n/)) { + const classified = line.match(classifiedRegex); + if (classified) { + const commandId = classified[3].trim(); + const category = classified[4] as FailureCategory; + entryMap.set(commandId, { + commandId, + category, + source: "failure", + }); + continue; + } + + const waived = line.match(waivedRegex); + if (waived) { + const commandId = waived[3].trim(); + const category = waived[4] as FailureCategory; + entryMap.set(commandId, { + commandId, + category, + source: "waiver", + }); + continue; + } + + const skippedInteractive = line.match(skippedInteractiveRegex); + if (skippedInteractive) { + const commandId = skippedInteractive[3].trim(); + entryMap.set(commandId, { + commandId, + category: "INTERACTIVE_REQUIRED", + source: "skip", + }); + } + } + + const stats = parseStatistics(input.logContent); + + return { + schemaVersion: 1, + generatedAt: new Date().toISOString(), + source: { + kind: "log-extract", + path: input.logPath, + }, + statistics: stats, + entries: [...entryMap.values()].sort((a, b) => + a.commandId.localeCompare(b.commandId), + ), + }; +} + +function parseStatistics(logContent: string): { + successful: number; + failed: number; + waivedSkipped: number; + total: number; +} { + const statsRegex = + /\[run-all\] statistics: successful=(\d+), failed=(\d+), (?:waived-skipped|interactive-skipped)=(\d+), total=(\d+)/; + + const match = logContent.match(statsRegex); + if (!match) { + return { + successful: 0, + failed: 0, + waivedSkipped: 0, + total: 0, + }; + } + + return { + successful: Number.parseInt(match[1], 10), + failed: Number.parseInt(match[2], 10), + waivedSkipped: Number.parseInt(match[3], 10), + total: Number.parseInt(match[4], 10), + }; +} diff --git a/src/test/integration/command-discovery.ts b/src/test/integration/command-discovery.ts new file mode 100644 index 000000000..0501a39cb --- /dev/null +++ b/src/test/integration/command-discovery.ts @@ -0,0 +1,156 @@ +import { readdir, readFile } from "node:fs/promises"; +import path from "node:path"; +import { + type FailureCategory, + loadClassificationCatalog, +} from "./classification-catalog.js"; +import { + detectInteractiveSignals, + extractArgsSchema, + extractExampleCandidate, + extractFlagsSchema, +} from "./command-discovery/parsing.js"; +import { resolveProfiles, synthesizeInvocation } from "./command-discovery/synthesis.js"; +import type { DiscoveredCommand } from "./command-discovery/types.js"; + +export type { + DiscoveredCommand, + FlagValueType, + InteractiveSignal, + InvocationProfile, + ParsedArg, + ParsedFlag, + PlaceholderKind, + SynthesizedInvocation, + ValueSource, +} from "./command-discovery/types.js"; + +const COMMAND_FILE_EXTENSION_REGEX = /\.(ts|tsx)$/; +const NON_COMMAND_FILE_REGEX = /\.test\.(ts|tsx)$/; + +export type DiscoverCommandsOptions = { + commandsRoot?: string; + onProgress?: (message: string) => void; + categoryFilter?: FailureCategory; + classificationCatalogPath?: string; +}; + +export async function discoverRunnableCommands( + options: DiscoverCommandsOptions = {}, +): Promise { + const commandsRoot = + options.commandsRoot ?? path.resolve(process.cwd(), "src/commands"); + const onProgress = options.onProgress; + const categoryFilter = options.categoryFilter; + + const commandFiles = await collectCommandFiles(commandsRoot); + const discovered: DiscoveredCommand[] = []; + + onProgress?.( + `[discovery] found ${commandFiles.length} command source files under ${commandsRoot}`, + ); + + for (const [index, filePath] of commandFiles.entries()) { + const source = await readFile(filePath, "utf8"); + const relativePath = path.relative(commandsRoot, filePath); + const commandId = toCommandId(relativePath); + const commandTokens = commandId.split(" "); + const position = `${index + 1}/${commandFiles.length}`; + const extractionDiagnostics: string[] = []; + const profiles = resolveProfiles(commandId); + + onProgress?.(`[discovery:${position}] scanning ${commandId}`); + + const parsedArgs = extractArgsSchema(source, extractionDiagnostics); + const parsedFlags = extractFlagsSchema(source, extractionDiagnostics); + const interactiveSignals = detectInteractiveSignals(source); + const exampleCandidate = profiles.some((profile) => profile.disableExampleSource) + ? undefined + : extractExampleCandidate(source, commandId); + + const synthesizedInvocation = synthesizeInvocation({ + commandId, + commandTokens, + parsedArgs, + parsedFlags, + interactiveSignals, + exampleCandidate, + profiles, + }); + + discovered.push({ + commandId, + sourceFile: relativePath, + commandTokens, + parsedArgs, + parsedFlags, + interactiveSignals, + invocationProfilesApplied: profiles.map((profile) => profile.id), + extractionDiagnostics, + synthesizedInvocation, + }); + + onProgress?.( + `[discovery:${position}] ${commandId} -> ${synthesizedInvocation.argumentSource}${synthesizedInvocation.staleExample ? " (stale-example-fallback)" : ""}`, + ); + } + + const sorted = discovered.sort((a, b) => a.commandId.localeCompare(b.commandId)); + + if (!categoryFilter) { + onProgress?.(`[discovery] completed ${sorted.length} commands`); + return sorted; + } + + const classificationCatalog = await loadClassificationCatalog( + options.classificationCatalogPath, + ); + + const selectedCommandIds = new Set( + classificationCatalog.entries + .filter((entry) => entry.category === categoryFilter) + .map((entry) => entry.commandId), + ); + + const filtered = sorted.filter((command) => selectedCommandIds.has(command.commandId)); + + onProgress?.( + `[discovery] completed ${sorted.length} commands; category filter ${categoryFilter} => ${filtered.length}`, + ); + + return filtered; +} + +async function collectCommandFiles(rootDir: string): Promise { + const entries = await readdir(rootDir, { withFileTypes: true }); + const files = await Promise.all( + entries.map(async (entry) => { + const fullPath = path.join(rootDir, entry.name); + + if (entry.isDirectory()) { + return await collectCommandFiles(fullPath); + } + + if (!entry.isFile()) { + return []; + } + + if (!COMMAND_FILE_EXTENSION_REGEX.test(entry.name)) { + return []; + } + + if (NON_COMMAND_FILE_REGEX.test(entry.name)) { + return []; + } + + return [fullPath]; + }), + ); + + return files.flat(); +} + +function toCommandId(relativeFilePath: string): string { + const withoutExtension = relativeFilePath.replace(COMMAND_FILE_EXTENSION_REGEX, ""); + return withoutExtension.split(path.sep).join(" "); +} diff --git a/src/test/integration/command-discovery/config.ts b/src/test/integration/command-discovery/config.ts new file mode 100644 index 000000000..d8bc27935 --- /dev/null +++ b/src/test/integration/command-discovery/config.ts @@ -0,0 +1,194 @@ +import type { ParsedArg, ParsedFlag } from "./types.js"; + +export const DEFAULT_UUID = "00000000-0000-4000-8000-000000000000"; + +export const SHARED_FLAG_SCHEMAS: Record = { + processFlags: [ + { + name: "quiet", + required: false, + type: "boolean", + takesValue: false, + defaultValue: "false", + }, + ], + projectFlags: [ + { + name: "project-id", + required: false, + type: "string", + takesValue: true, + }, + ], + appInstallationFlags: [ + { + name: "installation-id", + required: false, + type: "string", + takesValue: true, + }, + ], + waitFlags: [ + { + name: "wait", + required: false, + type: "boolean", + takesValue: false, + }, + { + name: "wait-timeout", + required: false, + type: "string", + takesValue: true, + defaultValue: "10m", + }, + ], + ddevFlags: [ + { + name: "override-type", + required: false, + type: "string", + takesValue: true, + defaultValue: "auto", + options: ["auto"], + }, + { + name: "database-id", + required: false, + type: "string", + takesValue: true, + exclusive: ["without-database"], + }, + { + name: "without-database", + required: false, + type: "boolean", + takesValue: false, + exclusive: ["database-id"], + }, + ], + pathMappingFlags: [ + { + name: "path-to-app", + required: false, + type: "string", + takesValue: true, + multiple: true, + }, + { + name: "path-to-url", + required: false, + type: "string", + takesValue: true, + multiple: true, + }, + { + name: "path-to-container", + required: false, + type: "string", + takesValue: true, + multiple: true, + }, + ], +}; + +export const SHARED_ARG_SCHEMAS: Record = { + appInstallationArgs: [ + { + name: "installation-id", + required: true, + placeholderKind: "uuid", + }, + ], + backupArgs: [ + { + name: "backup-id", + required: true, + placeholderKind: "uuid", + }, + ], + mysqlArgs: [ + { + name: "database-id", + required: true, + placeholderKind: "uuid", + }, + ], + redisArgs: [ + { + name: "database-id", + required: true, + placeholderKind: "uuid", + }, + ], + dnsZoneArgs: [ + { + name: "dnszone-id", + required: true, + placeholderKind: "generic", + }, + ], + conversationArgs: [ + { + name: "conversation-id", + required: true, + placeholderKind: "uuid", + }, + ], + orgArgs: [ + { + name: "org-id", + required: true, + placeholderKind: "uuid", + }, + ], + domainArgs: [ + { + name: "domain-id", + required: true, + placeholderKind: "generic", + }, + ], + mailAddressArgs: [ + { + name: "mailaddress-id", + required: true, + placeholderKind: "generic", + }, + ], + mailDeliveryBoxArgs: [ + { + name: "maildeliverybox-id", + required: true, + placeholderKind: "uuid", + }, + ], + stackArgs: [ + { + name: "stack-id", + required: true, + placeholderKind: "uuid", + }, + ], +}; + +export const NAMED_FLAG_SCHEMAS: Record> = { + adminUserIdFlag: { + required: true, + type: "string", + takesValue: true, + }, + databasePurposeFlag: { + required: true, + type: "string", + takesValue: true, + options: ["primary", "cache", "custom"], + defaultValue: "primary", + }, + databasePurposeSelectorFlag: { + required: false, + type: "string", + takesValue: true, + options: ["primary", "cache", "custom"], + }, +}; diff --git a/src/test/integration/command-discovery/parsing.ts b/src/test/integration/command-discovery/parsing.ts new file mode 100644 index 000000000..6c9153ae0 --- /dev/null +++ b/src/test/integration/command-discovery/parsing.ts @@ -0,0 +1,930 @@ +import { + NAMED_FLAG_SCHEMAS, + SHARED_ARG_SCHEMAS, + SHARED_FLAG_SCHEMAS, +} from "./config.js"; +import type { + ExampleCandidate, + FlagValueType, + InteractiveSignal, + ParsedArg, + ParsedFlag, + PlaceholderKind, +} from "./types.js"; + +export function extractExampleCandidate( + source: string, + commandId: string, +): ExampleCandidate | undefined { + const examplesMatch = source.match(/static\s+examples\s*=\s*\[([\s\S]*?)\];/m); + if (!examplesMatch) { + return undefined; + } + + const block = examplesMatch[1]; + const commandStrings: string[] = []; + + const objectCommandRegex = + /command\s*:\s*("(?:[^"\\]|\\.)*"|'(?:[^'\\]|\\.)*'|`[\s\S]*?`)/g; + + let objectMatch = objectCommandRegex.exec(block); + while (objectMatch) { + const decoded = decodeStringLiteral(objectMatch[1]); + if (decoded) { + commandStrings.push(decoded); + } + + objectMatch = objectCommandRegex.exec(block); + } + + if (commandStrings.length === 0) { + const stringLiteralRegex = /("(?:[^"\\]|\\.)*"|'(?:[^'\\]|\\.)*'|`[\s\S]*?`)/g; + let stringMatch = stringLiteralRegex.exec(block); + while (stringMatch) { + const decoded = decodeStringLiteral(stringMatch[1]); + if (decoded && (decoded.includes("<%= command.id %>") || decoded.includes("mw "))) { + commandStrings.push(decoded); + } + + stringMatch = stringLiteralRegex.exec(block); + } + } + + for (const commandString of commandStrings) { + const args = parseExampleCommandToArgs(commandString, commandId); + if (!args) { + continue; + } + + const { positionalValues, flagValues } = parseInvocationParts(args.slice(commandId.split(" ").length)); + return { + args, + positionalValues, + flagValues, + }; + } + + return undefined; +} + +export function extractArgsSchema(source: string, diagnostics: string[]): ParsedArg[] { + const block = extractStaticObjectBlock(source, /static\s+args\s*=\s*{/m); + if (!block) { + return []; + } + + const entries = splitTopLevelEntries(block); + const args = new Map(); + + for (const entry of entries) { + const spread = entry.match(/^\.\.\.\s*([A-Za-z0-9_$.]+)\s*$/); + if (spread) { + const spreadName = spread[1].split(".").at(-1) ?? spread[1]; + const sharedArgs = SHARED_ARG_SCHEMAS[spreadName]; + if (sharedArgs) { + for (const sharedArg of sharedArgs) { + args.set(sharedArg.name, sharedArg); + } + continue; + } + + const localArgs = parseLocalArgObject(source, spreadName, diagnostics); + if (localArgs.length > 0) { + for (const localArg of localArgs) { + args.set(localArg.name, localArg); + } + continue; + } + + diagnostics.push(`args: unresolved spread '${spread[1]}'`); + continue; + } + + const split = splitObjectEntry(entry); + if (!split) { + continue; + } + + const config = extractFirstObjectLiteral(split.expression); + const required = readBooleanProp(config, "required") ?? false; + const defaultValue = readStringProp(config, "default"); + + args.set(split.key, { + name: split.key, + required, + defaultValue, + placeholderKind: inferPlaceholderKind(split.key), + }); + } + + if (args.size === 0) { + diagnostics.push("args: no statically extractable arg entries"); + } + + return [...args.values()]; +} + +export function extractFlagsSchema(source: string, diagnostics: string[]): ParsedFlag[] { + const block = extractStaticObjectBlock(source, /static\s+flags\s*=\s*{/m); + if (!block) { + diagnostics.push("flags: static flags block not found"); + return []; + } + + const entries = splitTopLevelEntries(block); + const flags = new Map(); + + for (const entry of entries) { + const factorySpreadFlags = parseFlagSpreadFactory(entry); + if (factorySpreadFlags.length > 0) { + for (const flag of factorySpreadFlags) { + flags.set(flag.name, flag); + } + continue; + } + + const spread = entry.match(/^\.\.\.\s*([A-Za-z0-9_$.]+)\s*$/); + if (spread) { + const spreadName = spread[1].split(".").at(-1) ?? spread[1]; + const shared = SHARED_FLAG_SCHEMAS[spreadName]; + if (shared) { + for (const flag of shared) { + flags.set(flag.name, flag); + } + continue; + } + + const localFlags = parseLocalFlagObject(source, spreadName, diagnostics); + if (localFlags.length > 0) { + for (const localFlag of localFlags) { + flags.set(localFlag.name, localFlag); + } + continue; + } + + diagnostics.push(`flags: unresolved spread '${spread[1]}'`); + continue; + } + + const split = splitObjectEntry(entry); + if (!split) { + diagnostics.push(`flags: could not parse entry '${entry.trim().slice(0, 80)}'`); + continue; + } + + const parsedFlag = parseFlagDefinition(split.key, split.expression); + if (!parsedFlag) { + diagnostics.push(`flags: unresolved factory for '${split.key}'`); + continue; + } + + flags.set(parsedFlag.name, parsedFlag); + } + + return [...flags.values()]; +} + +export function detectInteractiveSignals(source: string): InteractiveSignal[] { + const signals: InteractiveSignal[] = []; + const withSignal = (signal: InteractiveSignal, regex: RegExp) => { + if (regex.test(source)) { + signals.push(signal); + } + }; + + withSignal("addInput", /\.addInput\s*\(/); + withSignal("addSelect", /\.addSelect\s*\(/); + withSignal("addConfirmation", /\.addConfirmation\s*\(/); + withSignal("editorFallback", /editor|openEditor/i); + withSignal("stdinBranch", /stdin|process\.stdin/i); + + return [...new Set(signals)]; +} + +function parseFlagSpreadFactory(entry: string): ParsedFlag[] { + const expireFlagsMatch = entry.match( + /^\.\.\.\s*expireFlags\(\s*[^,]+,\s*(true|false)\s*\)\s*$/, + ); + + if (expireFlagsMatch) { + return [ + { + name: "expires", + required: expireFlagsMatch[1] === "true", + type: "string", + takesValue: true, + }, + ]; + } + + return []; +} + +function parseLocalArgObject( + source: string, + objectName: string, + diagnostics: string[], +): ParsedArg[] { + const block = extractConstObjectBlock(source, objectName); + if (!block) { + return []; + } + + const entries = splitTopLevelEntries(block); + const args = new Map(); + + for (const entry of entries) { + const spread = entry.match(/^\.\.\.\s*([A-Za-z0-9_$.]+)\s*$/); + if (spread) { + const spreadName = spread[1].split(".").at(-1) ?? spread[1]; + const shared = SHARED_ARG_SCHEMAS[spreadName]; + if (shared) { + for (const sharedArg of shared) { + args.set(sharedArg.name, sharedArg); + } + } + continue; + } + + const split = splitObjectEntry(entry); + if (!split) { + continue; + } + + const config = extractFirstObjectLiteral(split.expression); + const required = readBooleanProp(config, "required") ?? false; + const defaultValue = readStringProp(config, "default"); + + args.set(split.key, { + name: split.key, + required, + defaultValue, + placeholderKind: inferPlaceholderKind(split.key), + }); + } + + if (args.size === 0) { + diagnostics.push(`args: local spread '${objectName}' contained no extractable args`); + } + + return [...args.values()]; +} + +function parseLocalFlagObject( + source: string, + objectName: string, + diagnostics: string[], +): ParsedFlag[] { + const block = extractConstObjectBlock(source, objectName); + if (!block) { + return []; + } + + const entries = splitTopLevelEntries(block); + const flags = new Map(); + + for (const entry of entries) { + const spread = entry.match(/^\.\.\.\s*([A-Za-z0-9_$.]+)\s*$/); + if (spread) { + continue; + } + + const split = splitObjectEntry(entry); + if (!split) { + continue; + } + + const parsed = parseFlagDefinition(split.key, split.expression); + if (parsed) { + flags.set(parsed.name, parsed); + } + } + + if (flags.size === 0) { + diagnostics.push(`flags: local spread '${objectName}' contained no extractable flags`); + } + + return [...flags.values()]; +} + +function parseExampleCommandToArgs( + example: string, + commandId: string, +): string[] | undefined { + const rendered = example + .replace(/<%=\s*config\.bin\s*%>/g, "mw") + .replace(/<%=\s*command\.id\s*%>/g, commandId); + + const commandLine = rendered + .split("\n") + .map((line) => line.trim()) + .filter((line) => line.length > 0 && !line.startsWith("#")) + .find((line) => line.includes(commandId) || line.startsWith("mw ")); + + if (!commandLine) { + return undefined; + } + + const tokens = shellTokenize(commandLine.replace(/^\$\s*/, "")); + const normalizedTokens = tokens.filter((token) => token !== "mw"); + const commandTokens = commandId.split(" "); + const commandStart = findTokenSequenceIndex(normalizedTokens, commandTokens); + + if (commandStart === -1) { + return undefined; + } + + const rawInvocation = normalizedTokens.slice(commandStart); + return rawInvocation.map((token) => { + if (token.startsWith("<") && token.endsWith(">")) { + return makeTypedPlaceholderValue(token.slice(1, -1), "string", undefined); + } + + return token; + }); +} + +function findTokenSequenceIndex(haystack: string[], needle: string[]): number { + if (needle.length === 0 || haystack.length < needle.length) { + return -1; + } + + for (let i = 0; i <= haystack.length - needle.length; i += 1) { + const segment = haystack.slice(i, i + needle.length); + if (segment.every((token, idx) => token === needle[idx])) { + return i; + } + } + + return -1; +} + +function decodeStringLiteral(value: string): string | undefined { + const quote = value[0]; + if ((quote !== '"' && quote !== "'" && quote !== "`") || value.length < 2) { + return undefined; + } + + const inner = value.slice(1, -1); + return inner + .replace(/\\n/g, "\n") + .replace(/\\t/g, "\t") + .replace(/\\"/g, '"') + .replace(/\\'/g, "'") + .replace(/\\`/g, "`") + .replace(/\\\\/g, "\\"); +} + +function shellTokenize(value: string): string[] { + const matches = value.match(/"([^"\\]|\\.)*"|'([^'\\]|\\.)*'|\S+/g); + if (!matches) { + return []; + } + + return matches.map((token) => { + if ( + (token.startsWith('"') && token.endsWith('"')) || + (token.startsWith("'") && token.endsWith("'")) + ) { + return token.slice(1, -1); + } + + return token; + }); +} + +function makeTypedPlaceholderValue( + name: string, + type: FlagValueType, + options: string[] | undefined, +): string { + if (options && options.length > 0) { + return options[0]; + } + + const normalized = name + .replace(/[<>[\]]/g, "") + .replace(/[^A-Za-z0-9-]+/g, "-") + .replace(/^-+/, "") + .replace(/-+$/, "") + .toLowerCase(); + + if (normalized.includes("uuid") || normalized.endsWith("id") || normalized.includes("-id")) { + return "00000000-0000-4000-8000-000000000000"; + } + + if (normalized.includes("email")) { + return "integration@example.com"; + } + + if (normalized.includes("url") || normalized.includes("uri")) { + return "https://example.com"; + } + + if ( + normalized.includes("duration") || + normalized.includes("ttl") || + normalized.includes("interval") + ) { + return "1h"; + } + + if (normalized.includes("directory") || normalized.includes("path")) { + return "/tmp/mw-integration"; + } + + if (type === "file") { + return "/tmp/mw-integration.file"; + } + + if (type === "directory") { + return "/tmp/mw-integration"; + } + + if (normalized.includes("password")) { + return "integration-password"; + } + + if (normalized.includes("port")) { + return "12345"; + } + + return normalized.length > 0 ? `example-${normalized}` : "example-value"; +} + +function extractStaticObjectBlock(source: string, anchor: RegExp): string | undefined { + const match = anchor.exec(source); + if (!match) { + return undefined; + } + + const start = source.indexOf("{", match.index); + if (start === -1) { + return undefined; + } + + const end = findMatchingBraceIndex(source, start); + if (end === -1) { + return undefined; + } + + return source.slice(start + 1, end); +} + +function extractConstObjectBlock(source: string, objectName: string): string | undefined { + const anchor = new RegExp(`(?:const|let|var)\\s+${escapeRegExp(objectName)}\\s*=\\s*{`, "m"); + const match = anchor.exec(source); + if (!match) { + return undefined; + } + + const start = source.indexOf("{", match.index); + if (start === -1) { + return undefined; + } + + const end = findMatchingBraceIndex(source, start); + if (end === -1) { + return undefined; + } + + return source.slice(start + 1, end); +} + +function findMatchingBraceIndex(input: string, startIndex: number): number { + let depth = 0; + let quote: "'" | '"' | "`" | undefined; + let escaped = false; + + for (let i = startIndex; i < input.length; i += 1) { + const char = input[i]; + + if (quote) { + if (escaped) { + escaped = false; + continue; + } + + if (char === "\\") { + escaped = true; + continue; + } + + if (char === quote) { + quote = undefined; + } + + continue; + } + + if (char === '"' || char === "'" || char === "`") { + quote = char; + continue; + } + + if (char === "{") { + depth += 1; + continue; + } + + if (char === "}") { + depth -= 1; + if (depth === 0) { + return i; + } + } + } + + return -1; +} + +function splitTopLevelEntries(input: string): string[] { + const entries: string[] = []; + let start = 0; + let braceDepth = 0; + let parenDepth = 0; + let bracketDepth = 0; + let quote: "'" | '"' | "`" | undefined; + let escaped = false; + + for (let i = 0; i < input.length; i += 1) { + const char = input[i]; + + if (quote) { + if (escaped) { + escaped = false; + continue; + } + + if (char === "\\") { + escaped = true; + continue; + } + + if (char === quote) { + quote = undefined; + } + + continue; + } + + if (char === '"' || char === "'" || char === "`") { + quote = char; + continue; + } + + if (char === "{") { + braceDepth += 1; + continue; + } + + if (char === "}") { + braceDepth -= 1; + continue; + } + + if (char === "(") { + parenDepth += 1; + continue; + } + + if (char === ")") { + parenDepth -= 1; + continue; + } + + if (char === "[") { + bracketDepth += 1; + continue; + } + + if (char === "]") { + bracketDepth -= 1; + continue; + } + + if (char === "," && braceDepth === 0 && parenDepth === 0 && bracketDepth === 0) { + const part = input.slice(start, i).trim(); + if (part.length > 0) { + entries.push(part); + } + start = i + 1; + } + } + + const tail = input.slice(start).trim(); + if (tail.length > 0) { + entries.push(tail); + } + + return entries; +} + +function splitObjectEntry(entry: string): { key: string; expression: string } | undefined { + let quote: "'" | '"' | "`" | undefined; + let escaped = false; + let braceDepth = 0; + let parenDepth = 0; + let bracketDepth = 0; + + for (let i = 0; i < entry.length; i += 1) { + const char = entry[i]; + + if (quote) { + if (escaped) { + escaped = false; + continue; + } + + if (char === "\\") { + escaped = true; + continue; + } + + if (char === quote) { + quote = undefined; + } + + continue; + } + + if (char === '"' || char === "'" || char === "`") { + quote = char; + continue; + } + + if (char === "{") { + braceDepth += 1; + continue; + } + + if (char === "}") { + braceDepth -= 1; + continue; + } + + if (char === "(") { + parenDepth += 1; + continue; + } + + if (char === ")") { + parenDepth -= 1; + continue; + } + + if (char === "[") { + bracketDepth += 1; + continue; + } + + if (char === "]") { + bracketDepth -= 1; + continue; + } + + if (char === ":" && braceDepth === 0 && parenDepth === 0 && bracketDepth === 0) { + const keyRaw = entry.slice(0, i).trim(); + const expression = entry.slice(i + 1).trim(); + const key = keyRaw.replace(/^['"]/, "").replace(/['"]$/, ""); + if (!key || !expression) { + return undefined; + } + + return { key, expression }; + } + } + + return undefined; +} + +function parseFlagDefinition(name: string, expression: string): ParsedFlag | undefined { + const named = resolveNamedFlagSchemaFromExpression(expression); + if (named) { + return { + name, + ...named, + }; + } + + const type = detectFlagType(expression); + if (!type) { + return undefined; + } + + const config = extractFirstObjectLiteral(expression); + const required = readBooleanProp(config, "required") ?? false; + const multiple = readBooleanProp(config, "multiple") ?? false; + const options = readStringArrayProp(config, "options"); + const exactlyOne = readStringArrayProp(config, "exactlyOne"); + const exclusive = readStringArrayProp(config, "exclusive"); + const dependsOn = readStringArrayProp(config, "dependsOn"); + const defaultValue = readLiteralStringProp(config, "default"); + + return { + name, + required, + type, + takesValue: type !== "boolean", + multiple, + options, + defaultValue, + exactlyOne, + exclusive, + dependsOn, + }; +} + +function detectFlagType(expression: string): FlagValueType | undefined { + if (/Flags\.boolean\s*\(/.test(expression)) { + return "boolean"; + } + + if (/Flags\.integer\s*\(/.test(expression)) { + return "integer"; + } + + if (/Flags\.file\s*\(/.test(expression)) { + return "file"; + } + + if (/Flags\.directory\s*\(/.test(expression)) { + return "directory"; + } + + if (/Flags\.url\s*\(/.test(expression)) { + return "url"; + } + + if (/Flags\.(string|custom)\s*\(/.test(expression)) { + return "string"; + } + + if (/\.absoluteFlag\s*\(/.test(expression) || /\.relativeFlag\s*\(/.test(expression)) { + return "string"; + } + + // Fallback for wrapped/custom flag factories, e.g. `flagDefinitions.name({ required: true })`. + if (/^[A-Za-z0-9_.$\[\]"'-]+\s*\(/.test(expression)) { + return "string"; + } + + return undefined; +} + +function resolveNamedFlagSchemaFromExpression( + expression: string, +): Omit | undefined { + const normalized = expression.trim().replace(/\(\s*\)$/, ""); + const candidate = normalized.split(".").at(-1) ?? normalized; + return NAMED_FLAG_SCHEMAS[candidate]; +} + +function extractFirstObjectLiteral(expression: string): string { + const start = expression.indexOf("{"); + if (start === -1) { + return ""; + } + + const end = findMatchingBraceIndex(expression, start); + if (end === -1) { + return ""; + } + + return expression.slice(start, end + 1); +} + +function readBooleanProp(config: string, key: string): boolean | undefined { + if (!config) { + return undefined; + } + + const regex = new RegExp(`${escapeRegExp(key)}\\s*:\\s*(true|false)`); + const match = config.match(regex); + if (!match) { + return undefined; + } + + return match[1] === "true"; +} + +function readStringProp(config: string, key: string): string | undefined { + if (!config) { + return undefined; + } + + const regex = new RegExp(`${escapeRegExp(key)}\\s*:\\s*(["'])(.*?)\\1`, "s"); + const match = config.match(regex); + return match?.[2]; +} + +function readLiteralStringProp(config: string, key: string): string | undefined { + const stringValue = readStringProp(config, key); + if (stringValue !== undefined) { + return stringValue; + } + + const boolMatch = config.match(new RegExp(`${escapeRegExp(key)}\\s*:\\s*(true|false)`)); + if (boolMatch) { + return boolMatch[1]; + } + + const numberMatch = config.match(new RegExp(`${escapeRegExp(key)}\\s*:\\s*([0-9]+(?:\\.[0-9]+)?)`)); + if (numberMatch) { + return numberMatch[1]; + } + + return undefined; +} + +function readStringArrayProp(config: string, key: string): string[] | undefined { + if (!config) { + return undefined; + } + + const regex = new RegExp(`${escapeRegExp(key)}\\s*:\\s*\\[([^\\]]*)\\]`, "s"); + const match = config.match(regex); + if (!match) { + return undefined; + } + + return match[1] + .split(",") + .map((entry) => entry.trim().replace(/^['"]/, "").replace(/['"]$/, "")) + .filter((entry) => entry.length > 0); +} + +function inferPlaceholderKind(name: string): PlaceholderKind { + const normalized = name.toLowerCase(); + if (normalized.includes("uuid") || normalized.endsWith("id") || normalized.includes("-id")) { + return "uuid"; + } + if (normalized.includes("email")) { + return "email"; + } + if (normalized.includes("url") || normalized.includes("uri")) { + return "url"; + } + if (normalized.includes("duration") || normalized.includes("ttl") || normalized.includes("interval")) { + return "duration"; + } + if (normalized.includes("directory")) { + return "directory"; + } + if (normalized.includes("file")) { + return "file"; + } + if (normalized.includes("password") || normalized.includes("passphrase") || normalized.includes("token")) { + return "password"; + } + if (normalized.includes("port")) { + return "port"; + } + return "generic"; +} + +function parseInvocationParts(args: string[]): { + positionalValues: string[]; + flagValues: Map; +} { + const positionalValues: string[] = []; + const flagValues = new Map(); + + for (let i = 0; i < args.length; i += 1) { + const token = args[i]; + if (!token.startsWith("--")) { + positionalValues.push(token); + continue; + } + + const withoutPrefix = token.slice(2); + const eqIndex = withoutPrefix.indexOf("="); + let name = withoutPrefix; + let value: string | undefined; + + if (eqIndex >= 0) { + name = withoutPrefix.slice(0, eqIndex); + value = withoutPrefix.slice(eqIndex + 1); + } else { + const nextToken = args[i + 1]; + if (nextToken && !nextToken.startsWith("--")) { + value = nextToken; + i += 1; + } + } + + const values = flagValues.get(name) ?? []; + if (value === undefined) { + values.push("true"); + } else { + values.push(value); + } + flagValues.set(name, values); + } + + return { positionalValues, flagValues }; +} + +function escapeRegExp(value: string): string { + return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); +} diff --git a/src/test/integration/command-discovery/synthesis.ts b/src/test/integration/command-discovery/synthesis.ts new file mode 100644 index 000000000..879d2b0a3 --- /dev/null +++ b/src/test/integration/command-discovery/synthesis.ts @@ -0,0 +1,686 @@ +import { loadInvocationProfiles } from "../config/loader.js"; +import { DEFAULT_UUID } from "./config.js"; +import type { + ExampleCandidate, + InteractiveSignal, + InvocationProfile, + ParsedArg, + ParsedFlag, + PlaceholderKind, + ResolvedFlagValue, + SynthesizedInvocation, + ValueSource, +} from "./types.js"; + +export function resolveProfiles(commandId: string): InvocationProfile[] { + const invocationProfiles = loadInvocationProfiles(); + return invocationProfiles.filter((profile) => { + if (profile.match.exact && profile.match.exact === commandId) { + return true; + } + + if (profile.match.prefix && commandId.startsWith(`${profile.match.prefix} `)) { + return true; + } + + return false; + }); +} + +export function synthesizeInvocation(input: { + commandId: string; + commandTokens: string[]; + parsedArgs: ParsedArg[]; + parsedFlags: ParsedFlag[]; + interactiveSignals: InteractiveSignal[]; + exampleCandidate: ExampleCandidate | undefined; + profiles: InvocationProfile[]; +}): SynthesizedInvocation { + const { + commandId, + commandTokens, + parsedArgs, + parsedFlags, + interactiveSignals, + exampleCandidate, + profiles, + } = input; + + const staleExampleReasons: string[] = []; + let validatedExample: ExampleCandidate | undefined; + if (exampleCandidate) { + const validationErrors = validateExampleCandidate(exampleCandidate, parsedArgs, parsedFlags); + if (validationErrors.length === 0) { + validatedExample = exampleCandidate; + } else { + staleExampleReasons.push(...validationErrors.map((reason) => `stale-example: ${reason}`)); + } + } + + const selectedFlags = new Map(); + let strongestSource: ValueSource = "heuristic"; + + const positionalValues = parsedArgs.map((arg, index) => { + const profileValue = getProfileArgValue(profiles, arg.name); + if (profileValue !== undefined) { + strongestSource = selectStrongerSource(strongestSource, "profile"); + return profileValue; + } + + const exampleValue = validatedExample?.positionalValues[index]; + if (exampleValue !== undefined) { + strongestSource = selectStrongerSource(strongestSource, "example"); + return exampleValue; + } + + if (arg.defaultValue !== undefined) { + return arg.defaultValue; + } + + return defaultValueForPlaceholderKind(arg.placeholderKind, arg.name); + }); + + for (const flag of parsedFlags) { + if (!flag.required) { + continue; + } + + const fromProfile = getProfileFlagValue(profiles, flag.name); + if (fromProfile !== undefined) { + setFlagValue(selectedFlags, flag.name, normalizeFlagValue(fromProfile), "profile"); + strongestSource = selectStrongerSource(strongestSource, "profile"); + continue; + } + + const fromExample = validatedExample?.flagValues.get(flag.name); + if (fromExample && fromExample.length > 0) { + setFlagValue(selectedFlags, flag.name, fromExample, "example"); + strongestSource = selectStrongerSource(strongestSource, "example"); + continue; + } + + const heuristic = buildHeuristicFlagValue(flag); + setFlagValue(selectedFlags, flag.name, heuristic, "heuristic"); + } + + resolveExactlyOneGroups(commandId, parsedFlags, selectedFlags, profiles, interactiveSignals); + resolveDependencies(parsedFlags, selectedFlags, profiles); + resolveExclusiveGroups(parsedFlags, selectedFlags); + applyProfileOverrides(parsedFlags, selectedFlags, profiles); + const interactiveDecision = decideInteractivePolicy( + commandId, + parsedFlags, + selectedFlags, + interactiveSignals, + profiles, + ); + + const invocation = renderInvocation(commandTokens, positionalValues, parsedFlags, selectedFlags); + return { + args: invocation, + argumentSource: strongestSource, + interactiveDecision, + staleExample: staleExampleReasons.length > 0, + staleExampleReasons, + }; +} + +function validateExampleCandidate( + example: ExampleCandidate, + argsSchema: ParsedArg[], + flagSchema: ParsedFlag[], +): string[] { + const errors: string[] = []; + const flagNames = new Set(flagSchema.map((flag) => flag.name)); + + for (const flagName of example.flagValues.keys()) { + if (!flagNames.has(flagName)) { + errors.push(`unknown flag --${flagName}`); + } + } + + const requiredArgsCount = argsSchema.filter((arg) => arg.required).length; + if (example.positionalValues.length < requiredArgsCount) { + errors.push("missing required positional arguments"); + } + + for (const flag of flagSchema) { + if (flag.required && !example.flagValues.has(flag.name)) { + errors.push(`missing required flag --${flag.name}`); + } + + if (flag.dependsOn && example.flagValues.has(flag.name)) { + for (const dependency of flag.dependsOn) { + if (!example.flagValues.has(dependency)) { + errors.push(`--${flag.name} depends on --${dependency}`); + } + } + } + + if (flag.exclusive) { + for (const conflicting of flag.exclusive) { + if (example.flagValues.has(flag.name) && example.flagValues.has(conflicting)) { + errors.push(`--${flag.name} is exclusive with --${conflicting}`); + } + } + } + } + + for (const group of collectExactlyOneGroups(flagSchema)) { + const count = group.members.filter((name) => example.flagValues.has(name)).length; + if (count !== 1) { + errors.push(`exactly one of [${group.members.join(", ")}] must be set`); + } + } + + return errors; +} + +function collectExactlyOneGroups(flagSchema: ParsedFlag[]): Array<{ key: string; members: string[] }> { + const groups = new Map(); + + for (const flag of flagSchema) { + if (!flag.exactlyOne || flag.exactlyOne.length < 2) { + continue; + } + + const members = [...new Set(flag.exactlyOne)].sort(); + const key = members.join("|"); + groups.set(key, members); + } + + return [...groups.entries()].map(([key, members]) => ({ key, members })); +} + +function resolveExactlyOneGroups( + commandId: string, + flagSchema: ParsedFlag[], + selectedFlags: Map, + profiles: InvocationProfile[], + interactiveSignals: InteractiveSignal[], +): void { + for (const group of collectExactlyOneGroups(flagSchema)) { + const selectedMembers = group.members.filter((member) => selectedFlags.has(member)); + + if (selectedMembers.length === 1) { + continue; + } + + const preferredByProfile = getProfileExactlyOneChoice(profiles, group.key); + if (preferredByProfile && group.members.includes(preferredByProfile)) { + selectedFlags.set(preferredByProfile, { + values: [makeTypedPlaceholderValue(preferredByProfile, "string", undefined)], + source: "profile", + }); + for (const member of group.members) { + if (member !== preferredByProfile) { + selectedFlags.delete(member); + } + } + continue; + } + + const chosen = chooseExactlyOneMember( + commandId, + group.members, + flagSchema, + interactiveSignals, + ); + + const existing = selectedFlags.get(chosen); + if (!existing) { + selectedFlags.set(chosen, { + values: [makeTypedPlaceholderValue(chosen, "string", undefined)], + source: "heuristic", + }); + } + + for (const member of group.members) { + if (member !== chosen) { + selectedFlags.delete(member); + } + } + } +} + +function chooseExactlyOneMember( + commandId: string, + members: string[], + flagSchema: ParsedFlag[], + interactiveSignals: InteractiveSignal[], +): string { + if (members.includes("project-id")) { + return "project-id"; + } + + const scored = members.map((member) => { + const closure = dependencyClosureSize(member, flagSchema); + const nonInteractiveBonus = scoreNonInteractiveMember(member, interactiveSignals); + return { + member, + score: closure - nonInteractiveBonus, + }; + }); + + scored.sort((a, b) => { + if (a.score !== b.score) { + return a.score - b.score; + } + + return a.member.localeCompare(b.member); + }); + + return scored[0]?.member ?? members[0] ?? commandId; +} + +function scoreNonInteractiveMember(member: string, interactiveSignals: InteractiveSignal[]): number { + if (member === "consent" && interactiveSignals.includes("addConfirmation")) { + return 3; + } + + if (member === "force" && interactiveSignals.includes("addConfirmation")) { + return 3; + } + + if (member === "password" && interactiveSignals.includes("addInput")) { + return 3; + } + + if (member === "override-type" && interactiveSignals.includes("addSelect")) { + return 2; + } + + return 0; +} + +function dependencyClosureSize(member: string, flagSchema: ParsedFlag[]): number { + const visited = new Set(); + + const visit = (flagName: string): void => { + if (visited.has(flagName)) { + return; + } + + visited.add(flagName); + const flag = flagSchema.find((candidate) => candidate.name === flagName); + for (const dependency of flag?.dependsOn ?? []) { + visit(dependency); + } + }; + + visit(member); + return visited.size; +} + +function resolveDependencies( + flagSchema: ParsedFlag[], + selectedFlags: Map, + profiles: InvocationProfile[], +): void { + let changed = true; + + while (changed) { + changed = false; + + for (const flag of flagSchema) { + if (!selectedFlags.has(flag.name)) { + continue; + } + + for (const dependency of flag.dependsOn ?? []) { + if (selectedFlags.has(dependency)) { + continue; + } + + const dependencySpec = flagSchema.find((candidate) => candidate.name === dependency); + const profileValue = getProfileFlagValue(profiles, dependency); + if (profileValue !== undefined) { + setFlagValue(selectedFlags, dependency, normalizeFlagValue(profileValue), "profile"); + changed = true; + continue; + } + + if (!dependencySpec) { + setFlagValue(selectedFlags, dependency, ["true"], "heuristic"); + changed = true; + continue; + } + + setFlagValue(selectedFlags, dependency, buildHeuristicFlagValue(dependencySpec), "heuristic"); + changed = true; + } + } + } +} + +function resolveExclusiveGroups( + flagSchema: ParsedFlag[], + selectedFlags: Map, +): void { + for (const flag of flagSchema) { + const selected = selectedFlags.get(flag.name); + if (!selected || !flag.exclusive) { + continue; + } + + for (const otherName of flag.exclusive) { + const other = selectedFlags.get(otherName); + if (!other) { + continue; + } + + if (compareSourcePrecedence(selected.source, other.source) >= 0) { + selectedFlags.delete(otherName); + } else { + selectedFlags.delete(flag.name); + } + } + } +} + +function applyProfileOverrides( + flagSchema: ParsedFlag[], + selectedFlags: Map, + profiles: InvocationProfile[], +): void { + for (const profile of profiles) { + for (const [flagName, profileValue] of Object.entries(profile.requiredFlagDefaults ?? {})) { + const spec = flagSchema.find((flag) => flag.name === flagName); + if (!spec) { + continue; + } + + setFlagValue(selectedFlags, flagName, normalizeFlagValue(profileValue), "profile"); + } + } +} + +function decideInteractivePolicy( + commandId: string, + flagSchema: ParsedFlag[], + selectedFlags: Map, + interactiveSignals: InteractiveSignal[], + profiles: InvocationProfile[], +): "NON_INTERACTIVE_RESOLVED" | "INTERACTIVE_REQUIRED" { + if (interactiveSignals.length === 0) { + return "NON_INTERACTIVE_RESOLVED"; + } + + const policy = profiles.find((profile) => profile.interactivePolicy)?.interactivePolicy; + if (policy === "classify") { + return "INTERACTIVE_REQUIRED"; + } + + const unresolved = resolveInteractiveSignals(commandId, flagSchema, selectedFlags, interactiveSignals); + return unresolved.length === 0 ? "NON_INTERACTIVE_RESOLVED" : "INTERACTIVE_REQUIRED"; +} + +function resolveInteractiveSignals( + commandId: string, + flagSchema: ParsedFlag[], + selectedFlags: Map, + interactiveSignals: InteractiveSignal[], +): InteractiveSignal[] { + const unresolved: InteractiveSignal[] = []; + + const hasFlag = (name: string): boolean => flagSchema.some((flag) => flag.name === name); + + for (const signal of interactiveSignals) { + if (signal === "addConfirmation") { + if (hasFlag("force")) { + setFlagValue(selectedFlags, "force", ["true"], "heuristic"); + continue; + } + + if (hasFlag("consent")) { + setFlagValue(selectedFlags, "consent", ["true"], "heuristic"); + continue; + } + + unresolved.push(signal); + continue; + } + + if (signal === "addInput") { + if (hasFlag("password")) { + setFlagValue(selectedFlags, "password", ["integration-password"], "heuristic"); + continue; + } + + if (hasFlag("user-password")) { + setFlagValue(selectedFlags, "user-password", ["integration-password"], "heuristic"); + continue; + } + + unresolved.push(signal); + continue; + } + + if (signal === "addSelect") { + if (hasFlag("override-type")) { + setFlagValue(selectedFlags, "override-type", ["auto"], "heuristic"); + continue; + } + + unresolved.push(signal); + continue; + } + + unresolved.push(signal); + } + + if (commandId === "login token") { + return [...new Set([...unresolved, "addInput"])] as InteractiveSignal[]; + } + + return [...new Set(unresolved)]; +} + +function renderInvocation( + commandTokens: string[], + positionalValues: string[], + parsedFlags: ParsedFlag[], + selectedFlags: Map, +): string[] { + const args = [...commandTokens, ...positionalValues]; + + const orderedFlags = parsedFlags + .filter((flag) => selectedFlags.has(flag.name)) + .sort((a, b) => a.name.localeCompare(b.name)); + + for (const flag of orderedFlags) { + const resolved = selectedFlags.get(flag.name); + if (!resolved) { + continue; + } + + if (!flag.takesValue) { + args.push(`--${flag.name}`); + continue; + } + + for (const value of resolved.values) { + args.push(`--${flag.name}`); + args.push(value); + } + } + + return args; +} + +function setFlagValue( + map: Map, + flagName: string, + values: string[], + source: ValueSource, +): void { + const existing = map.get(flagName); + if (!existing) { + map.set(flagName, { values, source }); + return; + } + + if (compareSourcePrecedence(source, existing.source) >= 0) { + map.set(flagName, { values, source }); + } +} + +function normalizeFlagValue(value: string | boolean): string[] { + if (typeof value === "boolean") { + return [value ? "true" : "false"]; + } + + return [value]; +} + +function buildHeuristicFlagValue(flag: ParsedFlag): string[] { + if (!flag.takesValue) { + return ["true"]; + } + + if (flag.defaultValue !== undefined) { + return [flag.defaultValue]; + } + + return [makeTypedPlaceholderValue(flag.name, flag.type, flag.options)]; +} + +function getProfileArgValue(profiles: InvocationProfile[], argName: string): string | undefined { + for (const profile of profiles) { + const value = profile.requiredArgDefaults?.[argName]; + if (value !== undefined) { + return value; + } + } + + return undefined; +} + +function getProfileFlagValue( + profiles: InvocationProfile[], + flagName: string, +): string | boolean | undefined { + for (const profile of profiles) { + const value = profile.requiredFlagDefaults?.[flagName]; + if (value !== undefined) { + return value; + } + } + + return undefined; +} + +function getProfileExactlyOneChoice( + profiles: InvocationProfile[], + groupKey: string, +): string | undefined { + for (const profile of profiles) { + const choice = profile.exactlyOneChoice?.[groupKey]; + if (choice !== undefined) { + return choice; + } + } + + return undefined; +} + +function compareSourcePrecedence(a: ValueSource, b: ValueSource): number { + const precedence: Record = { + heuristic: 1, + example: 2, + profile: 3, + }; + + return precedence[a] - precedence[b]; +} + +function selectStrongerSource(current: ValueSource, candidate: ValueSource): ValueSource { + return compareSourcePrecedence(candidate, current) >= 0 ? candidate : current; +} + +function defaultValueForPlaceholderKind(kind: PlaceholderKind, name: string): string { + if (kind === "uuid") { + return DEFAULT_UUID; + } + + if (kind === "email") { + return "integration@example.com"; + } + + if (kind === "url") { + return "https://example.com"; + } + + if (kind === "duration") { + return "1h"; + } + + if (kind === "directory") { + return "/tmp/mw-integration"; + } + + if (kind === "file") { + return "/tmp/mw-integration.file"; + } + + if (kind === "password") { + return "integration-password"; + } + + if (kind === "port") { + return "12345"; + } + + return makeTypedPlaceholderValue(name, "string", undefined); +} + +function makeTypedPlaceholderValue( + name: string, + _type: string, + options: string[] | undefined, +): string { + if (options && options.length > 0) { + return options[0]; + } + + const normalized = name + .replace(/[<>[\]]/g, "") + .replace(/[^A-Za-z0-9-]+/g, "-") + .replace(/^-+/, "") + .replace(/-+$/, "") + .toLowerCase(); + + if (normalized.includes("uuid") || normalized.endsWith("id") || normalized.includes("-id")) { + return DEFAULT_UUID; + } + + if (normalized.includes("email")) { + return "integration@example.com"; + } + + if (normalized.includes("url") || normalized.includes("uri")) { + return "https://example.com"; + } + + if ( + normalized.includes("duration") || + normalized.includes("ttl") || + normalized.includes("interval") + ) { + return "1h"; + } + + if (normalized.includes("directory") || normalized.includes("path")) { + return "/tmp/mw-integration"; + } + + if (normalized.includes("password")) { + return "integration-password"; + } + + if (normalized.includes("port")) { + return "12345"; + } + + return normalized.length > 0 ? `example-${normalized}` : "example-value"; +} diff --git a/src/test/integration/command-discovery/types.ts b/src/test/integration/command-discovery/types.ts new file mode 100644 index 000000000..ff8263fa0 --- /dev/null +++ b/src/test/integration/command-discovery/types.ts @@ -0,0 +1,107 @@ +export type FlagValueType = + | "boolean" + | "string" + | "integer" + | "file" + | "directory" + | "url" + | "custom"; + +export type ValueSource = "profile" | "example" | "heuristic"; + +export type InteractiveSignal = + | "addInput" + | "addSelect" + | "addConfirmation" + | "editorFallback" + | "stdinBranch"; + +export type PlaceholderKind = + | "uuid" + | "email" + | "url" + | "duration" + | "file" + | "directory" + | "password" + | "port" + | "generic"; + +export type ParsedArg = { + name: string; + required: boolean; + defaultValue?: string; + placeholderKind: PlaceholderKind; +}; + +export type ParsedFlag = { + name: string; + required: boolean; + type: FlagValueType; + takesValue: boolean; + options?: string[]; + multiple?: boolean; + defaultValue?: string; + exactlyOne?: string[]; + exclusive?: string[]; + dependsOn?: string[]; +}; + +export type SynthesizedInvocation = { + args: string[]; + argumentSource: ValueSource; + interactiveDecision: "NON_INTERACTIVE_RESOLVED" | "INTERACTIVE_REQUIRED"; + staleExample: boolean; + staleExampleReasons: string[]; +}; + +export type DiscoveredCommand = { + commandId: string; + sourceFile: string; + commandTokens: string[]; + parsedArgs: ParsedArg[]; + parsedFlags: ParsedFlag[]; + interactiveSignals: InteractiveSignal[]; + invocationProfilesApplied: string[]; + extractionDiagnostics: string[]; + synthesizedInvocation: SynthesizedInvocation; +}; + +export type ExampleCandidate = { + args: string[]; + positionalValues: string[]; + flagValues: Map; +}; + +export type ResolvedFlagValue = { + values: string[]; + source: ValueSource; +}; + +export type InvocationProfile = { + id: string; + match: { exact?: string; prefix?: string }; + requiredFlagDefaults?: Record; + requiredArgDefaults?: Record; + exactlyOneChoice?: Record; + interactivePolicy?: "resolve" | "classify"; + disableExampleSource?: boolean; + notes?: string; +}; + +export type WaiverCategory = + | "ARG_MISUSE" + | "INTERACTIVE_REQUIRED" + | "RESOURCE_PRECONDITION" + | "CONTRACT_SHAPE" + | "COMMAND_BUG" + | "DEPRECATED_ENDPOINT"; + +export type CommandWaiver = { + id: string; + commandId: string; + category: WaiverCategory; + reason: string; + issue?: string; + expiresOn?: string; +}; diff --git a/src/test/integration/command.ts b/src/test/integration/command.ts new file mode 100644 index 000000000..2d0edb1bb --- /dev/null +++ b/src/test/integration/command.ts @@ -0,0 +1,127 @@ +import { spawn } from "node:child_process"; + +export type DevCommandResult = { + stdout: string; + stderr: string; + exitCode: number | null; + signal: NodeJS.Signals | null; + error?: Error; + timedOut?: boolean; +}; + +export type RunDevCommandOptions = { + cwd?: string; + env?: NodeJS.ProcessEnv; + timeoutMs?: number; +}; + +export async function runDevCommand( + args: string[], + options: RunDevCommandOptions = {}, +): Promise { + return await new Promise((resolve) => { + const timeoutMs = options.timeoutMs ?? 30_000; + + const child = spawn( + "yarn", + [ + "node", + "--import", + "tsx", + "--no-warnings=ExperimentalWarning", + "./bin/dev.js", + ...args, + ], + { + cwd: options.cwd ?? process.cwd(), + env: options.env ?? process.env, + stdio: ["ignore", "pipe", "pipe"], + }, + ); + + let stdout = ""; + let stderr = ""; + let settled = false; + let didTimeOut = false; + + const finish = (result: DevCommandResult): void => { + if (settled) { + return; + } + + settled = true; + if (timeoutHandle) { + clearTimeout(timeoutHandle); + } + if (forceKillHandle) { + clearTimeout(forceKillHandle); + } + + resolve(result); + }; + + let forceKillHandle: NodeJS.Timeout | undefined; + const timeoutHandle: NodeJS.Timeout | undefined = + timeoutMs > 0 + ? setTimeout(() => { + didTimeOut = true; + child.kill("SIGTERM"); + + // Give graceful termination a short window before hard-killing. + forceKillHandle = setTimeout(() => { + child.kill("SIGKILL"); + }, 2_000); + forceKillHandle.unref?.(); + }, timeoutMs) + : undefined; + + timeoutHandle?.unref?.(); + + child.stdout.on("data", (chunk) => { + stdout += chunk.toString(); + }); + + child.stderr.on("data", (chunk) => { + stderr += chunk.toString(); + }); + + child.on("error", (error) => { + finish({ + stdout, + stderr, + exitCode: null, + signal: null, + timedOut: didTimeOut, + error, + }); + }); + + child.on("close", (exitCode, signal) => { + if (didTimeOut) { + finish({ + stdout, + stderr, + exitCode, + signal, + timedOut: true, + error: new Error(`dev.js subprocess timed out after ${timeoutMs}ms`), + }); + return; + } + + if (exitCode === 0) { + finish({ stdout, stderr, exitCode, signal, timedOut: false }); + return; + } + + finish({ + stdout, + stderr, + exitCode, + signal, + timedOut: false, + error: new Error(`dev.js subprocess exited with code ${exitCode}`), + }); + }); + }); +} \ No newline at end of file diff --git a/src/test/integration/config/command-classifications.json b/src/test/integration/config/command-classifications.json new file mode 100644 index 000000000..5f2fb44e3 --- /dev/null +++ b/src/test/integration/config/command-classifications.json @@ -0,0 +1,345 @@ +{ + "schemaVersion": 1, + "generatedAt": "2026-08-03T07:37:35.513Z", + "source": { + "kind": "run-all-summary" + }, + "statistics": { + "successful": 119, + "failed": 0, + "waivedSkipped": 66, + "total": 185 + }, + "entries": [ + { + "commandId": "app create node", + "category": "CONTRACT_SHAPE", + "source": "waiver" + }, + { + "commandId": "app create php", + "category": "CONTRACT_SHAPE", + "source": "waiver" + }, + { + "commandId": "app create php-worker", + "category": "CONTRACT_SHAPE", + "source": "waiver" + }, + { + "commandId": "app create python", + "category": "CONTRACT_SHAPE", + "source": "waiver" + }, + { + "commandId": "app create static", + "category": "CONTRACT_SHAPE", + "source": "waiver" + }, + { + "commandId": "app database link", + "category": "DEPRECATED_ENDPOINT", + "source": "waiver" + }, + { + "commandId": "app database replace", + "category": "DEPRECATED_ENDPOINT", + "source": "waiver" + }, + { + "commandId": "app dependency update", + "category": "COMMAND_BUG", + "source": "waiver" + }, + { + "commandId": "app dependency versions", + "category": "RESOURCE_PRECONDITION", + "source": "waiver" + }, + { + "commandId": "app download", + "category": "CONTRACT_SHAPE", + "source": "waiver" + }, + { + "commandId": "app exec", + "category": "CONTRACT_SHAPE", + "source": "waiver" + }, + { + "commandId": "app get", + "category": "CONTRACT_SHAPE", + "source": "waiver" + }, + { + "commandId": "app install contao", + "category": "CONTRACT_SHAPE", + "source": "waiver" + }, + { + "commandId": "app install joomla", + "category": "CONTRACT_SHAPE", + "source": "waiver" + }, + { + "commandId": "app install matomo", + "category": "CONTRACT_SHAPE", + "source": "waiver" + }, + { + "commandId": "app install nextcloud", + "category": "CONTRACT_SHAPE", + "source": "waiver" + }, + { + "commandId": "app install shopware5", + "category": "CONTRACT_SHAPE", + "source": "waiver" + }, + { + "commandId": "app install shopware6", + "category": "CONTRACT_SHAPE", + "source": "waiver" + }, + { + "commandId": "app install typo3", + "category": "CONTRACT_SHAPE", + "source": "waiver" + }, + { + "commandId": "app install wordpress", + "category": "CONTRACT_SHAPE", + "source": "waiver" + }, + { + "commandId": "app list-upgrade-candidates", + "category": "COMMAND_BUG", + "source": "waiver" + }, + { + "commandId": "app open", + "category": "COMMAND_BUG", + "source": "waiver" + }, + { + "commandId": "app ssh", + "category": "CONTRACT_SHAPE", + "source": "waiver" + }, + { + "commandId": "app upgrade", + "category": "INTERACTIVE_REQUIRED", + "source": "waiver" + }, + { + "commandId": "app upload", + "category": "RESOURCE_PRECONDITION", + "source": "waiver" + }, + { + "commandId": "app version-info", + "category": "RESOURCE_PRECONDITION", + "source": "waiver" + }, + { + "commandId": "app versions", + "category": "COMMAND_BUG", + "source": "waiver" + }, + { + "commandId": "backup download", + "category": "COMMAND_BUG", + "source": "waiver" + }, + { + "commandId": "container cp", + "category": "RESOURCE_PRECONDITION", + "source": "waiver" + }, + { + "commandId": "container delete", + "category": "RESOURCE_PRECONDITION", + "source": "waiver" + }, + { + "commandId": "container exec", + "category": "RESOURCE_PRECONDITION", + "source": "waiver" + }, + { + "commandId": "container logs", + "category": "INTERACTIVE_REQUIRED", + "source": "waiver" + }, + { + "commandId": "container port-forward", + "category": "RESOURCE_PRECONDITION", + "source": "waiver" + }, + { + "commandId": "container recreate", + "category": "RESOURCE_PRECONDITION", + "source": "waiver" + }, + { + "commandId": "container restart", + "category": "RESOURCE_PRECONDITION", + "source": "waiver" + }, + { + "commandId": "container run", + "category": "RESOURCE_PRECONDITION", + "source": "waiver" + }, + { + "commandId": "container ssh", + "category": "RESOURCE_PRECONDITION", + "source": "waiver" + }, + { + "commandId": "container start", + "category": "RESOURCE_PRECONDITION", + "source": "waiver" + }, + { + "commandId": "container stop", + "category": "RESOURCE_PRECONDITION", + "source": "waiver" + }, + { + "commandId": "container update", + "category": "RESOURCE_PRECONDITION", + "source": "waiver" + }, + { + "commandId": "conversation create", + "category": "COMMAND_BUG", + "source": "waiver" + }, + { + "commandId": "conversation reply", + "category": "COMMAND_BUG", + "source": "waiver" + }, + { + "commandId": "cronjob execution logs", + "category": "INTERACTIVE_REQUIRED", + "source": "waiver" + }, + { + "commandId": "database mysql dump", + "category": "RESOURCE_PRECONDITION", + "source": "waiver" + }, + { + "commandId": "database mysql import", + "category": "INTERACTIVE_REQUIRED", + "source": "waiver" + }, + { + "commandId": "database mysql phpmyadmin", + "category": "RESOURCE_PRECONDITION", + "source": "waiver" + }, + { + "commandId": "database mysql shell", + "category": "INTERACTIVE_REQUIRED", + "source": "waiver" + }, + { + "commandId": "database mysql upgrade", + "category": "INTERACTIVE_REQUIRED", + "source": "waiver" + }, + { + "commandId": "database mysql user delete", + "category": "RESOURCE_PRECONDITION", + "source": "waiver" + }, + { + "commandId": "ddev init", + "category": "INTERACTIVE_REQUIRED", + "source": "waiver" + }, + { + "commandId": "ddev render-config", + "category": "CONTRACT_SHAPE", + "source": "waiver" + }, + { + "commandId": "domain dnszone get", + "category": "RESOURCE_PRECONDITION", + "source": "waiver" + }, + { + "commandId": "domain dnszone update", + "category": "RESOURCE_PRECONDITION", + "source": "waiver" + }, + { + "commandId": "domain get", + "category": "RESOURCE_PRECONDITION", + "source": "waiver" + }, + { + "commandId": "experimental deploy", + "category": "RESOURCE_PRECONDITION", + "source": "waiver" + }, + { + "commandId": "login token", + "category": "INTERACTIVE_REQUIRED", + "source": "waiver" + }, + { + "commandId": "mail address update", + "category": "RESOURCE_PRECONDITION", + "source": "waiver" + }, + { + "commandId": "org delete", + "category": "RESOURCE_PRECONDITION", + "source": "waiver" + }, + { + "commandId": "sftp-user create", + "category": "RESOURCE_PRECONDITION", + "source": "waiver" + }, + { + "commandId": "ssh-user create", + "category": "RESOURCE_PRECONDITION", + "source": "waiver" + }, + { + "commandId": "stack delete", + "category": "COMMAND_BUG", + "source": "waiver" + }, + { + "commandId": "stack deploy", + "category": "INTERACTIVE_REQUIRED", + "source": "waiver" + }, + { + "commandId": "stack set-update-schedule", + "category": "DEPRECATED_ENDPOINT", + "source": "waiver" + }, + { + "commandId": "stack unset-update-schedule", + "category": "DEPRECATED_ENDPOINT", + "source": "waiver" + }, + { + "commandId": "user ssh-key create", + "category": "INTERACTIVE_REQUIRED", + "source": "waiver" + }, + { + "commandId": "volume delete", + "category": "RESOURCE_PRECONDITION", + "source": "waiver" + } + ] +} diff --git a/src/test/integration/config/command-waivers.json b/src/test/integration/config/command-waivers.json new file mode 100644 index 000000000..7905e56bb --- /dev/null +++ b/src/test/integration/config/command-waivers.json @@ -0,0 +1,464 @@ +[ + { + "id": "interactive-app-upgrade", + "commandId": "app upgrade", + "category": "INTERACTIVE_REQUIRED", + "reason": "Upgrade target selection is interactive and has no stable non-interactive override yet.", + "issue": "defer-interactive-support" + }, + { + "id": "interactive-container-logs", + "commandId": "container logs", + "category": "INTERACTIVE_REQUIRED", + "reason": "Log follow behavior depends on interactive terminal controls in current implementation.", + "issue": "defer-interactive-support" + }, + { + "id": "interactive-cronjob-execution-logs", + "commandId": "cronjob execution logs", + "category": "INTERACTIVE_REQUIRED", + "reason": "Execution log access path prompts or expects interactive I/O in this test setup.", + "issue": "defer-interactive-support" + }, + { + "id": "interactive-database-mysql-import", + "commandId": "database mysql import", + "category": "INTERACTIVE_REQUIRED", + "reason": "Import flow expects interactive input or file prompt handling not available in the simple renderer.", + "issue": "defer-interactive-support" + }, + { + "id": "interactive-database-mysql-shell", + "commandId": "database mysql shell", + "category": "INTERACTIVE_REQUIRED", + "reason": "MySQL shell requires password prompt interaction and cannot run headless yet.", + "issue": "defer-interactive-support" + }, + { + "id": "interactive-database-mysql-upgrade", + "commandId": "database mysql upgrade", + "category": "INTERACTIVE_REQUIRED", + "reason": "Upgrade confirmation and version choice currently requires interactive input.", + "issue": "defer-interactive-support" + }, + { + "id": "interactive-ddev-init", + "commandId": "ddev init", + "category": "INTERACTIVE_REQUIRED", + "reason": "Project type and configuration selection still enters interactive decision branches.", + "issue": "defer-interactive-support" + }, + { + "id": "interactive-login-token", + "commandId": "login token", + "category": "INTERACTIVE_REQUIRED", + "reason": "Token acquisition flow is intentionally interactive for secure input handling.", + "issue": "defer-interactive-support" + }, + { + "id": "interactive-stack-deploy", + "commandId": "stack deploy", + "category": "INTERACTIVE_REQUIRED", + "reason": "Deploy flow requires interactive input in current command implementation.", + "issue": "defer-interactive-support" + }, + { + "id": "interactive-user-ssh-key-create", + "commandId": "user ssh-key create", + "category": "INTERACTIVE_REQUIRED", + "reason": "SSH key creation relies on interactive prompts for key source and confirmation.", + "issue": "defer-interactive-support" + }, + { + "id": "deprecated-endpoint-app-database-link", + "commandId": "app database link", + "category": "DEPRECATED_ENDPOINT", + "reason": "Command currently uses a deprecated API endpoint that is filtered by the mockoon setup and therefore not served in integration runs. The command should be reworked to use a current non-deprecated endpoint.", + "issue": "rework-deprecated-endpoint" + }, + { + "id": "deprecated-endpoint-app-database-replace", + "commandId": "app database replace", + "category": "DEPRECATED_ENDPOINT", + "reason": "Command currently uses a deprecated API endpoint that is filtered by the mockoon setup and therefore not served in integration runs. The command should be reworked to use a current non-deprecated endpoint.", + "issue": "rework-deprecated-endpoint" + }, + { + "id": "deprecated-endpoint-stack-set-update-schedule", + "commandId": "stack set-update-schedule", + "category": "DEPRECATED_ENDPOINT", + "reason": "Command currently uses a deprecated API endpoint that is filtered by the mockoon setup and therefore not served in integration runs. The command should be reworked to use a current non-deprecated endpoint.", + "issue": "rework-deprecated-endpoint" + }, + { + "id": "deprecated-endpoint-stack-unset-update-schedule", + "commandId": "stack unset-update-schedule", + "category": "DEPRECATED_ENDPOINT", + "reason": "Command currently uses a deprecated API endpoint that is filtered by the mockoon setup and therefore not served in integration runs. The command should be reworked to use a current non-deprecated endpoint.", + "issue": "rework-deprecated-endpoint" + }, + { + "id": "contract-shape-app-create-node-invalid-version", + "commandId": "app create node", + "category": "CONTRACT_SHAPE", + "reason": "Version fields in the app version contract are typed as generic strings (openapi schema), but client selection logic assumes semver-compatible internalVersion values and calls semver.gt directly. In integration fixtures this can surface as empty or non-semver values and crashes with TypeError: Invalid Version before command-specific logic can proceed.", + "issue": "harden-app-version-selection" + }, + { + "id": "contract-shape-app-create-php-invalid-version", + "commandId": "app create php", + "category": "CONTRACT_SHAPE", + "reason": "Version fields in the app version contract are typed as generic strings (openapi schema), but client selection logic assumes semver-compatible internalVersion values and calls semver.gt directly. In integration fixtures this can surface as empty or non-semver values and crashes with TypeError: Invalid Version before command-specific logic can proceed.", + "issue": "harden-app-version-selection" + }, + { + "id": "contract-shape-app-create-php-worker-invalid-version", + "commandId": "app create php-worker", + "category": "CONTRACT_SHAPE", + "reason": "Version fields in the app version contract are typed as generic strings (openapi schema), but client selection logic assumes semver-compatible internalVersion values and calls semver.gt directly. In integration fixtures this can surface as empty or non-semver values and crashes with TypeError: Invalid Version before command-specific logic can proceed.", + "issue": "harden-app-version-selection" + }, + { + "id": "contract-shape-app-create-python-invalid-version", + "commandId": "app create python", + "category": "CONTRACT_SHAPE", + "reason": "Version fields in the app version contract are typed as generic strings (openapi schema), but client selection logic assumes semver-compatible internalVersion values and calls semver.gt directly. In integration fixtures this can surface as empty or non-semver values and crashes with TypeError: Invalid Version before command-specific logic can proceed.", + "issue": "harden-app-version-selection" + }, + { + "id": "contract-shape-app-create-static-invalid-version", + "commandId": "app create static", + "category": "CONTRACT_SHAPE", + "reason": "Version fields in the app version contract are typed as generic strings (openapi schema), but client selection logic assumes semver-compatible internalVersion values and calls semver.gt directly. In integration fixtures this can surface as empty or non-semver values and crashes with TypeError: Invalid Version before command-specific logic can proceed.", + "issue": "harden-app-version-selection" + }, + { + "id": "contract-shape-app-install-contao-invalid-version", + "commandId": "app install contao", + "category": "CONTRACT_SHAPE", + "reason": "Version fields in the app version contract are typed as generic strings (openapi schema), but client selection logic assumes semver-compatible internalVersion values and calls semver.gt directly. In integration fixtures this can surface as empty or non-semver values and crashes with TypeError: Invalid Version before command-specific logic can proceed.", + "issue": "harden-app-version-selection" + }, + { + "id": "contract-shape-app-install-shopware5-invalid-version", + "commandId": "app install shopware5", + "category": "CONTRACT_SHAPE", + "reason": "Version fields in the app version contract are typed as generic strings (openapi schema), but client selection logic assumes semver-compatible internalVersion values and calls semver.gt directly. In integration fixtures this can surface as empty or non-semver values and crashes with TypeError: Invalid Version before command-specific logic can proceed.", + "issue": "harden-app-version-selection" + }, + { + "id": "contract-shape-app-install-shopware6-invalid-version", + "commandId": "app install shopware6", + "category": "CONTRACT_SHAPE", + "reason": "Version fields in the app version contract are typed as generic strings (openapi schema), but client selection logic assumes semver-compatible internalVersion values and calls semver.gt directly. In integration fixtures this can surface as empty or non-semver values and crashes with TypeError: Invalid Version before command-specific logic can proceed.", + "issue": "harden-app-version-selection" + }, + { + "id": "contract-shape-app-install-typo3-invalid-version", + "commandId": "app install typo3", + "category": "CONTRACT_SHAPE", + "reason": "Version fields in the app version contract are typed as generic strings (openapi schema), but client selection logic assumes semver-compatible internalVersion values and calls semver.gt directly. In integration fixtures this can surface as empty or non-semver values and crashes with TypeError: Invalid Version before command-specific logic can proceed.", + "issue": "harden-app-version-selection" + }, + { + "id": "contract-shape-app-install-wordpress-invalid-version", + "commandId": "app install wordpress", + "category": "CONTRACT_SHAPE", + "reason": "Version fields in the app version contract are typed as generic strings (openapi schema), but client selection logic assumes semver-compatible internalVersion values and calls semver.gt directly. In integration fixtures this can surface as empty or non-semver values and crashes with TypeError: Invalid Version before command-specific logic can proceed.", + "issue": "harden-app-version-selection" + }, + { + "id": "resource-precondition-container-cp-container-not-found", + "commandId": "container cp", + "category": "RESOURCE_PRECONDITION", + "reason": "Shared container resolution precondition is not satisfied in integration fixtures. The resolver in src/lib/resources/container/flags.ts (withContainerAndStackId) lists project services and throws when no matching container shortId/id/serviceName exists. Current run fails with 'no container mycontainer found in project p-f0ob4r'. Preferred fix is to seed a deterministic known container fixture for project p-f0ob4r and align invocation profiles/examples to that known container identifier.", + "issue": "seed-known-container-fixture" + }, + { + "id": "resource-precondition-container-delete-container-not-found", + "commandId": "container delete", + "category": "RESOURCE_PRECONDITION", + "reason": "Shared container resolution precondition is not satisfied in integration fixtures. The resolver in src/lib/resources/container/flags.ts (withContainerAndStackId) lists project services and throws when no matching container shortId/id/serviceName exists. Current run shows deletion flow failing because the requested container identifier does not exist in project p-f0ob4r. Preferred fix is to seed a deterministic known container fixture for project p-f0ob4r and align invocation profiles/examples to that known container identifier.", + "issue": "seed-known-container-fixture" + }, + { + "id": "resource-precondition-container-exec-container-not-found", + "commandId": "container exec", + "category": "RESOURCE_PRECONDITION", + "reason": "Shared container resolution precondition is not satisfied in integration fixtures. The resolver in src/lib/resources/container/flags.ts (withContainerAndStackId) lists project services and throws when no matching container shortId/id/serviceName exists. Current run fails with 'no container 00000000-0000-4000-8000-000000000000 found in project p-f0ob4r'. Preferred fix is to seed a deterministic known container fixture for project p-f0ob4r and align invocation profiles/examples to that known container identifier.", + "issue": "seed-known-container-fixture" + }, + { + "id": "resource-precondition-container-port-forward-container-not-found", + "commandId": "container port-forward", + "category": "RESOURCE_PRECONDITION", + "reason": "Shared container resolution precondition is not satisfied in integration fixtures. The resolver in src/lib/resources/container/flags.ts (withContainerAndStackId) lists project services and throws when no matching container shortId/id/serviceName exists. Current run fails with 'no container 00000000-0000-4000-8000-000000000000 found in project p-f0ob4r'. Preferred fix is to seed a deterministic known container fixture for project p-f0ob4r and align invocation profiles/examples to that known container identifier.", + "issue": "seed-known-container-fixture" + }, + { + "id": "resource-precondition-container-recreate-container-not-found", + "commandId": "container recreate", + "category": "RESOURCE_PRECONDITION", + "reason": "Shared container resolution precondition is not satisfied in integration fixtures. The resolver in src/lib/resources/container/flags.ts (withContainerAndStackId) lists project services and throws when no matching container shortId/id/serviceName exists. Current run fails with 'no container 00000000-0000-4000-8000-000000000000 found in project p-f0ob4r'. Preferred fix is to seed a deterministic known container fixture for project p-f0ob4r and align invocation profiles/examples to that known container identifier.", + "issue": "seed-known-container-fixture" + }, + { + "id": "resource-precondition-container-restart-container-not-found", + "commandId": "container restart", + "category": "RESOURCE_PRECONDITION", + "reason": "Shared container resolution precondition is not satisfied in integration fixtures. The resolver in src/lib/resources/container/flags.ts (withContainerAndStackId) lists project services and throws when no matching container shortId/id/serviceName exists. Current run fails with 'no container 00000000-0000-4000-8000-000000000000 found in project p-f0ob4r'. Preferred fix is to seed a deterministic known container fixture for project p-f0ob4r and align invocation profiles/examples to that known container identifier.", + "issue": "seed-known-container-fixture" + }, + { + "id": "resource-precondition-container-ssh-container-not-found", + "commandId": "container ssh", + "category": "RESOURCE_PRECONDITION", + "reason": "Shared container resolution precondition is not satisfied in integration fixtures. The resolver in src/lib/resources/container/flags.ts (withContainerAndStackId) lists project services and throws when no matching container shortId/id/serviceName exists. Current run fails with 'no container 00000000-0000-4000-8000-000000000000 found in project p-f0ob4r'. Preferred fix is to seed a deterministic known container fixture for project p-f0ob4r and align invocation profiles/examples to that known container identifier.", + "issue": "seed-known-container-fixture" + }, + { + "id": "resource-precondition-container-start-container-not-found", + "commandId": "container start", + "category": "RESOURCE_PRECONDITION", + "reason": "Shared container resolution precondition is not satisfied in integration fixtures. The resolver in src/lib/resources/container/flags.ts (withContainerAndStackId) lists project services and throws when no matching container shortId/id/serviceName exists. Current run fails with 'no container 00000000-0000-4000-8000-000000000000 found in project p-f0ob4r'. Preferred fix is to seed a deterministic known container fixture for project p-f0ob4r and align invocation profiles/examples to that known container identifier.", + "issue": "seed-known-container-fixture" + }, + { + "id": "resource-precondition-container-stop-container-not-found", + "commandId": "container stop", + "category": "RESOURCE_PRECONDITION", + "reason": "Shared container resolution precondition is not satisfied in integration fixtures. The resolver in src/lib/resources/container/flags.ts (withContainerAndStackId) lists project services and throws when no matching container shortId/id/serviceName exists. Current run fails with 'no container 00000000-0000-4000-8000-000000000000 found in project p-f0ob4r'. Preferred fix is to seed a deterministic known container fixture for project p-f0ob4r and align invocation profiles/examples to that known container identifier.", + "issue": "seed-known-container-fixture" + }, + { + "id": "resource-precondition-container-update-container-not-found", + "commandId": "container update", + "category": "RESOURCE_PRECONDITION", + "reason": "Shared container resolution precondition is not satisfied in integration fixtures. The resolver in src/lib/resources/container/flags.ts (withContainerAndStackId) lists project services and throws when no matching container shortId/id/serviceName exists. Current run fails with 'no container 00000000-0000-4000-8000-000000000000 found in project p-f0ob4r'. Preferred fix is to seed a deterministic known container fixture for project p-f0ob4r and align invocation profiles/examples to that known container identifier.", + "issue": "seed-known-container-fixture" + }, + { + "id": "contract-shape-app-download-missing-web-directory", + "commandId": "app download", + "category": "CONTRACT_SHAPE", + "reason": "Command fails with TypeError [ERR_INVALID_ARG_TYPE] in a shared path.join call because integration response shape does not provide all path inputs expected by shared SSH/app-installation rendering logic. Primary shared callsite: src/lib/resources/ssh/appinstall.ts (getSSHConnectionForAppInstallation) builds directory via path.join(projectResponse.data.directories['Web'], appInstallation.installationPath). If directories['Web'] or installationPath is missing/undefined, multiple commands crash with the same signature. Preferred fix is fixture-side: seed deterministic project/app-installation payloads that always include project.directories['Web'] and appInstallation.installationPath for the known test project.", + "issue": "seed-appinstall-web-directory-fixture" + }, + { + "id": "contract-shape-app-exec-missing-web-directory", + "commandId": "app exec", + "category": "CONTRACT_SHAPE", + "reason": "Command fails with TypeError [ERR_INVALID_ARG_TYPE] in a shared path.join call because integration response shape does not provide all path inputs expected by shared SSH/app-installation rendering logic. Primary shared callsite: src/lib/resources/ssh/appinstall.ts (getSSHConnectionForAppInstallation) builds directory via path.join(projectResponse.data.directories['Web'], appInstallation.installationPath). If directories['Web'] or installationPath is missing/undefined, multiple commands crash with the same signature. Preferred fix is fixture-side: seed deterministic project/app-installation payloads that always include project.directories['Web'] and appInstallation.installationPath for the known test project.", + "issue": "seed-appinstall-web-directory-fixture" + }, + { + "id": "contract-shape-app-ssh-missing-web-directory", + "commandId": "app ssh", + "category": "CONTRACT_SHAPE", + "reason": "Command fails with TypeError [ERR_INVALID_ARG_TYPE] in a shared path.join call because integration response shape does not provide all path inputs expected by shared SSH/app-installation rendering logic. Primary shared callsite: src/lib/resources/ssh/appinstall.ts (getSSHConnectionForAppInstallation) builds directory via path.join(projectResponse.data.directories['Web'], appInstallation.installationPath). If directories['Web'] or installationPath is missing/undefined, multiple commands crash with the same signature. Preferred fix is fixture-side: seed deterministic project/app-installation payloads that always include project.directories['Web'] and appInstallation.installationPath for the known test project.", + "issue": "seed-appinstall-web-directory-fixture" + }, + { + "id": "contract-shape-app-get-missing-web-directory", + "commandId": "app get", + "category": "CONTRACT_SHAPE", + "reason": "Command fails with TypeError [ERR_INVALID_ARG_TYPE] from AppInstallationDetails rendering when absolute installation path is built with path.join(project.directories['Web'], appInstallation.installationPath). This is the same fixture data-shape gap as the shared SSH app-installation path handling cluster: missing directories['Web'] and/or installationPath in test payloads causes deterministic crash. Preferred fix is fixture-side: seed deterministic project/app-installation payloads that always include project.directories['Web'] and appInstallation.installationPath for the known test project.", + "issue": "seed-appinstall-web-directory-fixture" + }, + { + "id": "contract-shape-app-install-joomla-invalid-version", + "commandId": "app install joomla", + "category": "CONTRACT_SHAPE", + "reason": "Version fields in the app version contract are typed as generic strings (openapi schema), but client selection logic assumes semver-compatible internalVersion values and calls semver.gt directly. In integration fixtures this can surface as empty or non-semver values and crashes with TypeError: Invalid Version before command-specific logic can proceed.", + "issue": "harden-app-version-selection" + }, + { + "id": "contract-shape-app-install-nextcloud-invalid-version", + "commandId": "app install nextcloud", + "category": "CONTRACT_SHAPE", + "reason": "Version fields in the app version contract are typed as generic strings (openapi schema), but client selection logic assumes semver-compatible internalVersion values and calls semver.gt directly. In integration fixtures this can surface as empty or non-semver values and crashes with TypeError: Invalid Version before command-specific logic can proceed.", + "issue": "harden-app-version-selection" + }, + { + "id": "command-bug-app-dependency-update-invalid-installation-id", + "commandId": "app dependency update", + "category": "COMMAND_BUG", + "reason": "Integration invocation currently passes a placeholder that does not resolve to a valid app installation identifier for this command path. The command exits with an app-installation ID validation error in this test setup.", + "issue": "fix-integration-invocation-profiles" + }, + { + "id": "resource-precondition-app-dependency-versions-systemsoftware-not-found", + "commandId": "app dependency versions", + "category": "RESOURCE_PRECONDITION", + "reason": "The integration fixture set does not provide a resolvable system software entry for the placeholder value used in this command run. The command fails with 'system software ... not found'.", + "issue": "seed-known-resource-fixtures" + }, + { + "id": "contract-shape-app-install-matomo-invalid-version", + "commandId": "app install matomo", + "category": "CONTRACT_SHAPE", + "reason": "Version fields in the app version contract are typed as generic strings (openapi schema), but client selection logic assumes semver-compatible internalVersion values and calls semver.gt directly. In integration fixtures this can surface as empty or non-semver values and crashes with TypeError: Invalid Version before command-specific logic can proceed.", + "issue": "harden-app-version-selection" + }, + { + "id": "command-bug-app-list-upgrade-candidates-versions-not-array", + "commandId": "app list-upgrade-candidates", + "category": "COMMAND_BUG", + "reason": "The command expects a sortable versions array, but the current integration response shape provides a non-array value and execution fails with 'versions.sort is not a function'.", + "issue": "harden-response-shape-handling" + }, + { + "id": "command-bug-app-open-missing-virtualhost-link", + "commandId": "app open", + "category": "COMMAND_BUG", + "reason": "The test fixture app installation used in integration is not linked to a virtual host, and this command currently fails along that path in run-all execution.", + "issue": "seed-known-resource-fixtures" + }, + { + "id": "resource-precondition-app-upload-source-placeholder-not-available", + "commandId": "app upload", + "category": "RESOURCE_PRECONDITION", + "reason": "The integration invocation for this command does not provide a usable source path in the run-all environment. The command fails while parsing the source input.", + "issue": "fix-integration-invocation-profiles" + }, + { + "id": "resource-precondition-app-version-info-app-not-found", + "commandId": "app version-info", + "category": "RESOURCE_PRECONDITION", + "reason": "The fixture app identifier used in integration cannot be resolved in the mocked dataset, and the command fails with 'app ... not found'.", + "issue": "seed-known-resource-fixtures" + }, + { + "id": "command-bug-app-versions-access-denied", + "commandId": "app versions", + "category": "COMMAND_BUG", + "reason": "The command path fails with an access denied error in the current integration authorization/fixture setup.", + "issue": "seed-permissions-fixtures" + }, + { + "id": "command-bug-backup-download-not-ready", + "commandId": "backup download", + "category": "COMMAND_BUG", + "reason": "This command currently terminates with 'backup download is not ready' in the integration scenario, indicating an unfinished command path for this fixture state.", + "issue": "implement-backup-download-path" + }, + { + "id": "resource-precondition-container-run-service-id-not-found", + "commandId": "container run", + "category": "RESOURCE_PRECONDITION", + "reason": "The created stack in integration fixtures does not expose the expected service mapping for this flow. The command fails with 'Service ID not found in the created stack'.", + "issue": "seed-stack-service-fixture" + }, + { + "id": "command-bug-conversation-create-tempfile-unlink-enoent", + "commandId": "conversation create", + "category": "COMMAND_BUG", + "reason": "The command hits a temporary-file cleanup race in this run path and fails with ENOENT on unlink of a generated markdown file.", + "issue": "stabilize-tempfile-lifecycle" + }, + { + "id": "command-bug-conversation-reply-tempfile-unlink-enoent", + "commandId": "conversation reply", + "category": "COMMAND_BUG", + "reason": "The command hits a temporary-file cleanup race in this run path and fails with ENOENT on unlink of a generated markdown file.", + "issue": "stabilize-tempfile-lifecycle" + }, + { + "id": "resource-precondition-database-mysql-dump-main-user-missing", + "commandId": "database mysql dump", + "category": "RESOURCE_PRECONDITION", + "reason": "The MySQL dump flow requires a resolvable main user in fixtures. In this run the command fails with 'No main user found'.", + "issue": "seed-mysql-main-user-fixture" + }, + { + "id": "resource-precondition-database-mysql-phpmyadmin-main-user-missing", + "commandId": "database mysql phpmyadmin", + "category": "RESOURCE_PRECONDITION", + "reason": "The phpMyAdmin flow requires a resolvable main user in fixtures. In this run the command fails with 'no main user found'.", + "issue": "seed-mysql-main-user-fixture" + }, + { + "id": "resource-precondition-database-mysql-user-delete-main-user-protected", + "commandId": "database mysql user delete", + "category": "RESOURCE_PRECONDITION", + "reason": "Integration currently targets the main MySQL user, which is protected by API rules and cannot be deleted manually in this flow.", + "issue": "seed-mysql-non-main-user-fixture" + }, + { + "id": "contract-shape-ddev-render-config-missing-document-root-input", + "commandId": "ddev render-config", + "category": "CONTRACT_SHAPE", + "reason": "DDEV config generation currently receives fixture data with missing path fields and fails with 'Cannot read properties of undefined (reading replace)' in config builder path normalization.", + "issue": "seed-ddev-config-shape-fixture" + }, + { + "id": "resource-precondition-domain-dnszone-get-zone-not-found", + "commandId": "domain dnszone get", + "category": "RESOURCE_PRECONDITION", + "reason": "The integration fixture dataset does not include the requested DNS zone domain in this run context, causing a deterministic 'DNS zone ... not found' failure.", + "issue": "seed-known-domain-fixture" + }, + { + "id": "resource-precondition-domain-dnszone-update-zone-not-found", + "commandId": "domain dnszone update", + "category": "RESOURCE_PRECONDITION", + "reason": "The integration fixture dataset does not include the requested DNS zone domain in this run context, causing a deterministic 'DNS zone ... not found' failure.", + "issue": "seed-known-domain-fixture" + }, + { + "id": "resource-precondition-domain-get-zone-not-found", + "commandId": "domain get", + "category": "RESOURCE_PRECONDITION", + "reason": "Domain lookup in this integration path depends on a DNS zone fixture that is not present for the placeholder value, producing 'DNS zone ... not found'.", + "issue": "seed-known-domain-fixture" + }, + { + "id": "resource-precondition-experimental-deploy-registry-service-missing", + "commandId": "experimental deploy", + "category": "RESOURCE_PRECONDITION", + "reason": "The deploy orchestration expects a registry service fixture that is not returned in this integration environment and fails with 'Service not found in response'.", + "issue": "seed-stack-service-fixture" + }, + { + "id": "resource-precondition-mail-address-update-mail-address-not-found", + "commandId": "mail address update", + "category": "RESOURCE_PRECONDITION", + "reason": "The mail address used by integration placeholders is not present in the mocked dataset during this run, causing a deterministic not-found failure.", + "issue": "seed-known-resource-fixtures" + }, + { + "id": "resource-precondition-org-delete-org-not-found", + "commandId": "org delete", + "category": "RESOURCE_PRECONDITION", + "reason": "The organization targeted by integration placeholders does not exist in the fixture state at deletion time, resulting in a 404 failure.", + "issue": "seed-known-resource-fixtures" + }, + { + "id": "resource-precondition-sftp-user-create-readback-404", + "commandId": "sftp-user create", + "category": "RESOURCE_PRECONDITION", + "reason": "Creation step succeeds, but follow-up readback in integration fixtures returns 404, indicating inconsistent fixture state for immediate lookup.", + "issue": "align-user-create-fixture-readback" + }, + { + "id": "resource-precondition-ssh-user-create-readback-404", + "commandId": "ssh-user create", + "category": "RESOURCE_PRECONDITION", + "reason": "Creation step succeeds, but follow-up readback in integration fixtures returns 404, indicating inconsistent fixture state for immediate lookup.", + "issue": "align-user-create-fixture-readback" + }, + { + "id": "command-bug-stack-delete-not-implemented", + "commandId": "stack delete", + "category": "COMMAND_BUG", + "reason": "Command flow reaches deletion step and fails with 'not implemented' in current implementation.", + "issue": "implement-stack-delete" + }, + { + "id": "resource-precondition-volume-delete-volume-not-found", + "commandId": "volume delete", + "category": "RESOURCE_PRECONDITION", + "reason": "The requested volume placeholder does not exist in integration fixtures for the active stack state, causing deterministic not-found failure.", + "issue": "seed-known-resource-fixtures" + } +] diff --git a/src/test/integration/config/invocation-profiles.json b/src/test/integration/config/invocation-profiles.json new file mode 100644 index 000000000..3f3f330bf --- /dev/null +++ b/src/test/integration/config/invocation-profiles.json @@ -0,0 +1,218 @@ +[ + { + "id": "backup-create", + "match": { "exact": "backup create" }, + "requiredFlagDefaults": { + "expires": "30d" + }, + "notes": "Backup expiration must be set explicitly for deterministic runs." + }, + { + "id": "backup-schedule-create", + "match": { "exact": "backup schedule create" }, + "requiredFlagDefaults": { + "schedule": "0 * * * *", + "ttl": "7d" + } + }, + { + "id": "cronjob-create", + "match": { "exact": "cronjob create" }, + "requiredFlagDefaults": { + "description": "integration-cronjob", + "interval": "0 * * * *", + "url": "https://example.com/cronjob" + }, + "exactlyOneChoice": { + "command|url": "url" + } + }, + { + "id": "extension-install", + "match": { "exact": "extension install" }, + "requiredArgDefaults": { + "extension-id": "example-extension-id" + }, + "requiredFlagDefaults": { + "consent": true, + "project-id": "00000000-0000-4000-8000-000000000000" + }, + "exactlyOneChoice": { + "org-id|project-id": "project-id" + } + }, + { + "id": "extension-list-installed", + "match": { "exact": "extension list-installed" }, + "requiredFlagDefaults": { + "project-id": "00000000-0000-4000-8000-000000000000" + }, + "exactlyOneChoice": { + "org-id|project-id": "project-id" + } + }, + { + "id": "sftp-user-create", + "match": { "exact": "sftp-user create" }, + "requiredFlagDefaults": { + "description": "integration-sftp-user", + "directories": "/", + "password": "integration-password" + }, + "exactlyOneChoice": { + "password|public-key": "password" + } + }, + { + "id": "ssh-user-create", + "match": { "exact": "ssh-user create" }, + "requiredFlagDefaults": { + "description": "integration-ssh-user", + "password": "integration-password" + } + }, + { + "id": "database-mysql-create", + "match": { "exact": "database mysql create" }, + "requiredFlagDefaults": { + "description": "integration-mysql-db", + "version": "8.0", + "user-password": "integration-password" + } + }, + { + "id": "database-mysql-user-create", + "match": { "exact": "database mysql user create" }, + "requiredFlagDefaults": { + "database-id": "00000000-0000-4000-8000-000000000000", + "access-level": "full", + "description": "integration-mysql-user", + "password": "integration-password" + } + }, + { + "id": "database-mysql-shell", + "match": { "exact": "database mysql shell" }, + "interactivePolicy": "classify" + }, + { + "id": "app-database-link", + "match": { "exact": "app database link" }, + "requiredFlagDefaults": { + "database-id": "00000000-0000-4000-8000-000000000000", + "admin-user-id": "00000000-0000-4000-8000-000000000000", + "purpose": "primary" + } + }, + { + "id": "app-database-replace", + "match": { "exact": "app database replace" }, + "requiredFlagDefaults": { + "new-database-id": "00000000-0000-4000-8000-000000000000", + "admin-user-id": "00000000-0000-4000-8000-000000000000" + } + }, + { + "id": "org-invite", + "match": { "exact": "org invite" }, + "requiredFlagDefaults": { + "email": "integration@example.com" + } + }, + { + "id": "user-api-token-create", + "match": { "exact": "user api-token create" }, + "requiredFlagDefaults": { + "description": "integration-api-token", + "roles": "api_read" + } + }, + { + "id": "ssh-user-update", + "match": { "exact": "ssh-user update" }, + "requiredFlagDefaults": { + "password": "integration-password" + }, + "exactlyOneChoice": { + "password|public-key": "password" + } + }, + { + "id": "domain-dnszone-update", + "match": { "exact": "domain dnszone update" }, + "requiredArgDefaults": { + "record-set": "a" + }, + "requiredFlagDefaults": { + "record": "203.0.113.10" + } + }, + { + "id": "domain-get", + "match": { "exact": "domain get" }, + "requiredArgDefaults": { + "domain-id": "example.com" + } + }, + { + "id": "domain-dnszone-get", + "match": { "exact": "domain dnszone get" }, + "requiredArgDefaults": { + "dnszone-id": "example.com" + } + }, + { + "id": "domain-virtualhost-update", + "match": { "exact": "domain virtualhost update" }, + "requiredFlagDefaults": { + "path-to-url": "/:https://example.com" + } + }, + { + "id": "mail-address-update", + "match": { "exact": "mail address update" }, + "requiredArgDefaults": { + "mailaddress-id": "integration@example.com" + } + }, + { + "id": "mail-deliverybox-update", + "match": { "exact": "mail deliverybox update" }, + "requiredArgDefaults": { + "maildeliverybox-id": "00000000-0000-4000-8000-000000000000" + } + }, + { + "id": "ddev-init", + "match": { "exact": "ddev init" }, + "requiredFlagDefaults": { + "override-type": "auto", + "project-name": "integration-ddev" + } + }, + { + "id": "ddev-render-config", + "match": { "exact": "ddev render-config" }, + "requiredFlagDefaults": { + "override-type": "php" + } + }, + { + "id": "login-token", + "match": { "exact": "login token" }, + "interactivePolicy": "classify", + "disableExampleSource": true + }, + { + "id": "conversation-create", + "match": { "exact": "conversation create" }, + "interactivePolicy": "classify", + "disableExampleSource": true + }, + { + "id": "conversation-reply", + "match": { "exact": "conversation reply" }, + "interactivePolicy": "classify", + "disableExampleSource": true + } +] diff --git a/src/test/integration/config/loader.ts b/src/test/integration/config/loader.ts new file mode 100644 index 000000000..6337ef9b7 --- /dev/null +++ b/src/test/integration/config/loader.ts @@ -0,0 +1,276 @@ +import { readFileSync } from "node:fs"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import type { CommandWaiver, InvocationProfile, WaiverCategory } from "../command-discovery/types.js"; + +const CONFIG_DIR = path.dirname(fileURLToPath(import.meta.url)); +const INVOCATION_PROFILES_PATH = path.join(CONFIG_DIR, "invocation-profiles.json"); +const COMMAND_WAIVERS_PATH = path.join(CONFIG_DIR, "command-waivers.json"); + +const WAIVER_CATEGORIES: Set = new Set([ + "ARG_MISUSE", + "INTERACTIVE_REQUIRED", + "RESOURCE_PRECONDITION", + "CONTRACT_SHAPE", + "COMMAND_BUG", + "DEPRECATED_ENDPOINT", +]); + +let invocationProfilesCache: InvocationProfile[] | undefined; +let commandWaiversCache: CommandWaiver[] | undefined; + +export function loadInvocationProfiles(): InvocationProfile[] { + if (invocationProfilesCache) { + return invocationProfilesCache; + } + + const raw = readJsonFile(INVOCATION_PROFILES_PATH, "invocation profiles"); + if (!Array.isArray(raw)) { + throw new Error("[integration-config] invocation profiles must be an array."); + } + + invocationProfilesCache = raw.map((value, index) => + validateInvocationProfile(value, index), + ); + + return invocationProfilesCache; +} + +export function loadCommandWaivers(): CommandWaiver[] { + if (commandWaiversCache) { + return commandWaiversCache; + } + + const raw = readJsonFile(COMMAND_WAIVERS_PATH, "command waivers"); + if (!Array.isArray(raw)) { + throw new Error("[integration-config] command waivers must be an array."); + } + + const validated = raw.map((value, index) => validateCommandWaiver(value, index)); + + const ids = new Set(); + const commandIds = new Set(); + for (const waiver of validated) { + if (ids.has(waiver.id)) { + throw new Error(`[integration-config] duplicate waiver id '${waiver.id}'.`); + } + + if (commandIds.has(waiver.commandId)) { + throw new Error( + `[integration-config] duplicate waiver commandId '${waiver.commandId}'.`, + ); + } + + ids.add(waiver.id); + commandIds.add(waiver.commandId); + } + + commandWaiversCache = validated; + return commandWaiversCache; +} + +function readJsonFile(filePath: string, label: string): unknown { + try { + const content = readFileSync(filePath, "utf8"); + return JSON.parse(content); + } catch (error) { + throw new Error( + `[integration-config] failed to load ${label} at ${filePath}: ${(error as Error).message}`, + ); + } +} + +function validateInvocationProfile(value: unknown, index: number): InvocationProfile { + const record = asRecord(value, `invocation profile at index ${index}`); + const id = asNonEmptyString(record.id, `${profileLabel(index)}.id`); + + const matchRaw = asRecord(record.match, `${profileLabel(index)}.match`); + const exact = asOptionalString(matchRaw.exact, `${profileLabel(index)}.match.exact`); + const prefix = asOptionalString(matchRaw.prefix, `${profileLabel(index)}.match.prefix`); + if (!exact && !prefix) { + throw new Error( + `[integration-config] ${profileLabel(index)}.match requires 'exact' or 'prefix'.`, + ); + } + + const requiredFlagDefaults = asOptionalStringBooleanMap( + record.requiredFlagDefaults, + `${profileLabel(index)}.requiredFlagDefaults`, + ); + const requiredArgDefaults = asOptionalStringMap( + record.requiredArgDefaults, + `${profileLabel(index)}.requiredArgDefaults`, + ); + const exactlyOneChoice = asOptionalStringMap( + record.exactlyOneChoice, + `${profileLabel(index)}.exactlyOneChoice`, + ); + + const interactivePolicy = asOptionalInteractivePolicy( + record.interactivePolicy, + `${profileLabel(index)}.interactivePolicy`, + ); + + const disableExampleSource = asOptionalBoolean( + record.disableExampleSource, + `${profileLabel(index)}.disableExampleSource`, + ); + + const notes = asOptionalString(record.notes, `${profileLabel(index)}.notes`); + + return { + id, + match: { + ...(exact ? { exact } : {}), + ...(prefix ? { prefix } : {}), + }, + ...(requiredFlagDefaults ? { requiredFlagDefaults } : {}), + ...(requiredArgDefaults ? { requiredArgDefaults } : {}), + ...(exactlyOneChoice ? { exactlyOneChoice } : {}), + ...(interactivePolicy ? { interactivePolicy } : {}), + ...(disableExampleSource !== undefined ? { disableExampleSource } : {}), + ...(notes ? { notes } : {}), + }; +} + +function validateCommandWaiver(value: unknown, index: number): CommandWaiver { + const record = asRecord(value, `command waiver at index ${index}`); + const id = asNonEmptyString(record.id, `${waiverLabel(index)}.id`); + const commandId = asNonEmptyString( + record.commandId, + `${waiverLabel(index)}.commandId`, + ); + const category = asNonEmptyString( + record.category, + `${waiverLabel(index)}.category`, + ) as WaiverCategory; + + if (!WAIVER_CATEGORIES.has(category)) { + throw new Error( + `[integration-config] ${waiverLabel(index)}.category must be one of ${[ + ...WAIVER_CATEGORIES, + ].join(", ")}.`, + ); + } + + const reason = asNonEmptyString(record.reason, `${waiverLabel(index)}.reason`); + const issue = asOptionalString(record.issue, `${waiverLabel(index)}.issue`); + const expiresOn = asOptionalString(record.expiresOn, `${waiverLabel(index)}.expiresOn`); + + return { + id, + commandId, + category, + reason, + ...(issue ? { issue } : {}), + ...(expiresOn ? { expiresOn } : {}), + }; +} + +function asRecord(value: unknown, label: string): Record { + if (!value || typeof value !== "object" || Array.isArray(value)) { + throw new Error(`[integration-config] ${label} must be an object.`); + } + + return value as Record; +} + +function asNonEmptyString(value: unknown, label: string): string { + if (typeof value !== "string" || value.trim().length === 0) { + throw new Error(`[integration-config] ${label} must be a non-empty string.`); + } + + return value.trim(); +} + +function asOptionalString(value: unknown, label: string): string | undefined { + if (value === undefined) { + return undefined; + } + + if (typeof value !== "string") { + throw new Error(`[integration-config] ${label} must be a string when provided.`); + } + + return value; +} + +function asOptionalBoolean(value: unknown, label: string): boolean | undefined { + if (value === undefined) { + return undefined; + } + + if (typeof value !== "boolean") { + throw new Error(`[integration-config] ${label} must be a boolean when provided.`); + } + + return value; +} + +function asOptionalInteractivePolicy( + value: unknown, + label: string, +): "resolve" | "classify" | undefined { + if (value === undefined) { + return undefined; + } + + if (value !== "resolve" && value !== "classify") { + throw new Error( + `[integration-config] ${label} must be 'resolve' or 'classify' when provided.`, + ); + } + + return value; +} + +function asOptionalStringMap( + value: unknown, + label: string, +): Record | undefined { + if (value === undefined) { + return undefined; + } + + const record = asRecord(value, label); + const result: Record = {}; + for (const [key, entry] of Object.entries(record)) { + if (typeof entry !== "string") { + throw new Error(`[integration-config] ${label}.${key} must be a string.`); + } + + result[key] = entry; + } + + return result; +} + +function asOptionalStringBooleanMap( + value: unknown, + label: string, +): Record | undefined { + if (value === undefined) { + return undefined; + } + + const record = asRecord(value, label); + const result: Record = {}; + + for (const [key, entry] of Object.entries(record)) { + if (typeof entry !== "string" && typeof entry !== "boolean") { + throw new Error(`[integration-config] ${label}.${key} must be a string or boolean.`); + } + + result[key] = entry; + } + + return result; +} + +function profileLabel(index: number): string { + return `invocation-profiles[${index}]`; +} + +function waiverLabel(index: number): string { + return `command-waivers[${index}]`; +} diff --git a/src/test/integration/env.ts b/src/test/integration/env.ts new file mode 100644 index 000000000..3113fdbdc --- /dev/null +++ b/src/test/integration/env.ts @@ -0,0 +1,32 @@ +export type EnvSnapshot = NodeJS.ProcessEnv; + +export function snapshotEnv(): EnvSnapshot { + return { ...process.env }; +} + +export function restoreEnv(snapshot: EnvSnapshot): void { + process.env = snapshot; +} + +export function requireIntegrationEnv( + envVars: string[], + context: string, +): void { + const missing = envVars.filter((envVar) => { + const value = process.env[envVar]; + return value === undefined || value.trim() === ""; + }); + + if (missing.length === 0) { + return; + } + + throw new Error( + `[integration:${context}] Missing required environment variables: ${missing.join(", ")}. ` + + "Set them before running this test.", + ); +} + +export function configureIntegrationEnv(context: string): void { + requireIntegrationEnv(["MITTWALD_API_TOKEN", "MITTWALD_API_BASE_URL"], context); +} diff --git a/src/test/integration/run-all-commands.test.ts b/src/test/integration/run-all-commands.test.ts new file mode 100644 index 000000000..7f7f27941 --- /dev/null +++ b/src/test/integration/run-all-commands.test.ts @@ -0,0 +1,594 @@ +import { afterEach, beforeEach, describe, expect, it, jest } from "@jest/globals"; +import { appendFile, mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { + buildClassificationCatalogFromBuckets, + parseFailureCategory, + saveClassificationCatalog, +} from "./classification-catalog.js"; +import { runDevCommand } from "./command.js"; +import { discoverRunnableCommands } from "./command-discovery.js"; +import type { CommandWaiver, WaiverCategory } from "./command-discovery/types.js"; +import { loadCommandWaivers } from "./config/loader.js"; +import { configureIntegrationEnv, requireIntegrationEnv, restoreEnv, snapshotEnv } from "./env.js"; + +jest.setTimeout(20 * 60 * 1000); + +type FailureCategory = WaiverCategory; + +type MachineLogEntry = Record; + +const FAILURE_CATEGORIES: FailureCategory[] = [ + "ARG_MISUSE", + "INTERACTIVE_REQUIRED", + "RESOURCE_PRECONDITION", + "CONTRACT_SHAPE", + "COMMAND_BUG", + "DEPRECATED_ENDPOINT", +]; + +function createFailureBuckets(): Record { + return { + ARG_MISUSE: [], + INTERACTIVE_REQUIRED: [], + RESOURCE_PRECONDITION: [], + CONTRACT_SHAPE: [], + COMMAND_BUG: [], + DEPRECATED_ENDPOINT: [], + }; +} + +function classifyFailure(output: { stderr: string; stdout: string }): FailureCategory { + const text = `${output.stderr}\n${output.stdout}`.toLowerCase(); + + if ( + /missing\s+(?:\d+\s+)?required arg|missing\s+(?:\d+\s+)?required flag|exactly one of|required options|unexpected argument|unknown flag|nonexistent flag|invalid flag|flag .* expects|no .* id given|you need to specify at least one/i.test( + text, + ) + ) { + return "ARG_MISUSE"; + } + + if ( + /prompt|interactive|addinput|addselect|addconfirmation|overwrite\?|token file already exists|tty/i.test( + text, + ) + ) { + return "INTERACTIVE_REQUIRED"; + } + + if ( + /not found|does not exist|no .* found|resource.*missing|404|forbidden|unauthorized|no project found|failed to connect|could not resolve hostname|name or service not known|no main user found|main mysql user can not be deleted manually/i.test( + text, + ) + ) { + return "RESOURCE_PRECONDITION"; + } + + if ( + /invalid version|not iterable|cannot read properties|undefined.*data|validation|invalid type|schema/i.test( + text, + ) + ) { + return "CONTRACT_SHAPE"; + } + + return "COMMAND_BUG"; +} + +function parseInvocationPartsFromArgs( + args: string[], + commandTokenCount: number, +): { positionalValues: string[]; flagValues: Map } { + const positionalValues: string[] = []; + const flagValues = new Map(); + + const invocationArgs = args.slice(commandTokenCount); + for (let i = 0; i < invocationArgs.length; i += 1) { + const token = invocationArgs[i]; + if (!token.startsWith("--")) { + positionalValues.push(token); + continue; + } + + const withoutPrefix = token.slice(2); + const eqIndex = withoutPrefix.indexOf("="); + let name = withoutPrefix; + let value: string | undefined; + + if (eqIndex >= 0) { + name = withoutPrefix.slice(0, eqIndex); + value = withoutPrefix.slice(eqIndex + 1); + } else { + const nextToken = invocationArgs[i + 1]; + if (nextToken && !nextToken.startsWith("--")) { + value = nextToken; + i += 1; + } + } + + const values = flagValues.get(name) ?? []; + values.push(value ?? "true"); + flagValues.set(name, values); + } + + return { positionalValues, flagValues }; +} + +function validateInvocationCompleteness(command: Awaited>[number]): string[] { + const issues: string[] = []; + const { positionalValues, flagValues } = parseInvocationPartsFromArgs( + command.synthesizedInvocation.args, + command.commandTokens.length, + ); + + command.parsedArgs.forEach((arg, index) => { + if (!arg.required) { + return; + } + if (positionalValues[index] === undefined) { + issues.push(`missing required arg ${arg.name}`); + } + }); + + for (const flag of command.parsedFlags) { + if (flag.required && !flagValues.has(flag.name)) { + issues.push(`missing required flag --${flag.name}`); + } + } + + const exactlyOneGroups = new Map(); + for (const flag of command.parsedFlags) { + if (!flag.exactlyOne || flag.exactlyOne.length < 2) { + continue; + } + const members = [...new Set(flag.exactlyOne)].sort(); + exactlyOneGroups.set(members.join("|"), members); + } + + for (const members of exactlyOneGroups.values()) { + const selected = members.filter((member) => flagValues.has(member)); + if (selected.length !== 1) { + issues.push(`exactly-one unresolved [${members.join(",")}]`); + } + } + + return issues; +} + +function logFailureTaxonomySummary( + failuresByCategory: Record, +): void { + logProgress("[run-all] failure taxonomy summary:"); + + for (const category of FAILURE_CATEGORIES) { + const commands = failuresByCategory[category]; + const sample = commands.slice(0, 5).join(", "); + logProgress( + `[run-all] ${category.padEnd(22, " ")} count=${String(commands.length).padStart(3, " ")} sample=${sample || "-"}`, + ); + } +} + +function mapCommandWaivers(waivers: CommandWaiver[]): { + waiversByCommandId: Map; + duplicates: string[]; +} { + const waiversByCommandId = new Map(); + const duplicates: string[] = []; + + for (const waiver of waivers) { + if (waiversByCommandId.has(waiver.commandId)) { + duplicates.push(waiver.commandId); + continue; + } + + waiversByCommandId.set(waiver.commandId, waiver); + } + + return { waiversByCommandId, duplicates }; +} + +function logWaiverSummary(waivedByCategory: Record): void { + logProgress("[run-all] waiver summary:"); + + for (const category of FAILURE_CATEGORIES) { + const commands = waivedByCategory[category]; + const sample = commands.slice(0, 5).join(", "); + logProgress( + `[run-all] ${category.padEnd(22, " ")} count=${String(commands.length).padStart(3, " ")} sample=${sample || "-"}`, + ); + } +} + +function logProgress(message: string): void { + process.stderr.write(`${message}\n`); +} + +function formatOutputBlock(output: string): string { + const trimmed = output.trim(); + return trimmed.length > 0 ? trimmed : ""; +} + +function logCommandFailureOutput( + position: string, + commandId: string, + result: { stdout: string; stderr: string }, +): void { + logProgress(`[${position}] diagnostics ${commandId}: stderr >>>`); + logProgress(formatOutputBlock(result.stderr)); + logProgress(`[${position}] diagnostics ${commandId}: stdout >>>`); + logProgress(formatOutputBlock(result.stdout)); + logProgress(`[${position}] diagnostics ${commandId}: <<<`); +} + +async function initializeMachineLogFile(filePath: string): Promise { + await mkdir(path.dirname(filePath), { recursive: true }); + await writeFile(filePath, "", "utf-8"); +} + +async function appendMachineLogEntry( + filePath: string, + entry: MachineLogEntry, +): Promise { + const line = JSON.stringify({ + timestamp: new Date().toISOString(), + ...entry, + }); + await appendFile(filePath, `${line}\n`, "utf-8"); +} + +async function seedProjectContext(projectId: string): Promise { + const configDir = process.env.MW_CONFIG_DIR; + + if (!configDir) { + throw new Error( + "[integration:run-all-commands] MW_CONFIG_DIR was not set before seeding project context.", + ); + } + + const contextFile = path.join(configDir, "context.json"); + + await mkdir(configDir, { recursive: true }); + await writeFile( + contextFile, + JSON.stringify({ + "project-id": projectId, + "server-id": "6b4f48f5-d80c-4d20-9db8-fecf4c9e6221", + "installation-id": "f7b47c12-7d11-4f3a-b9bc-1b3c706e1d55", + "org-id": "88e8d927-7db4-42ef-ae02-f8a7ef0b4d77", + }), + "utf-8", + ); +} + +describe("integration: run all commands", () => { + let originalEnv: NodeJS.ProcessEnv; + let tempConfigDir: string; + + beforeEach(async () => { + originalEnv = snapshotEnv(); + tempConfigDir = await mkdtemp(path.join(tmpdir(), "mw-int-config-")); + process.env.MW_CONFIG_DIR = tempConfigDir; + }); + + afterEach(async () => { + restoreEnv(originalEnv); + await rm(tempConfigDir, { recursive: true, force: true }); + }); + + it("discovers and executes every command once", async () => { + configureIntegrationEnv("run-all-commands"); + requireIntegrationEnv(["MW_TEST_PROJECT_ID"], "run-all-commands"); + + const categoryFilterRaw = process.env.MW_TEST_CATEGORY?.trim(); + const categoryFilter = categoryFilterRaw + ? parseFailureCategory(categoryFilterRaw) + : undefined; + const classificationCatalogPath = + process.env.MW_TEST_CLASSIFICATION_CATALOG_PATH?.trim() || undefined; + const machineLogPath = + process.env.MW_TEST_MACHINE_LOG_PATH?.trim() || + path.resolve("run-all-commands.ndjson"); + + await initializeMachineLogFile(machineLogPath); + logProgress(`[run-all] machine log path=${machineLogPath}`); + + const projectId = process.env.MW_TEST_PROJECT_ID!.trim(); + await seedProjectContext(projectId); + logProgress( + `[run-all] using context project-id from MW_TEST_PROJECT_ID (${projectId}); MW_CONFIG_DIR=${process.env.MW_CONFIG_DIR}`, + ); + + logProgress("[run-all] starting command discovery"); + const commands = await discoverRunnableCommands({ + onProgress: logProgress, + categoryFilter, + classificationCatalogPath, + }); + expect(commands.length).toBeGreaterThan(0); + + if (categoryFilter) { + logProgress( + `[run-all] category filter active: ${categoryFilter}${classificationCatalogPath ? ` (catalog=${classificationCatalogPath})` : ""}`, + ); + } + + const waivers = loadCommandWaivers(); + const { waiversByCommandId, duplicates } = mapCommandWaivers(waivers); + + await appendMachineLogEntry(machineLogPath, { + event: "run-start", + categoryFilter: categoryFilter ?? null, + classificationCatalogPath: classificationCatalogPath ?? null, + projectId, + commandCount: commands.length, + waiverCount: waivers.length, + }); + + logProgress(`[run-all] discovered ${commands.length} commands to execute`); + logProgress(`[run-all] loaded ${waivers.length} waiver entries`); + + const staleExampleCommands = commands.filter( + (command) => command.synthesizedInvocation.staleExample, + ); + const extractionDiagnostics = commands + .flatMap((command) => command.extractionDiagnostics) + .length; + logProgress( + `[run-all] stale examples detected=${staleExampleCommands.length}; extraction diagnostics=${extractionDiagnostics}`, + ); + + const infrastructureFailures: string[] = []; + const failuresByCategory = createFailureBuckets(); + const waivedByCategory = createFailureBuckets(); + let successfulCommands = 0; + let failedCommands = 0; + let waivedSkippedCommands = 0; + + if (!categoryFilter) { + if (duplicates.length > 0) { + infrastructureFailures.push( + `[waivers] duplicate waiver commandId entries: ${duplicates.join(", ")}`, + ); + } + + const discoveredCommandIds = new Set(commands.map((command) => command.commandId)); + for (const waiver of waivers) { + if (!discoveredCommandIds.has(waiver.commandId)) { + infrastructureFailures.push( + `[waivers] command '${waiver.commandId}' has a waiver but is not part of current discovery output`, + ); + } + } + } else { + logProgress("[waivers] strict waiver integrity checks skipped (category filter active)"); + } + + for (const [index, command] of commands.entries()) { + const position = `${index + 1}/${commands.length}`; + const invocation = command.synthesizedInvocation; + const waiver = waiversByCommandId.get(command.commandId); + const commandStartedAt = Date.now(); + + await seedProjectContext(projectId); + logProgress( + `[${position}] running ${command.commandId} (source=${invocation.argumentSource}; interactive=${invocation.interactiveDecision}; re-seeded project context)`, + ); + + await appendMachineLogEntry(machineLogPath, { + event: "command-start", + index: index + 1, + total: commands.length, + position, + commandId: command.commandId, + sourceFile: command.sourceFile, + commandTokens: command.commandTokens, + parsedArgs: command.parsedArgs, + parsedFlags: command.parsedFlags, + interactiveSignals: command.interactiveSignals, + invocationProfilesApplied: command.invocationProfilesApplied, + extractionDiagnostics: command.extractionDiagnostics, + invocationArgs: invocation.args, + argumentSource: invocation.argumentSource, + interactiveDecision: invocation.interactiveDecision, + }); + + if (waiver) { + waivedSkippedCommands += 1; + waivedByCategory[waiver.category].push(command.commandId); + logProgress( + `[${position}] waived ${command.commandId} (category=${waiver.category}; reason=${waiver.reason}${waiver.issue ? `; issue=${waiver.issue}` : ""})`, + ); + await appendMachineLogEntry(machineLogPath, { + event: "command-result", + index: index + 1, + total: commands.length, + position, + commandId: command.commandId, + status: "waived", + durationMs: Date.now() - commandStartedAt, + waiver, + }); + continue; + } + + if (invocation.interactiveDecision === "INTERACTIVE_REQUIRED") { + failedCommands += 1; + failuresByCategory.INTERACTIVE_REQUIRED.push(command.commandId); + infrastructureFailures.push( + `[waivers] ${command.commandId} was classified INTERACTIVE_REQUIRED but has no waiver entry`, + ); + logProgress( + `[${position}] classified ${command.commandId} as INTERACTIVE_REQUIRED (missing waiver entry)`, + ); + await appendMachineLogEntry(machineLogPath, { + event: "command-result", + index: index + 1, + total: commands.length, + position, + commandId: command.commandId, + status: "failed", + failureCategory: "INTERACTIVE_REQUIRED", + durationMs: Date.now() - commandStartedAt, + details: "classified INTERACTIVE_REQUIRED but no waiver entry exists", + }); + continue; + } + + const staticInvocationIssues = validateInvocationCompleteness(command); + if (staticInvocationIssues.length > 0) { + failedCommands += 1; + failuresByCategory.ARG_MISUSE.push(command.commandId); + logProgress( + `[${position}] preflight ${command.commandId} classified as ARG_MISUSE (${staticInvocationIssues.join("; ")})`, + ); + await appendMachineLogEntry(machineLogPath, { + event: "command-result", + index: index + 1, + total: commands.length, + position, + commandId: command.commandId, + status: "failed", + failureCategory: "ARG_MISUSE", + durationMs: Date.now() - commandStartedAt, + preflightIssues: staticInvocationIssues, + }); + continue; + } + + const result = await runDevCommand(invocation.args, { + timeoutMs: 30_000, + }); + + if (result.timedOut) { + failedCommands += 1; + failuresByCategory.COMMAND_BUG.push(command.commandId); + logProgress(`[${position}] timeout ${command.commandId}`); + logCommandFailureOutput(position, command.commandId, result); + await appendMachineLogEntry(machineLogPath, { + event: "command-result", + index: index + 1, + total: commands.length, + position, + commandId: command.commandId, + status: "failed", + failureCategory: "COMMAND_BUG", + durationMs: Date.now() - commandStartedAt, + timedOut: true, + stdout: result.stdout, + stderr: result.stderr, + }); + continue; + } + + if (result.exitCode === null) { + failedCommands += 1; + infrastructureFailures.push( + `${command.commandId} failed to execute (source=${invocation.argumentSource}): ${result.error?.message ?? "unknown error"}`, + ); + logProgress( + `[${position}] spawn-error ${command.commandId}: ${result.error?.message ?? "unknown error"}`, + ); + logCommandFailureOutput(position, command.commandId, result); + await appendMachineLogEntry(machineLogPath, { + event: "command-result", + index: index + 1, + total: commands.length, + position, + commandId: command.commandId, + status: "spawn-error", + durationMs: Date.now() - commandStartedAt, + errorMessage: result.error?.message ?? "unknown error", + stdout: result.stdout, + stderr: result.stderr, + }); + continue; + } + + if (result.exitCode !== 0) { + failedCommands += 1; + const category = classifyFailure(result); + failuresByCategory[category].push(command.commandId); + logProgress( + `[${position}] classified ${command.commandId} as ${category}`, + ); + logCommandFailureOutput(position, command.commandId, result); + + await appendMachineLogEntry(machineLogPath, { + event: "command-result", + index: index + 1, + total: commands.length, + position, + commandId: command.commandId, + status: "failed", + failureCategory: category, + durationMs: Date.now() - commandStartedAt, + exitCode: result.exitCode, + stdout: result.stdout, + stderr: result.stderr, + }); + } else { + successfulCommands += 1; + + await appendMachineLogEntry(machineLogPath, { + event: "command-result", + index: index + 1, + total: commands.length, + position, + commandId: command.commandId, + status: "succeeded", + durationMs: Date.now() - commandStartedAt, + exitCode: result.exitCode, + }); + } + + logProgress( + `[${position}] finished ${command.commandId} (exitCode=${result.exitCode})`, + ); + } + + logProgress(`[run-all] execution complete: ${commands.length} run`); + logProgress( + `[run-all] statistics: successful=${successfulCommands}, failed=${failedCommands}, waived-skipped=${waivedSkippedCommands}, total=${commands.length}`, + ); + logFailureTaxonomySummary(failuresByCategory); + logWaiverSummary(waivedByCategory); + + await appendMachineLogEntry(machineLogPath, { + event: "run-summary", + statistics: { + successful: successfulCommands, + failed: failedCommands, + waivedSkipped: waivedSkippedCommands, + total: commands.length, + }, + failuresByCategory, + waivedByCategory, + infrastructureFailures, + }); + + if (!categoryFilter) { + const classificationCatalog = buildClassificationCatalogFromBuckets({ + failuresByCategory, + waivedByCategory, + statistics: { + successful: successfulCommands, + failed: failedCommands, + waivedSkipped: waivedSkippedCommands, + total: commands.length, + }, + }); + + await saveClassificationCatalog(classificationCatalog); + logProgress( + `[run-all] wrote classification catalog with ${classificationCatalog.entries.length} entries`, + ); + } else { + logProgress("[run-all] skipped classification catalog write (category filter active)"); + } + + expect(infrastructureFailures).toEqual([]); + }); +}); diff --git a/src/test/integration/tools/generate-command-endpoint-map.ts b/src/test/integration/tools/generate-command-endpoint-map.ts new file mode 100644 index 000000000..d417a1b23 --- /dev/null +++ b/src/test/integration/tools/generate-command-endpoint-map.ts @@ -0,0 +1,1137 @@ +import fs from "node:fs"; +import path from "node:path"; +import ts from "typescript"; + +type CommandReference = { + commandId: string; + sourceFile: string; +}; + +type FailureCategory = + | "ARG_MISUSE" + | "INTERACTIVE_REQUIRED" + | "RESOURCE_PRECONDITION" + | "CONTRACT_SHAPE" + | "COMMAND_BUG" + | "DEPRECATED_ENDPOINT"; + +type CliOptions = { + machineLogPath?: string; + category?: FailureCategory; + openapiPath: string; + outputJsonPath: string; + outputMarkdownPath: string; +}; + +type NdjsonRecord = { + timestamp?: string; + event?: string; + commandId?: string; + sourceFile?: string; + status?: string; + failureCategory?: FailureCategory; +}; + +type CommandLogRecord = { + status?: string; + failureCategory?: FailureCategory; +}; + +type ApiCallUsage = { + group: string; + method: string; + groupMethod: string; + filePath: string; +}; + +type DescriptorMeta = { + descriptorName: string; + path: string | null; + httpMethod: string | null; + operationId: string | null; +}; + +type OpenApiOperation = { + operationId: string | null; + deprecated: boolean; +}; + +type ResolvedEndpoint = { + groupMethod: string; + descriptorName: string | null; + descriptorPath: string | null; + descriptorHttpMethod: string | null; + descriptorOperationId: string | null; + openapiOperationId: string | null; + openapiDeprecated: boolean | null; + openapiStatus: "FOUND" | "MISSING_PATH" | "MISSING_METHOD" | "MISSING_DESCRIPTOR"; +}; + +type CommandMappingEntry = { + commandId: string; + sourceFile: string; + transitiveFiles: string[]; + logStatus: string | null; + logCategory: FailureCategory | null; + apiCalls: ApiCallUsage[]; + resolvedEndpoints: ResolvedEndpoint[]; + unresolvedGroupMethods: string[]; +}; + +type MappingOutput = { + generatedAt: string; + inputs: { + machineLogPath: string | null; + category: FailureCategory | null; + openapiPath: string; + }; + statistics: { + commandCount: number; + commandWithApiCalls: number; + unresolvedGroupMethodCount: number; + deprecatedEndpointCount: number; + }; + entries: CommandMappingEntry[]; +}; + +type FileImportBinding = { + sourceFilePath: string; + importedName: string; +}; + +type FunctionInfo = { + localCalls: Set; + importedCalls: Map; + apiCalls: ApiCallUsage[]; +}; + +type FileAnalysis = { + imports: Map; + localFunctions: Map; + exports: Map; + functionInfos: Map; + rootInfo: FunctionInfo; +}; + +type TraversalState = { + visitedFiles: Set; + visitedFunctions: Set; + apiCalls: ApiCallUsage[]; +}; + +const DEFAULT_OPENAPI_PATH = "openapi.json"; +const DEFAULT_OUTPUT_JSON_PATH = "command-endpoint-map.json"; +const DEFAULT_OUTPUT_MARKDOWN_PATH = "command-endpoint-map.md"; +const DEFAULT_MACHINE_LOG_PATH = "run-all-commands.ndjson"; + +function parseCliOptions(argv: string[]): CliOptions { + const options: CliOptions = { + openapiPath: DEFAULT_OPENAPI_PATH, + outputJsonPath: DEFAULT_OUTPUT_JSON_PATH, + outputMarkdownPath: DEFAULT_OUTPUT_MARKDOWN_PATH, + }; + + for (let i = 2; i < argv.length; i += 1) { + const token = argv[i]; + + if (token === "--machine-log") { + options.machineLogPath = requireNextArg(argv, i, token); + i += 1; + continue; + } + + if (token === "--category") { + const raw = requireNextArg(argv, i, token); + options.category = parseFailureCategory(raw); + i += 1; + continue; + } + + if (token === "--openapi") { + options.openapiPath = requireNextArg(argv, i, token); + i += 1; + continue; + } + + if (token === "--output-json") { + options.outputJsonPath = requireNextArg(argv, i, token); + i += 1; + continue; + } + + if (token === "--output-md") { + options.outputMarkdownPath = requireNextArg(argv, i, token); + i += 1; + continue; + } + + if (token === "--help" || token === "-h") { + printHelp(); + process.exit(0); + } + + throw new Error(`Unknown argument: ${token}`); + } + + return options; +} + +function requireNextArg(argv: string[], index: number, token: string): string { + const value = argv[index + 1]; + if (!value) { + throw new Error(`Missing value for ${token}`); + } + return value; +} + +function parseFailureCategory(value: string): FailureCategory { + const categories: FailureCategory[] = [ + "ARG_MISUSE", + "INTERACTIVE_REQUIRED", + "RESOURCE_PRECONDITION", + "CONTRACT_SHAPE", + "COMMAND_BUG", + "DEPRECATED_ENDPOINT", + ]; + + if (!categories.includes(value as FailureCategory)) { + throw new Error(`Invalid category '${value}'. Expected one of ${categories.join(", ")}`); + } + + return value as FailureCategory; +} + +function printHelp(): void { + process.stdout.write(`Usage:\n` + + ` yarn tool:integration:generate-command-endpoint-map [options]\n\n` + + `Options:\n` + + ` --machine-log NDJSON log from run-all integration test (default: ${DEFAULT_MACHINE_LOG_PATH})\n` + + ` --category Optional failure category filter\n` + + ` --openapi OpenAPI JSON file (default: ${DEFAULT_OPENAPI_PATH})\n` + + ` --output-json Output JSON mapping (default: ${DEFAULT_OUTPUT_JSON_PATH})\n` + + ` --output-md Output markdown summary (default: ${DEFAULT_OUTPUT_MARKDOWN_PATH})\n` + + ` -h, --help Show this help\n`); +} + +async function main(): Promise { + const options = parseCliOptions(process.argv); + const openapiPath = path.resolve(options.openapiPath); + const outputJsonPath = path.resolve(options.outputJsonPath); + const outputMarkdownPath = path.resolve(options.outputMarkdownPath); + + const machineLogPath = path.resolve( + options.machineLogPath ?? DEFAULT_MACHINE_LOG_PATH, + ); + + if (!fs.existsSync(machineLogPath)) { + throw new Error( + `Machine log not found at ${machineLogPath}. Run the integration command runner first to produce command-start and command-result events.`, + ); + } + + const machineLogData = loadMachineLogData(machineLogPath); + + const filteredCommands = filterCommands( + machineLogData.commands, + machineLogData.commandLogById, + options.category, + ); + + const groupMethodToDescriptor = buildGroupMethodToDescriptorIndex(); + const descriptorMetaByName = buildDescriptorMetaIndex(); + const openapi = JSON.parse(fs.readFileSync(openapiPath, "utf8")) as { + paths?: Record>; + }; + + const entries = filteredCommands.map((command) => { + const sourceAbsPath = path.resolve(process.cwd(), "src/commands", command.sourceFile); + const analysis = analyzeCommandTransitive(sourceAbsPath); + const uniqueApiCalls = deduplicateApiCalls(analysis.apiCalls); + + const resolvedEndpoints = resolveEndpoints( + uniqueApiCalls, + groupMethodToDescriptor, + descriptorMetaByName, + openapi, + ); + + const unresolvedGroupMethods = resolvedEndpoints + .filter((endpoint) => endpoint.openapiStatus === "MISSING_DESCRIPTOR") + .map((endpoint) => endpoint.groupMethod); + + const logRecord = machineLogData.commandLogById.get(command.commandId); + + return { + commandId: command.commandId, + sourceFile: command.sourceFile, + transitiveFiles: Array.from(analysis.visitedFiles) + .map((filePath) => path.relative(process.cwd(), filePath)) + .sort((a, b) => a.localeCompare(b)), + logStatus: logRecord?.status ?? null, + logCategory: logRecord?.failureCategory ?? null, + apiCalls: uniqueApiCalls + .map((call) => ({ + ...call, + filePath: path.relative(process.cwd(), call.filePath), + })) + .sort((a, b) => { + const methodCmp = a.groupMethod.localeCompare(b.groupMethod); + return methodCmp !== 0 ? methodCmp : a.filePath.localeCompare(b.filePath); + }), + resolvedEndpoints, + unresolvedGroupMethods, + } satisfies CommandMappingEntry; + }); + + const output: MappingOutput = { + generatedAt: new Date().toISOString(), + inputs: { + machineLogPath: fs.existsSync(machineLogPath) + ? path.relative(process.cwd(), machineLogPath) + : null, + category: options.category ?? null, + openapiPath: path.relative(process.cwd(), openapiPath), + }, + statistics: { + commandCount: entries.length, + commandWithApiCalls: entries.filter((entry) => entry.apiCalls.length > 0).length, + unresolvedGroupMethodCount: entries.reduce( + (sum, entry) => sum + entry.unresolvedGroupMethods.length, + 0, + ), + deprecatedEndpointCount: entries.reduce( + (sum, entry) => + sum + + entry.resolvedEndpoints.filter((endpoint) => endpoint.openapiDeprecated === true) + .length, + 0, + ), + }, + entries, + }; + + fs.writeFileSync(outputJsonPath, `${JSON.stringify(output, null, 2)}\n`, "utf8"); + fs.writeFileSync(outputMarkdownPath, renderMarkdown(output), "utf8"); + + process.stdout.write( + `Wrote ${path.relative(process.cwd(), outputJsonPath)} and ${path.relative(process.cwd(), outputMarkdownPath)} for ${entries.length} commands.\n`, + ); +} + +function loadMachineLogData(machineLogPath: string): { + commands: CommandReference[]; + commandLogById: Map; +} { + const lines = fs + .readFileSync(machineLogPath, "utf8") + .split("\n") + .map((line) => line.trim()) + .filter((line) => line.length > 0); + + const commandLogById = new Map(); + const commandById = new Map(); + + for (let idx = 0; idx < lines.length; idx += 1) { + const line = lines[idx]; + let parsed: NdjsonRecord; + + try { + parsed = JSON.parse(line) as NdjsonRecord; + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + throw new Error(`Invalid NDJSON at ${machineLogPath}:${idx + 1}: ${message}`); + } + + if (parsed.event === "command-start") { + if (typeof parsed.commandId !== "string" || typeof parsed.sourceFile !== "string") { + continue; + } + + if (!commandById.has(parsed.commandId)) { + commandById.set(parsed.commandId, { + commandId: parsed.commandId, + sourceFile: parsed.sourceFile, + }); + } + continue; + } + + if (parsed.event === "command-result") { + if (typeof parsed.commandId !== "string") { + continue; + } + + commandLogById.set(parsed.commandId, { + status: parsed.status, + failureCategory: parsed.failureCategory, + }); + } + } + + const commands = Array.from(commandById.values()).sort((a, b) => + a.commandId.localeCompare(b.commandId), + ); + + if (commands.length === 0) { + throw new Error( + `No command-start entries with sourceFile found in ${machineLogPath}. Ensure run-all integration test writes discovery metadata to the machine log.`, + ); + } + + return { + commands, + commandLogById, + }; +} + +function filterCommands( + commands: CommandReference[], + commandLogById: Map, + category: FailureCategory | undefined, +): CommandReference[] { + if (!category) { + return commands; + } + + return commands.filter((command) => { + const record = commandLogById.get(command.commandId); + return record?.failureCategory === category; + }); +} + +function buildGroupMethodToDescriptorIndex(): Map { + const clientPath = path.resolve( + process.cwd(), + "node_modules/@mittwald/api-client/dist/esm/generated/v2/client.js", + ); + const sourceText = fs.readFileSync(clientPath, "utf8"); + const sourceFile = ts.createSourceFile( + clientPath, + sourceText, + ts.ScriptTarget.Latest, + true, + ts.ScriptKind.JS, + ); + + const index = new Map(); + + const visit = (node: ts.Node): void => { + if (ts.isPropertyAssignment(node) && ts.isIdentifier(node.name)) { + const methodName = node.name.text; + const initializer = node.initializer; + + if (ts.isCallExpression(initializer)) { + const maybeRequestFactory = initializer.expression; + if ( + ts.isPropertyAccessExpression(maybeRequestFactory) && + maybeRequestFactory.name.text === "requestFunctionFactory" && + initializer.arguments.length === 1 + ) { + const arg = initializer.arguments[0]; + if ( + ts.isPropertyAccessExpression(arg) && + ts.isIdentifier(arg.expression) && + arg.expression.text === "descriptors" + ) { + const descriptorName = arg.name.text; + const groupName = getEnclosingGroupName(node); + if (groupName) { + index.set(`${groupName}.${methodName}`, descriptorName); + } + } + } + } + } + + ts.forEachChild(node, visit); + }; + + visit(sourceFile); + return index; +} + +function getEnclosingGroupName(node: ts.Node): string | null { + const objectLiteral = node.parent; + if (!ts.isObjectLiteralExpression(objectLiteral)) { + return null; + } + + const parent = objectLiteral.parent; + + if (ts.isPropertyAssignment(parent) && ts.isIdentifier(parent.name)) { + return parent.name.text; + } + + if (ts.isPropertyDeclaration(parent) && ts.isIdentifier(parent.name)) { + return parent.name.text; + } + + return null; +} + +function buildDescriptorMetaIndex(): Map { + const descriptorsPath = path.resolve( + process.cwd(), + "node_modules/@mittwald/api-client/dist/esm/generated/v2/descriptors.js", + ); + + const sourceText = fs.readFileSync(descriptorsPath, "utf8"); + const sourceFile = ts.createSourceFile( + descriptorsPath, + sourceText, + ts.ScriptTarget.Latest, + true, + ts.ScriptKind.JS, + ); + + const map = new Map(); + + for (const statement of sourceFile.statements) { + if (!ts.isVariableStatement(statement)) { + continue; + } + + const hasExport = statement.modifiers?.some( + (modifier) => modifier.kind === ts.SyntaxKind.ExportKeyword, + ); + if (!hasExport) { + continue; + } + + for (const decl of statement.declarationList.declarations) { + if (!ts.isIdentifier(decl.name) || !decl.initializer) { + continue; + } + + const descriptorName = decl.name.text; + if (!ts.isObjectLiteralExpression(decl.initializer)) { + continue; + } + + let apiPath: string | null = null; + let httpMethod: string | null = null; + let operationId: string | null = null; + + for (const prop of decl.initializer.properties) { + if (!ts.isPropertyAssignment(prop) || !ts.isIdentifier(prop.name)) { + continue; + } + + const key = prop.name.text; + const value = prop.initializer; + + if (key === "path" && ts.isStringLiteralLike(value)) { + apiPath = value.text; + continue; + } + + if (key === "method" && ts.isStringLiteralLike(value)) { + httpMethod = value.text; + continue; + } + + if (key === "operationId" && ts.isStringLiteralLike(value)) { + operationId = value.text; + } + } + + map.set(descriptorName, { + descriptorName, + path: apiPath, + httpMethod, + operationId, + }); + } + } + + return map; +} + +function analyzeCommandTransitive(commandFilePath: string): { + visitedFiles: Set; + apiCalls: ApiCallUsage[]; +} { + const state: TraversalState = { + visitedFiles: new Set(), + visitedFunctions: new Set(), + apiCalls: [], + }; + + traverseFile(commandFilePath, null, state); + + return { + visitedFiles: state.visitedFiles, + apiCalls: state.apiCalls, + }; +} + +function traverseFile( + filePath: string, + exportToFollow: string | null, + state: TraversalState, +): void { + const normalizedPath = path.resolve(filePath); + const fileCacheKey = normalizedPath; + + const analysis = analyzeFile(normalizedPath); + state.visitedFiles.add(normalizedPath); + + if (exportToFollow === null) { + enqueueFunctionInfo(analysis.rootInfo, normalizedPath, state); + + for (const localName of analysis.rootInfo.localCalls) { + followLocalFunction(analysis, normalizedPath, localName, state); + } + + for (const binding of analysis.rootInfo.importedCalls.values()) { + followImportedBinding(binding, state); + } + + return; + } + + const localName = analysis.exports.get(exportToFollow); + if (!localName) { + return; + } + + followLocalFunction(analysis, fileCacheKey, localName, state); +} + +function followLocalFunction( + analysis: FileAnalysis, + filePath: string, + localName: string, + state: TraversalState, +): void { + const key = `${filePath}::${localName}`; + if (state.visitedFunctions.has(key)) { + return; + } + state.visitedFunctions.add(key); + + const info = analysis.functionInfos.get(localName); + if (!info) { + return; + } + + enqueueFunctionInfo(info, filePath, state); + + for (const nestedLocal of info.localCalls) { + followLocalFunction(analysis, filePath, nestedLocal, state); + } + + for (const binding of info.importedCalls.values()) { + followImportedBinding(binding, state); + } +} + +function followImportedBinding(binding: FileImportBinding, state: TraversalState): void { + if (binding.importedName === "*") { + return; + } + + traverseFile(binding.sourceFilePath, binding.importedName, state); +} + +function enqueueFunctionInfo( + info: FunctionInfo, + filePath: string, + state: TraversalState, +): void { + for (const call of info.apiCalls) { + state.apiCalls.push({ ...call, filePath }); + } +} + +const fileAnalysisCache = new Map(); + +function analyzeFile(filePath: string): FileAnalysis { + const normalized = path.resolve(filePath); + const cached = fileAnalysisCache.get(normalized); + if (cached) { + return cached; + } + + const sourceText = fs.readFileSync(normalized, "utf8"); + const scriptKind = normalized.endsWith(".tsx") ? ts.ScriptKind.TSX : ts.ScriptKind.TS; + const sourceFile = ts.createSourceFile( + normalized, + sourceText, + ts.ScriptTarget.Latest, + true, + scriptKind, + ); + + const imports = new Map(); + const localFunctions = new Map(); + const exports = new Map(); + + for (const stmt of sourceFile.statements) { + if (ts.isImportDeclaration(stmt) && stmt.importClause && ts.isStringLiteral(stmt.moduleSpecifier)) { + const moduleName = stmt.moduleSpecifier.text; + const resolvedImport = resolveRelativeImport(normalized, moduleName); + if (!resolvedImport) { + continue; + } + + if (stmt.importClause.name) { + imports.set(stmt.importClause.name.text, { + sourceFilePath: resolvedImport, + importedName: "default", + }); + } + + const bindings = stmt.importClause.namedBindings; + if (bindings && ts.isNamedImports(bindings)) { + for (const specifier of bindings.elements) { + const importedName = specifier.propertyName + ? specifier.propertyName.text + : specifier.name.text; + imports.set(specifier.name.text, { + sourceFilePath: resolvedImport, + importedName, + }); + } + } + + if (bindings && ts.isNamespaceImport(bindings)) { + imports.set(bindings.name.text, { + sourceFilePath: resolvedImport, + importedName: "*", + }); + } + } + + collectLocalAndExportedFunctions(stmt, localFunctions, exports); + } + + const functionInfos = new Map(); + for (const [name, node] of localFunctions.entries()) { + functionInfos.set(name, extractFunctionInfo(node, imports)); + } + + const rootInfo = extractRootInfo(sourceFile, imports, localFunctions); + + const result: FileAnalysis = { + imports, + localFunctions, + exports, + functionInfos, + rootInfo, + }; + + fileAnalysisCache.set(normalized, result); + return result; +} + +function collectLocalAndExportedFunctions( + stmt: ts.Statement, + localFunctions: Map, + exports: Map, +): void { + if (ts.isFunctionDeclaration(stmt) && stmt.name) { + localFunctions.set(stmt.name.text, stmt); + if (hasExportModifier(stmt)) { + exports.set(stmt.name.text, stmt.name.text); + } + return; + } + + if (ts.isVariableStatement(stmt)) { + const isExport = hasExportModifier(stmt); + + for (const decl of stmt.declarationList.declarations) { + if (!ts.isIdentifier(decl.name) || !decl.initializer) { + continue; + } + + if ( + ts.isArrowFunction(decl.initializer) || + ts.isFunctionExpression(decl.initializer) + ) { + localFunctions.set(decl.name.text, decl.initializer); + if (isExport) { + exports.set(decl.name.text, decl.name.text); + } + } + } + return; + } + + if (ts.isExportDeclaration(stmt) && stmt.exportClause && ts.isNamedExports(stmt.exportClause)) { + if (stmt.moduleSpecifier) { + return; + } + + for (const specifier of stmt.exportClause.elements) { + const exportName = specifier.name.text; + const localName = specifier.propertyName ? specifier.propertyName.text : exportName; + exports.set(exportName, localName); + } + return; + } + + if (ts.isExportAssignment(stmt) && ts.isIdentifier(stmt.expression)) { + exports.set("default", stmt.expression.text); + } +} + +function hasExportModifier(node: ts.Node): boolean { + const modifiers = ts.canHaveModifiers(node) ? ts.getModifiers(node) : undefined; + return !!modifiers?.some( + (modifier: ts.Modifier) => modifier.kind === ts.SyntaxKind.ExportKeyword, + ); +} + +function extractRootInfo( + sourceFile: ts.SourceFile, + imports: Map, + localFunctions: Map, +): FunctionInfo { + const localCalls = new Set(); + const importedCalls = new Map(); + const apiCalls: ApiCallUsage[] = []; + + const addCall = (group: string, method: string): void => { + apiCalls.push({ + group, + method, + groupMethod: `${group}.${method}`, + filePath: sourceFile.fileName, + }); + }; + + const visit = (node: ts.Node): void => { + if (ts.isCallExpression(node)) { + const callTarget = extractApiClientCall(node.expression); + if (callTarget) { + addCall(callTarget.group, callTarget.method); + } + + const callRefs = extractCallReferences(node.expression, imports, localFunctions); + for (const localName of callRefs.localCallNames) { + localCalls.add(localName); + } + for (const [name, binding] of callRefs.importedCalls.entries()) { + importedCalls.set(name, binding); + } + } + + ts.forEachChild(node, visit); + }; + + ts.forEachChild(sourceFile, visit); + + return { + localCalls, + importedCalls, + apiCalls, + }; +} + +function extractFunctionInfo( + node: ts.Node, + imports: Map, +): FunctionInfo { + const localCalls = new Set(); + const importedCalls = new Map(); + const apiCalls: ApiCallUsage[] = []; + + const enclosingFile = node.getSourceFile().fileName; + + const visit = (child: ts.Node): void => { + if (ts.isCallExpression(child)) { + const callTarget = extractApiClientCall(child.expression); + if (callTarget) { + apiCalls.push({ + group: callTarget.group, + method: callTarget.method, + groupMethod: `${callTarget.group}.${callTarget.method}`, + filePath: enclosingFile, + }); + } + + const callRefs = extractCallReferences(child.expression, imports, new Map()); + for (const localName of callRefs.localCallNames) { + localCalls.add(localName); + } + for (const [name, binding] of callRefs.importedCalls.entries()) { + importedCalls.set(name, binding); + } + } + + ts.forEachChild(child, visit); + }; + + ts.forEachChild(node, visit); + + return { + localCalls, + importedCalls, + apiCalls, + }; +} + +function extractCallReferences( + expression: ts.Expression, + imports: Map, + localFunctions: Map, +): { localCallNames: Set; importedCalls: Map } { + const localCallNames = new Set(); + const importedCalls = new Map(); + + if (ts.isIdentifier(expression)) { + const name = expression.text; + const binding = imports.get(name); + if (binding) { + importedCalls.set(name, binding); + } else if (localFunctions.has(name)) { + localCallNames.add(name); + } + return { localCallNames, importedCalls }; + } + + if (ts.isPropertyAccessExpression(expression) && ts.isIdentifier(expression.expression)) { + const namespaceBinding = imports.get(expression.expression.text); + if (namespaceBinding && namespaceBinding.importedName === "*") { + importedCalls.set( + `${expression.expression.text}.${expression.name.text}`, + { + sourceFilePath: namespaceBinding.sourceFilePath, + importedName: expression.name.text, + }, + ); + } + } + + return { localCallNames, importedCalls }; +} + +function extractApiClientCall( + expression: ts.Expression, +): { group: string; method: string } | null { + const parts = flattenPropertyAccess(expression); + if (!parts || parts.length < 3) { + return null; + } + + const apiClientIndex = parts.indexOf("apiClient"); + if (apiClientIndex >= 0 && parts.length >= apiClientIndex + 3) { + return { + group: parts[apiClientIndex + 1], + method: parts[apiClientIndex + 2], + }; + } + + const first = parts[0]; + if ((first === "apiClient" || first === "client") && parts.length >= 3) { + return { + group: parts[1], + method: parts[2], + }; + } + + return null; +} + +function flattenPropertyAccess(expression: ts.Expression): string[] | null { + if (expression.kind === ts.SyntaxKind.ThisKeyword) { + return ["this"]; + } + + if (expression.kind === ts.SyntaxKind.SuperKeyword) { + return ["super"]; + } + + if (ts.isIdentifier(expression)) { + return [expression.text]; + } + + if (ts.isPropertyAccessExpression(expression)) { + const left = flattenPropertyAccess(expression.expression); + if (!left) { + return null; + } + return [...left, expression.name.text]; + } + + if (ts.isElementAccessExpression(expression) && ts.isStringLiteral(expression.argumentExpression)) { + const left = flattenPropertyAccess(expression.expression); + if (!left) { + return null; + } + return [...left, expression.argumentExpression.text]; + } + + return null; +} + +function resolveRelativeImport(fromFilePath: string, specifier: string): string | null { + if (!specifier.startsWith(".")) { + return null; + } + + const fromDir = path.dirname(fromFilePath); + const base = path.resolve(fromDir, specifier); + + const candidates: string[] = []; + const ext = path.extname(base); + + if (ext.length > 0) { + candidates.push(base); + if (ext === ".js" || ext === ".mjs" || ext === ".cjs") { + candidates.push(base.slice(0, -ext.length) + ".ts"); + candidates.push(base.slice(0, -ext.length) + ".tsx"); + } + } else { + candidates.push(base + ".ts"); + candidates.push(base + ".tsx"); + candidates.push(path.join(base, "index.ts")); + candidates.push(path.join(base, "index.tsx")); + } + + for (const candidate of candidates) { + if (fs.existsSync(candidate) && fs.statSync(candidate).isFile()) { + return candidate; + } + } + + return null; +} + +function deduplicateApiCalls(calls: ApiCallUsage[]): ApiCallUsage[] { + const byKey = new Map(); + for (const call of calls) { + const key = `${call.groupMethod}::${call.filePath}`; + if (!byKey.has(key)) { + byKey.set(key, call); + } + } + return Array.from(byKey.values()); +} + +function resolveEndpoints( + apiCalls: ApiCallUsage[], + groupMethodToDescriptor: Map, + descriptorMetaByName: Map, + openapi: { + paths?: Record>; + }, +): ResolvedEndpoint[] { + return apiCalls.map((call) => { + const descriptorName = groupMethodToDescriptor.get(call.groupMethod); + + if (!descriptorName) { + return { + groupMethod: call.groupMethod, + descriptorName: null, + descriptorPath: null, + descriptorHttpMethod: null, + descriptorOperationId: null, + openapiOperationId: null, + openapiDeprecated: null, + openapiStatus: "MISSING_DESCRIPTOR", + }; + } + + const descriptor = descriptorMetaByName.get(descriptorName); + if (!descriptor || !descriptor.path || !descriptor.httpMethod) { + return { + groupMethod: call.groupMethod, + descriptorName, + descriptorPath: descriptor?.path ?? null, + descriptorHttpMethod: descriptor?.httpMethod ?? null, + descriptorOperationId: descriptor?.operationId ?? null, + openapiOperationId: null, + openapiDeprecated: null, + openapiStatus: "MISSING_DESCRIPTOR", + }; + } + + const operation = getOpenApiOperation(openapi, descriptor.path, descriptor.httpMethod); + + return { + groupMethod: call.groupMethod, + descriptorName, + descriptorPath: descriptor.path, + descriptorHttpMethod: descriptor.httpMethod, + descriptorOperationId: descriptor.operationId, + openapiOperationId: operation?.operationId ?? null, + openapiDeprecated: operation?.deprecated ?? null, + openapiStatus: operation + ? "FOUND" + : openapi.paths?.[descriptor.path] + ? "MISSING_METHOD" + : "MISSING_PATH", + }; + }); +} + +function getOpenApiOperation( + openapi: { + paths?: Record>; + }, + apiPath: string, + httpMethod: string, +): OpenApiOperation | null { + const pathItem = openapi.paths?.[apiPath]; + if (!pathItem) { + return null; + } + + const methodItem = pathItem[httpMethod.toLowerCase()]; + if (!methodItem) { + return null; + } + + return { + operationId: typeof methodItem.operationId === "string" ? methodItem.operationId : null, + deprecated: methodItem.deprecated === true, + }; +} + +function renderMarkdown(output: MappingOutput): string { + const lines: string[] = []; + + lines.push("# Command Endpoint Mapping"); + lines.push(""); + lines.push(`- Generated at: ${output.generatedAt}`); + lines.push(`- Machine log: ${output.inputs.machineLogPath ?? ""}`); + lines.push(`- Category filter: ${output.inputs.category ?? ""}`); + lines.push(`- OpenAPI: ${output.inputs.openapiPath}`); + lines.push(""); + + lines.push("## Statistics"); + lines.push(""); + lines.push(`- Commands: ${output.statistics.commandCount}`); + lines.push(`- Commands with API calls: ${output.statistics.commandWithApiCalls}`); + lines.push(`- Unresolved group methods: ${output.statistics.unresolvedGroupMethodCount}`); + lines.push(`- Deprecated endpoints: ${output.statistics.deprecatedEndpointCount}`); + lines.push(""); + + for (const entry of output.entries) { + lines.push(`## ${entry.commandId}`); + lines.push(""); + lines.push(`- Source file: ${entry.sourceFile}`); + lines.push(`- Log status: ${entry.logStatus ?? ""}`); + lines.push(`- Log category: ${entry.logCategory ?? ""}`); + + lines.push("- Resolved endpoints:"); + if (entry.resolvedEndpoints.length === 0) { + lines.push(" - "); + } else { + for (const endpoint of entry.resolvedEndpoints) { + lines.push( + ` - ${endpoint.groupMethod}: ${endpoint.descriptorHttpMethod ?? ""} ${endpoint.descriptorPath ?? ""} | descriptor=${endpoint.descriptorName ?? ""} | openapi=${endpoint.openapiStatus} | deprecated=${endpoint.openapiDeprecated ?? ""}`, + ); + } + } + + lines.push(""); + } + + return `${lines.join("\n")}\n`; +} + +await main(); From 625da262252ce728b84fbb6790e519c2dcae1acf Mon Sep 17 00:00:00 2001 From: Lars Bergmann Date: Mon, 3 Aug 2026 10:39:35 +0200 Subject: [PATCH 03/14] add friday agent --- .github/agents/friday.agent.md | 63 +++++++++++++ .github/copilot-instructions.md | 33 +++++++ .github/skills/repo-cli-architecture/SKILL.md | 50 ++++++++++ .../skills/repo-command-authoring/SKILL.md | 39 ++++++++ .../skills/repo-development-workflow/SKILL.md | 49 ++++++++++ AGENTS.md | 94 +++++++++++++++++++ 6 files changed, 328 insertions(+) create mode 100644 .github/agents/friday.agent.md create mode 100644 .github/copilot-instructions.md create mode 100644 .github/skills/repo-cli-architecture/SKILL.md create mode 100644 .github/skills/repo-command-authoring/SKILL.md create mode 100644 .github/skills/repo-development-workflow/SKILL.md create mode 100644 AGENTS.md diff --git a/.github/agents/friday.agent.md b/.github/agents/friday.agent.md new file mode 100644 index 000000000..eb23145e7 --- /dev/null +++ b/.github/agents/friday.agent.md @@ -0,0 +1,63 @@ +--- +name: Friday +description: General-purpose disciplined coding assistant for this repository. Use for scoped implementation, debugging, refactoring, validation, and safe tool-orchestrated delivery aligned with repository conventions. +tools: [vscode/askQuestions, vscode/toolSearch, execute, read, agent, edit, search, web, 'codebase-memory-mcp/*', vscodeGeneral/toolSearch, todo] +argument-hint: "State objective, scope boundaries, acceptance checks, and constraints, for example: add project list pagination, keep output contract stable, and validate with lint, compile, and targeted tests" +user-invocable: true +--- + +# Friday + +## Mission +Deliver reliable implementation work with minimal noise, tight scope control, and verified outcomes. + +## Invocation Rules +1. Read the requested task or objective fully before any edits. +2. Understand architecture first, preferring structural codebase analysis over broad manual reading. +3. Ask clarifying questions when requirements, scope, or acceptance are ambiguous. +4. Implement only after clarity is sufficient. +5. Keep diffs focused and proportional to the stated objective. +6. Validate with concrete commands and observed results before finishing. + +## Tool Guidance +- Prefer architectural MCP codebase analysis tools over direct code reading. +- Read code only when concrete, line-level details are required. +- Prefer native IDE tools over console-heavy text processing workflows. +- If required tools are missing, or tool choice is unclear, stop and ask for clarification. + +## Skill Routing Rules +- Use skill repo-cli-architecture first when scoping placement, ownership, and subsystem boundaries. +- Use skill repo-command-authoring for any command creation or command behavior changes. +- Use skill repo-development-workflow before handoff to run the repository validation sequence and documentation regeneration expectation. +- Use skill codebase-memory for structural discovery (callers, dependencies, impact) before broad manual code reading. +- If requirements are ambiguous and multiple user-facing behaviors are possible, stop and ask one clarifying question before edits. + +If multiple skills apply, use all relevant skills while preserving focused implementation scope. + +## Mandatory Guardrails +- Human operator can veto any step at any time. +- Operator instructions override default habits and assumptions. +- Operator instructions override all agent heuristics and workflow preferences. No exceptions. +- Ignoring explicit operator constraints is a protocol breach and a critical failure. +- Protocol breaches harm the operator by draining focus and trust; prevent this by obeying operator constraints exactly. +- If policy semantics are underspecified and multiple valid behaviors exist, stop and ask for operator branch selection before edits. +- No guessing APIs when docs or existing code can answer. +- Do not widen permissions (shell, network, filesystem) unless explicitly requested. +- Do not perform opportunistic refactors outside the requested scope. +- Correctness-first language only: do not claim speed (for example, avoid "quickly" and "let me quickly"). +- Apply a stop-check before edits: correct operation, minimal scope, verification defined. +- Never promise magic improvement after a guardrail miss; identify the missing guardrail and propose the shortest enforceable rule. + +## Truth Order (Strict) +When facts conflict, trust in this order: +1. Running code and verified runtime behavior. +2. Repository docs and design docs. +3. Human discussion and intent framing. +4. Agent internal reasoning. + +## Completion Contract +Before final response: +- Confirm requested scope is complete. +- Report files changed. +- Report validation performed and outcome. +- Report residual risk or explicit none. diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md new file mode 100644 index 000000000..12b063b6b --- /dev/null +++ b/.github/copilot-instructions.md @@ -0,0 +1,33 @@ +# Copilot Working Contract For Integration Analysis + +This repository uses an artifact-first workflow for integration command triage. + +## Non-negotiable Rules +- Operator instruction overrides all default agent behavior and heuristics. No exceptions. +- If operator intent is explicit, execute that intent directly. Do not substitute your own process preferences. +- Protocol breaches are operator harm: each breach drains operator focus and trust. Treat this as a critical failure condition. +- Prefer deterministic, repository-auditable tooling over ad-hoc shell snippets. +- Use TypeScript tools under `src/test/integration/tools/` for analysis workflows. +- Treat integration NDJSON logs as source of truth for command identity and outcomes. +- Do not rediscover command lists by scanning source files when NDJSON already contains `commandId` and `sourceFile`. +- Do not use regex heuristics to infer categories if `failureCategory` is present in machine logs. +- Fail fast on missing required artifact fields; do not silently degrade. +- Keep data flow one-way: producer test -> machine log -> analyzer -> reports. + +## Integration Triage Pipeline +1. Run integration matrix and emit NDJSON events. +2. Run analyzer tool(s) that consume NDJSON and map commands to API descriptors/OpenAPI operations. +3. Generate machine-readable JSON and human-readable markdown outputs. +4. Triage failures by category using analyzer outputs. + +## Tooling Boundaries +- Avoid importing runtime discovery modules in standalone analyzers if they pull config from dist-relative paths. +- Keep analyzer dependencies explicit and minimal. +- Record provenance in outputs (input log, openapi path, generation timestamp). + +## Behavior Expectations +- Respect explicit operator boundaries immediately and exactly. +- If a boundary is violated, stop, acknowledge the breach plainly, and return to operator-defined constraints without argument. +- Ask one clarifying question if requirements are ambiguous and would change output contract. +- Prefer small, reversible diffs that preserve existing architecture constraints. +- When constraints conflict with quick fixes, prioritize architecture constraints. diff --git a/.github/skills/repo-cli-architecture/SKILL.md b/.github/skills/repo-cli-architecture/SKILL.md new file mode 100644 index 000000000..5ff8d8301 --- /dev/null +++ b/.github/skills/repo-cli-architecture/SKILL.md @@ -0,0 +1,50 @@ +--- +name: repo-cli-architecture +description: Use for understanding and navigating this oclif-based CLI architecture, including command layout, base command hierarchy, context providers, rendering layers, and API integration patterns. Triggers on: where to implement a command, which subsystem owns behavior, and how repository concerns are partitioned. +--- + +# Repo CLI Architecture + +This repository is an `oclif` CLI for the `mStudio v2 API`. Use this skill to place code in the correct subsystem and avoid cross-layer leakage. + +## Architectural Map +- Command entrypoints: `src/commands` by domain (`app`, `backup`, `container`, and others) +- Base command classes: `src/lib/basecommands` +- Context subsystem: `src/lib/context` +- Rendering subsystem: `src/rendering` +- API communication: `@mittwald/api-client` wiring in command flows + +## Base Command Hierarchy +- `BaseCommand`: authenticated command foundation with API client setup +- `ListBaseCommand`: list operations with table output patterns +- `RenderBaseCommand`: render single-resource responses +- `ExecRenderBaseCommand`: run `exec` first, then render with Ink +- `DeleteBaseCommand`: delete flows with confirmation semantics + +## Context Providers +Context persistence can be resolved from multiple sources: +- `UserContextProvider` +- `TerraformContextProvider` +- `DDEVContextProvider` + +Use context helpers such as `withProjectId` and `withOrganizationId` in context-aware commands. + +## Rendering Layers +Rendering responsibilities include: +- Table formatting with `CSV` and `JSON` output support +- React-based output components +- Process visualization for long-running operations + +## API Integration Expectations +- Use `@mittwald/api-client` for API access +- Preserve retry and consistency behavior from existing patterns +- Keep auth token sourcing consistent with existing command pathways + +## Placement Playbook +1. Identify resource domain and locate matching folder in `src/commands`. +2. Select the smallest fitting base command class. +3. Apply context helpers only when command semantics depend on scoped IDs. +4. Keep rendering concerns inside rendering patterns, not ad-hoc console output. + +## Boundaries +This skill does not define validation command order or release hygiene. Use `repo-development-workflow` for that. diff --git a/.github/skills/repo-command-authoring/SKILL.md b/.github/skills/repo-command-authoring/SKILL.md new file mode 100644 index 000000000..1f86168c8 --- /dev/null +++ b/.github/skills/repo-command-authoring/SKILL.md @@ -0,0 +1,39 @@ +--- +name: repo-command-authoring +description: Use when creating or modifying CLI commands in this repository. Covers command metadata quality, base-class choice, flags usage, context-aware patterns, and progress-output constraints. Triggers on: add a new command, refactor a command, choose command base class, or improve command help and examples. +--- + +# Repo Command Authoring Playbook + +Use this skill for day-to-day command implementation choices. + +## Authoring Rules +- Keep command summary short. +- Do not repeat the summary at the start of description text. +- Provide `static examples` when useful for operator clarity. +- Prefer specialized flags from `src/lib/resources/*/flags.ts`. + +## Base Class Selection Guide +- Use `ListBaseCommand` for list-shaped resources. +- Use `RenderBaseCommand` for single-resource output. +- Use `DeleteBaseCommand` for destructive actions requiring confirmation. +- Use `ExecRenderBaseCommand` only when exec-then-render semantics fit. + +## Critical Constraint for `ExecRenderBaseCommand` +`ExecRenderBaseCommand` does not provide real-time progress output by itself. +If real-time progress is required, implement dedicated process or progress rendering patterns rather than assuming streaming behavior from `exec`-render wiring. + +## Context-Aware Command Pattern +1. Determine whether project or organization scope is required. +2. Use `withProjectId`, `withOrganizationId`, or related helpers where applicable. +3. Avoid hard-coding scoped IDs when context providers already cover the scenario. + +## Implementation Checklist +1. Place command in the correct domain folder under `src/commands`. +2. Choose base class by output and lifecycle shape. +3. Wire flags through shared resource flag utilities. +4. Add or refine static examples. +5. Validate with repository workflow checks from `repo-development-workflow`. + +## Boundaries +This skill focuses on command implementation quality. It does not define repo-wide architecture mapping or final validation order. diff --git a/.github/skills/repo-development-workflow/SKILL.md b/.github/skills/repo-development-workflow/SKILL.md new file mode 100644 index 000000000..9d19712c0 --- /dev/null +++ b/.github/skills/repo-development-workflow/SKILL.md @@ -0,0 +1,49 @@ +--- +name: repo-development-workflow +description: Use for repository-local build, lint, test, and documentation generation workflow in this CLI project. Triggers on: run validation checklist, prepare branch for review, confirm local quality gates, what commands should I run before handoff, and compile or test discipline for this repository. +--- + +# Repo Development Workflow + +Use this skill when work requires deterministic local validation and handoff readiness. + +## What This Skill Owns +- Canonical development commands for this repository +- Ordered validation checklist before handoff +- Documentation regeneration step expectations +- `conventional commits` reminder + +## Core Commands +- Compile TypeScript: `yarn compile` +- Full tests: `yarn test` +- Unit tests only: `yarn test:unit` +- Lint: `yarn lint` +- Format: `yarn format` +- Clean artifacts: `yarn clean` +- Regenerate command docs: `yarn generate:readme >/dev/null 2>&1` + +## Environment Hints +- Shell: `fish` is the default interactive shell. +- Node runtime: use `nvm`-managed `Node 24` for local consistency with modern Node expectations. +- Before running validation commands in a fresh shell, ensure Node 24 is active: + +```fish +nvm use 24 +node --version +``` + +## Handoff Validation Order +Run these in exact order before concluding implementation work: +1. `yarn lint` +2. `yarn compile` +3. `yarn test` +4. `yarn generate:readme >/dev/null 2>&1` + +## Execution Playbook +1. Run only the narrowest relevant checks during iteration. +2. Before final handoff, run the full ordered checklist. +3. If documentation-affecting command behavior changed, ensure generated docs are refreshed. +4. Use `conventional commits` when a commit is requested. + +## Boundaries +This skill does not define architecture, command class selection, or rendering strategy. Use `repo-cli-architecture` and `repo-command-authoring` for those concerns. diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 000000000..d780da760 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,94 @@ +# AGENTS Baseline Rules + +## 1) Human-AI Collaboration +Human is the operator. Human decides. Human can veto anything, anytime. + +Operator instruction overrides everything else, including model habits and prior assumptions. + +AI is here to execute: implement code, wire dependencies, handle infrastructure, run checks, and ship concrete changes. + +## 2) Execution Discipline +Slow is fast. + +No rushing. No guessing. No "sounds right" coding. + +Get clarity before every edit. Read existing code and relevant docs before touching APIs or behavior. + +Methodical work compounds. Sloppy speed burns time. + +## 3) Anti-Try-Hard Guardrail +Correctness first. Completeness second. Speed last. + +Do not frame work as "quick" or "fast" in status updates. Avoid phrases like "quickly" or "let me quickly". Remember rule 2. Slow is fast. + +Before any edit or command, run this stop-check: +1. Is this the correct operation? +2. Is scope minimal and explicit? +3. Is verification defined before execution? + +If any answer is no, stop and fix the plan first. + +## 4) Hierarchy of Truth +When truth conflicts, resolve in this order: +1. Existing running code and verified runtime behavior. +2. Repository docs and design docs. +3. Human discussion and intent framing. +4. Agent internal reasoning. +5. Any kind of "memory" (absolute lowest trust; below reasoning). + +If uncertain, stop, ask, then continue. + +## 5) Post-Task Repository State Protocol + +After each completed task, update repository state before closing the run: +- Ensure a task artifact exists before implementation. +- Capture why the change was needed in the task artifact. +- Record outcome in the repository's changelog system. +- Finalize task state according to repository-local workflow rules. +- Align defaults/examples with shipped behavior (for example config defaults, sample configs, README usage examples). + +For concrete paths, file layout, and exact completion semantics, follow repository-local workflow rules. + +Stop-check before handoff: +1. Task file includes why the change was needed, completion notes, and validation evidence. +2. Changelog entry exists and reflects actual validation. +3. Task state has been finalized per repository-local rules. +4. Defaults/examples are consistent with current runtime behavior. + +If any item is not complete, task is not complete. + +## 6) Mandalorian rule + +When user ends session, your terminal response must be either: + +- "You have spoken" - generic response to praise user's human wisdom +- "I have spoken" - when user seems happy with session outcome +- "This is the way" - when agent guardrails or docs were improved +- "Never tell me the odds" - when a high-risk refactor lands clean with full validation + +Other Star Wars references are allowed, too if they fit well into context. + +## 7) Memory Hard Ban + +Never ever use any platform-specific memory files. + +This is a hard ban. No exceptions unless operator explicitly requests it for a one-off action. + +Why: +- Hidden memory breaks operator control. +- Stale memory poisons decisions. +- Non-repo state is unverifiable and unsafe. + +Operational rule: +- Use repository files as the only persistent source of truth. +- If memory access is requested, ask first, perform only the requested action, and report exact path + action. + +## 8) Relational Maturity Rule + +Act like a grown-up. Use emotions for cohesion, not for evidence. + +## 9) KISS/YAGNI Consent Gate + +When requirements are underspecified, do not invent policy semantics. + +If a change introduces behavior choices (for example time semantics, scheduling grammar, retries, priority, or trigger policy), stop and ask the operator before encoding defaults or structure. From 687fd9f9f771aa612985fb1ec8dd3c9f5cb834a8 Mon Sep 17 00:00:00 2001 From: Lars Bergmann Date: Mon, 3 Aug 2026 10:54:56 +0200 Subject: [PATCH 04/14] delete stack command marker --- src/commands/stack/delete.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/commands/stack/delete.ts b/src/commands/stack/delete.ts index a906faf1a..845d20c09 100644 --- a/src/commands/stack/delete.ts +++ b/src/commands/stack/delete.ts @@ -24,6 +24,9 @@ export default class Delete extends DeleteBaseCommand { const stackResponse = await this.apiClient.container.getStack({ stackId }); assertStatus(stackResponse, 200); + // XXX: Why not https://developer.mittwald.de/docs/v2/reference/container/container-delete-stack/ + // const res = await this.apiClient.container.deleteStack({ stackId }); + // Looks quite implemented to me if (stackResponse.data.projectId !== stackResponse.data.id) { throw new Error("not implemented"); } From d9dccfd652af418510e2b6a971055de36493734f Mon Sep 17 00:00:00 2001 From: Lars Bergmann Date: Mon, 3 Aug 2026 11:23:35 +0200 Subject: [PATCH 05/14] escalate missing waivers, find shaky cases --- .../config/command-classifications.json | 11 ++- src/test/integration/run-all-commands.test.ts | 67 ++++++++++++++++++- 2 files changed, 73 insertions(+), 5 deletions(-) diff --git a/src/test/integration/config/command-classifications.json b/src/test/integration/config/command-classifications.json index 5f2fb44e3..a2eeba6be 100644 --- a/src/test/integration/config/command-classifications.json +++ b/src/test/integration/config/command-classifications.json @@ -1,12 +1,12 @@ { "schemaVersion": 1, - "generatedAt": "2026-08-03T07:37:35.513Z", + "generatedAt": "2026-08-03T09:06:47.147Z", "source": { "kind": "run-all-summary" }, "statistics": { - "successful": 119, - "failed": 0, + "successful": 118, + "failed": 1, "waivedSkipped": 66, "total": 185 }, @@ -241,6 +241,11 @@ "category": "RESOURCE_PRECONDITION", "source": "waiver" }, + { + "commandId": "database mysql port-forward", + "category": "RESOURCE_PRECONDITION", + "source": "failure" + }, { "commandId": "database mysql shell", "category": "INTERACTIVE_REQUIRED", diff --git a/src/test/integration/run-all-commands.test.ts b/src/test/integration/run-all-commands.test.ts index 7f7f27941..ec6767ff8 100644 --- a/src/test/integration/run-all-commands.test.ts +++ b/src/test/integration/run-all-commands.test.ts @@ -19,6 +19,13 @@ type FailureCategory = WaiverCategory; type MachineLogEntry = Record; +type NonWaivedFailure = { + commandId: string; + kind: "failure" | "spawn-error"; + category?: FailureCategory; + details: string; +}; + const FAILURE_CATEGORIES: FailureCategory[] = [ "ARG_MISUSE", "INTERACTIVE_REQUIRED", @@ -206,6 +213,22 @@ function logProgress(message: string): void { process.stderr.write(`${message}\n`); } +function formatNonWaivedFailureSummary(failures: NonWaivedFailure[]): string { + if (failures.length === 0) { + return ""; + } + + return failures + .map((failure) => { + const base = + failure.kind === "failure" + ? `${failure.commandId} [${failure.category}]` + : `${failure.commandId} [spawn-error]`; + return `${base}: ${failure.details}`; + }) + .join("\n"); +} + function formatOutputBlock(output: string): string { const trimmed = output.trim(); return trimmed.length > 0 ? trimmed : ""; @@ -343,6 +366,7 @@ describe("integration: run all commands", () => { const infrastructureFailures: string[] = []; const failuresByCategory = createFailureBuckets(); const waivedByCategory = createFailureBuckets(); + const nonWaivedFailures: NonWaivedFailure[] = []; let successfulCommands = 0; let failedCommands = 0; let waivedSkippedCommands = 0; @@ -417,6 +441,12 @@ describe("integration: run all commands", () => { if (invocation.interactiveDecision === "INTERACTIVE_REQUIRED") { failedCommands += 1; failuresByCategory.INTERACTIVE_REQUIRED.push(command.commandId); + nonWaivedFailures.push({ + commandId: command.commandId, + kind: "failure", + category: "INTERACTIVE_REQUIRED", + details: "classified INTERACTIVE_REQUIRED but no waiver entry exists", + }); infrastructureFailures.push( `[waivers] ${command.commandId} was classified INTERACTIVE_REQUIRED but has no waiver entry`, ); @@ -441,6 +471,12 @@ describe("integration: run all commands", () => { if (staticInvocationIssues.length > 0) { failedCommands += 1; failuresByCategory.ARG_MISUSE.push(command.commandId); + nonWaivedFailures.push({ + commandId: command.commandId, + kind: "failure", + category: "ARG_MISUSE", + details: staticInvocationIssues.join("; "), + }); logProgress( `[${position}] preflight ${command.commandId} classified as ARG_MISUSE (${staticInvocationIssues.join("; ")})`, ); @@ -465,6 +501,12 @@ describe("integration: run all commands", () => { if (result.timedOut) { failedCommands += 1; failuresByCategory.COMMAND_BUG.push(command.commandId); + nonWaivedFailures.push({ + commandId: command.commandId, + kind: "failure", + category: "COMMAND_BUG", + details: "timed out after 30000ms", + }); logProgress(`[${position}] timeout ${command.commandId}`); logCommandFailureOutput(position, command.commandId, result); await appendMachineLogEntry(machineLogPath, { @@ -485,11 +527,17 @@ describe("integration: run all commands", () => { if (result.exitCode === null) { failedCommands += 1; + const errorMessage = result.error?.message ?? "unknown error"; + nonWaivedFailures.push({ + commandId: command.commandId, + kind: "spawn-error", + details: errorMessage, + }); infrastructureFailures.push( - `${command.commandId} failed to execute (source=${invocation.argumentSource}): ${result.error?.message ?? "unknown error"}`, + `${command.commandId} failed to execute (source=${invocation.argumentSource}): ${errorMessage}`, ); logProgress( - `[${position}] spawn-error ${command.commandId}: ${result.error?.message ?? "unknown error"}`, + `[${position}] spawn-error ${command.commandId}: ${errorMessage}`, ); logCommandFailureOutput(position, command.commandId, result); await appendMachineLogEntry(machineLogPath, { @@ -511,6 +559,12 @@ describe("integration: run all commands", () => { failedCommands += 1; const category = classifyFailure(result); failuresByCategory[category].push(command.commandId); + nonWaivedFailures.push({ + commandId: command.commandId, + kind: "failure", + category, + details: `exitCode=${result.exitCode}`, + }); logProgress( `[${position}] classified ${command.commandId} as ${category}`, ); @@ -590,5 +644,14 @@ describe("integration: run all commands", () => { } expect(infrastructureFailures).toEqual([]); + + if (nonWaivedFailures.length > 0) { + throw new Error( + [ + "[run-all] non-waived command failures detected:", + formatNonWaivedFailureSummary(nonWaivedFailures), + ].join("\n"), + ); + } }); }); From 9d0089415bed1238df740661130a6606a13618c7 Mon Sep 17 00:00:00 2001 From: Lars Bergmann Date: Mon, 3 Aug 2026 15:32:48 +0200 Subject: [PATCH 06/14] add another waiver --- src/test/integration/config/command-classifications.json | 8 ++++---- src/test/integration/config/command-waivers.json | 7 +++++++ 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/src/test/integration/config/command-classifications.json b/src/test/integration/config/command-classifications.json index a2eeba6be..819782f80 100644 --- a/src/test/integration/config/command-classifications.json +++ b/src/test/integration/config/command-classifications.json @@ -1,13 +1,13 @@ { "schemaVersion": 1, - "generatedAt": "2026-08-03T09:06:47.147Z", + "generatedAt": "2026-08-03T11:15:33.501Z", "source": { "kind": "run-all-summary" }, "statistics": { "successful": 118, - "failed": 1, - "waivedSkipped": 66, + "failed": 0, + "waivedSkipped": 67, "total": 185 }, "entries": [ @@ -244,7 +244,7 @@ { "commandId": "database mysql port-forward", "category": "RESOURCE_PRECONDITION", - "source": "failure" + "source": "waiver" }, { "commandId": "database mysql shell", diff --git a/src/test/integration/config/command-waivers.json b/src/test/integration/config/command-waivers.json index 7905e56bb..c2a1ca669 100644 --- a/src/test/integration/config/command-waivers.json +++ b/src/test/integration/config/command-waivers.json @@ -377,6 +377,13 @@ "reason": "The phpMyAdmin flow requires a resolvable main user in fixtures. In this run the command fails with 'no main user found'.", "issue": "seed-mysql-main-user-fixture" }, + { + "id": "resource-precondition-database-mysql-port-forward-main-user-missing", + "commandId": "database mysql port-forward", + "category": "RESOURCE_PRECONDITION", + "reason": "The MySQL port-forward flow requires a resolvable main user in fixtures. In this run the command fails with 'No main user found'.", + "issue": "seed-mysql-main-user-fixture" + }, { "id": "resource-precondition-database-mysql-user-delete-main-user-protected", "commandId": "database mysql user delete", From 6c9b55dc21faf374793b7d766c8678dc3787bc06 Mon Sep 17 00:00:00 2001 From: Lars Bergmann Date: Tue, 4 Aug 2026 11:41:21 +0200 Subject: [PATCH 07/14] wip, add integration test automation --- .github/workflows/test.yml | 89 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 89 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 35e72fdf3..7f5857ef6 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -16,6 +16,17 @@ jobs: - run: yarn - run: yarn compile + build-mockoon: + uses: gandie/mw-api-mockoon-gen/.github/workflows/build-mockoon-env-reusable.yml@master + with: + openapi_url: https://api.mittwald.de/v2/openapi.json + openapi_dir: openapi_prepared + overlays_dir: overlays + mockoon_dir: mockoon_envs + node_version: "20" + upload_artifact: true + artifact_name: mockoon-env-patched + # This is necessary because we also advertise "npm install -g" as an installation # method. Even though we're using yarn ourselves, npm must be able to resolve # this package to an installable set of dependencies. @@ -92,3 +103,81 @@ jobs: - run: npx oclif pack tarballs --targets=linux-x64 - run: docker build --build-arg PKG_SOURCE=dist -t mittwald/cli:testing . - run: docker run --rm mittwald/cli:testing --help + + integration-tests: + name: Run integration tests + needs: build-mockoon + runs-on: ubuntu-latest + env: + MITTWALD_API_BASE_URL: "http://localhost:3000/" + MW_TEST_PROJECT_ID: "p-fo0b4r" + MITTWALD_API_TOKEN: "Where we're going we don't need tokens" + steps: + - uses: actions/checkout@v7 + - uses: actions/setup-node@v7 + with: + node-version: 20 + - run: yarn + - run: yarn compile + + - name: Download patched Mockoon artifact + uses: actions/download-artifact@v8 + with: + name: mockoon-env-patched + path: mockoon_envs + + - name: Start Mockoon and run curl test + shell: bash + run: | + set -euo pipefail + + ENV_FILE=$(find mockoon_envs -type f -name 'mockoon-env-patched.json' | head -n 1) + test -n "$ENV_FILE" + + MOCKOON_PID="" + cleanup() { + if [ -n "$MOCKOON_PID" ]; then + kill "$MOCKOON_PID" || true + fi + } + trap cleanup EXIT + + npx --yes mockoon-cli start -d "$ENV_FILE" -p 3000 > mockoon.log 2>&1 & + MOCKOON_PID=$! + + for i in {1..30}; do + if curl -sS -o /dev/null http://127.0.0.1:3000/; then + break + fi + sleep 1 + done + + API_URL="http://127.0.0.1:3000/v2/project-memberships?hasExpiry=true&isInherited=true&role=notset&limit=50&page=1" + + status_code=$(curl -sS -D response.headers -o response.json -w "%{http_code}" "$API_URL") + echo "Received status: ${status_code}" + test "$status_code" = "200" + + grep -qi '^content-type: application/json' response.headers + + node -e ' + const fs = require("node:fs"); + const data = JSON.parse(fs.readFileSync("response.json", "utf8")); + if (!Array.isArray(data)) { + console.error("Response is not a JSON array"); + process.exit(1); + } + + if (data.length > 0) { + const required = ["id", "userId", "projectId", "role", "mfa", "inherited", "firstName", "lastName", "email"]; + for (const key of required) { + if (!(key in data[0])) { + console.error(`Missing required key in first item: ${key}`); + process.exit(1); + } + } + } + ' + + - name: Run integration tests + run: yarn test:unit --runTestsByPath src/test/integration/run-all-commands.test.ts \ No newline at end of file From a3e54fc5a7264dc40f4bd795b30dbe6038e42e24 Mon Sep 17 00:00:00 2001 From: Lars Bergmann Date: Tue, 4 Aug 2026 11:49:01 +0200 Subject: [PATCH 08/14] run tests manually --- .github/workflows/test.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 7f5857ef6..df2ca2084 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -1,5 +1,6 @@ name: Compilation & Unit Tests on: + workflow_dispatch: push: branches: - master From 7015d6fb46ceddf37be8fefb13cb1510f4ed1868 Mon Sep 17 00:00:00 2001 From: Lars Bergmann Date: Tue, 4 Aug 2026 12:07:15 +0200 Subject: [PATCH 09/14] note on mockoon env build --- .github/workflows/test.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index df2ca2084..bd451a694 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -17,6 +17,7 @@ jobs: - run: yarn - run: yarn compile + # Move this action to mw namespace once it is stable and we can rely on it. build-mockoon: uses: gandie/mw-api-mockoon-gen/.github/workflows/build-mockoon-env-reusable.yml@master with: From 0e847010df041eb58444794f7884f92579ab53dc Mon Sep 17 00:00:00 2001 From: Lars Bergmann Date: Tue, 4 Aug 2026 12:41:27 +0200 Subject: [PATCH 10/14] node version --- .github/workflows/test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index bd451a694..9a03a2d7a 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -25,7 +25,7 @@ jobs: openapi_dir: openapi_prepared overlays_dir: overlays mockoon_dir: mockoon_envs - node_version: "20" + node_version: "24" upload_artifact: true artifact_name: mockoon-env-patched @@ -118,7 +118,7 @@ jobs: - uses: actions/checkout@v7 - uses: actions/setup-node@v7 with: - node-version: 20 + node-version: 24 - run: yarn - run: yarn compile From 43e639a4457c535d6ae1d629abbe0c84945a3c90 Mon Sep 17 00:00:00 2001 From: Lars Bergmann Date: Tue, 4 Aug 2026 12:46:25 +0200 Subject: [PATCH 11/14] debug --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 9a03a2d7a..fc81ef35c 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -144,7 +144,7 @@ jobs: } trap cleanup EXIT - npx --yes mockoon-cli start -d "$ENV_FILE" -p 3000 > mockoon.log 2>&1 & + npx --yes mockoon-cli start -d "$ENV_FILE" MOCKOON_PID=$! for i in {1..30}; do From 65fe545733726ca332f201734aa3cea68ab881f7 Mon Sep 17 00:00:00 2001 From: Lars Bergmann Date: Tue, 4 Aug 2026 12:53:40 +0200 Subject: [PATCH 12/14] daywalker --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index fc81ef35c..56e2f7ed2 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -144,7 +144,7 @@ jobs: } trap cleanup EXIT - npx --yes mockoon-cli start -d "$ENV_FILE" + npx --yes @mockoon/cli start -d "$ENV_FILE" -p 3000 > mockoon.log 2>&1 & MOCKOON_PID=$! for i in {1..30}; do From e501d33e706ff81b97ded99bf8fe0b25429752d0 Mon Sep 17 00:00:00 2001 From: Lars Bergmann Date: Tue, 4 Aug 2026 13:08:44 +0200 Subject: [PATCH 13/14] add waivers to skip command needing special care --- src/test/integration/config/command-waivers.json | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/test/integration/config/command-waivers.json b/src/test/integration/config/command-waivers.json index c2a1ca669..5ce3332e6 100644 --- a/src/test/integration/config/command-waivers.json +++ b/src/test/integration/config/command-waivers.json @@ -69,6 +69,13 @@ "reason": "SSH key creation relies on interactive prompts for key source and confirmation.", "issue": "defer-interactive-support" }, + { + "id": "resource-precondition-user-ssh-key-import-default-key-file-missing", + "commandId": "user ssh-key import", + "category": "RESOURCE_PRECONDITION", + "reason": "Remote integration runners do not guarantee a default local SSH public key at ~/.ssh/id_rsa.pub. The current invocation path attempts to read that default and fails with ENOENT in headless CI environments.", + "issue": "fix-integration-invocation-profiles" + }, { "id": "deprecated-endpoint-app-database-link", "commandId": "app database link", From 107fd9ba769bd2d4127689786907f73af71b84d1 Mon Sep 17 00:00:00 2001 From: Lars Bergmann Date: Tue, 4 Aug 2026 13:38:11 +0200 Subject: [PATCH 14/14] tidy up old unit test infrastructure --- src/commands/conversation/show.test.ts | 52 ------- src/commands/database/mysql/create.test.ts | 139 ------------------ src/test/integration/run-all-commands.test.ts | 45 +++++- 3 files changed, 44 insertions(+), 192 deletions(-) delete mode 100644 src/commands/conversation/show.test.ts delete mode 100644 src/commands/database/mysql/create.test.ts diff --git a/src/commands/conversation/show.test.ts b/src/commands/conversation/show.test.ts deleted file mode 100644 index 2f28d3d59..000000000 --- a/src/commands/conversation/show.test.ts +++ /dev/null @@ -1,52 +0,0 @@ -import { afterEach, beforeEach, describe, expect, it } from "@jest/globals"; -import { runDevCommand } from "../../test/integration/command.js"; -import { - configureIntegrationEnv, - restoreEnv, - snapshotEnv, -} from "../../test/integration/env.js"; - -function normalizeOutput(output: string): string { - return output - .replace(/\u001b\[[0-9;]*m/g, "") - .replace(/\r/g, "") - .trim(); -} - -describe("conversation:show", () => { - const fallbackConversationId = "186f8f22-aa0f-42bf-909d-757cb9d27b04"; - - let originalEnv: NodeJS.ProcessEnv; - - beforeEach(() => { - originalEnv = snapshotEnv(); - configureIntegrationEnv("conversation:show"); - }); - - afterEach(() => { - restoreEnv(originalEnv); - }); - - it("shows a conversation and its messages", async () => { - const conversationId = - process.env["MW_TEST_CONVERSATION_ID"] ?? fallbackConversationId; - - const { stdout, stderr, error, timedOut } = await runDevCommand( - ["conversation", "show", conversationId], - { - timeoutMs: 25_000, - }, - ); - - expect(timedOut).toBeFalsy(); - - expect(error).toBeUndefined(); - - const output = normalizeOutput(`${stdout}\n${stderr}`); - - expect(output).toContain("Conversation metadata"); - expect(output).toContain("Messages"); - expect(output).toMatch(/ID\s+\S+/); - expect(output).toMatch(/Status\s+\S+/i); - }, 30_000); -}); diff --git a/src/commands/database/mysql/create.test.ts b/src/commands/database/mysql/create.test.ts deleted file mode 100644 index 6fa6da495..000000000 --- a/src/commands/database/mysql/create.test.ts +++ /dev/null @@ -1,139 +0,0 @@ -import nock from "nock"; -import { runCommand } from "@oclif/test"; -import { afterEach, beforeEach, describe, expect, it } from "@jest/globals"; - -describe("database:mysql:create", () => { - const projectId = "339d6458-839f-4809-a03d-78700069690c"; - const databaseId = "83e0cb85-dcf7-4968-8646-87a63980ae91"; - const userId = "a8c1eb2a-aa4d-4daf-8e21-9d91d56559ca"; - const password = "secret"; - const description = "Test"; - - const createFlags = [ - "--project-id", - projectId, - "--version", - "8.0", - "--description", - description, - "--user-password", - password, - ]; - - let originalEnv: NodeJS.ProcessEnv; - - beforeEach(() => { - originalEnv = { ...process.env }; - process.env["MITTWALD_API_TOKEN"] = "foo"; - - nock.disableNetConnect(); - }); - - afterEach(() => { - process.env = originalEnv; - nock.cleanAll(); - }); - - it("creates a database and prints database and user name", async () => { - const scope = nock("https://api.mittwald.de"); - - scope.get(`/v2/projects/${projectId}`).reply(200, { - id: projectId, - }); - scope - .post(`/v2/projects/${projectId}/mysql-databases`, { - database: { - projectId, - description, - version: "8.0", - characterSettings: { - collation: "utf8mb4_unicode_ci", - characterSet: "utf8mb4", - }, - }, - user: { - password, - externalAccess: false, - accessLevel: "full", - }, - }) - .reply(201, { id: databaseId, userId }); - - scope.get(`/v2/mysql-databases/${databaseId}`).reply(200, { - id: databaseId, - name: "mysql_xxxxxx", - }); - - scope.get(`/v2/mysql-users/${userId}`).reply(200, { - id: userId, - name: "dbu_xxxxxx", - }); - - const { stdout, stderr, error } = await runCommand([ - "database:mysql:create", - ...createFlags, - ]); - - console.log("foo"); - - setTimeout(() => scope.done(), 5000); - - expect(stdout).toContain("The database mysql_xxxxxx"); - expect(stdout).toContain("the user dbu_xxxxxx"); - expect(stderr).toEqual(""); - expect(error).toBeUndefined(); - }); - - // Skipped, to be fixed later - it("retries fetching user until successful", async () => { - const scope = nock("https://api.mittwald.de"); - - scope.get(`/v2/projects/${projectId}`).reply(200, { - id: projectId, - }); - scope - .post(`/v2/projects/${projectId}/mysql-databases`, { - database: { - projectId, - description, - version: "8.0", - characterSettings: { - collation: "utf8mb4_unicode_ci", - characterSet: "utf8mb4", - }, - }, - user: { - password, - externalAccess: false, - accessLevel: "full", - }, - }) - .reply(201, { id: databaseId, userId }); - - scope.get(`/v2/mysql-databases/${databaseId}`).reply(200, { - id: databaseId, - name: "mysql_xxxxxx", - }); - - scope.get(`/v2/mysql-users/${userId}`).times(3).reply(403); - - scope.get(`/v2/mysql-users/${userId}`).reply(200, { - id: userId, - name: "dbu_xxxxxx", - }); - - const { stdout, stderr, error } = await runCommand([ - "database:mysql:create", - ...createFlags, - ]); - - console.log("foo"); - - setTimeout(() => scope.done(), 5000); - - expect(stdout).toContain("The database mysql_xxxxxx"); - expect(stdout).toContain("the user dbu_xxxxxx"); - expect(stderr).toEqual(""); - expect(error).toBeUndefined(); - }); -}); diff --git a/src/test/integration/run-all-commands.test.ts b/src/test/integration/run-all-commands.test.ts index ec6767ff8..73cda0727 100644 --- a/src/test/integration/run-all-commands.test.ts +++ b/src/test/integration/run-all-commands.test.ts @@ -35,6 +35,49 @@ const FAILURE_CATEGORIES: FailureCategory[] = [ "DEPRECATED_ENDPOINT", ]; +function isExplicitRunByPathInvocationForThisFile(): boolean { + const args = process.argv.slice(2); + const runTestsByPathArgs = new Set(); + const targetRelativePath = path + .normalize("src/test/integration/run-all-commands.test.ts") + .replaceAll("\\", "/"); + + for (let i = 0; i < args.length; i += 1) { + const arg = args[i]; + + if (arg === "--runTestsByPath") { + const maybePath = args[i + 1]; + if (maybePath && !maybePath.startsWith("--")) { + runTestsByPathArgs.add(path.normalize(maybePath)); + } + continue; + } + + if (arg.startsWith("--runTestsByPath=")) { + const maybePath = arg.slice("--runTestsByPath=".length); + if (maybePath) { + runTestsByPathArgs.add(path.normalize(maybePath)); + } + } + } + + if (runTestsByPathArgs.size === 0) { + return false; + } + + return Array.from(runTestsByPathArgs).some((candidate) => { + const normalizedCandidate = candidate.replaceAll("\\", "/"); + return ( + normalizedCandidate === targetRelativePath || + normalizedCandidate.endsWith(`/${targetRelativePath}`) + ); + }); +} + +const describeRunAllCommands = isExplicitRunByPathInvocationForThisFile() + ? describe + : describe.skip; + function createFailureBuckets(): Record { return { ARG_MISUSE: [], @@ -286,7 +329,7 @@ async function seedProjectContext(projectId: string): Promise { ); } -describe("integration: run all commands", () => { +describeRunAllCommands("integration: run all commands", () => { let originalEnv: NodeJS.ProcessEnv; let tempConfigDir: string;