diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md new file mode 100644 index 000000000..b6e65509f --- /dev/null +++ b/.github/copilot-instructions.md @@ -0,0 +1,28 @@ +# Copilot Working Contract For Integration Analysis + +This repository uses an artifact-first workflow for integration command triage. + +## Non-negotiable Rules +- Prefer deterministic, repository-auditable tooling over ad-hoc shell snippets. +- Use TypeScript tools under `src/test/integration/tools/` for analysis workflows. +- Treat integration NDJSON logs as source of truth for command identity and outcomes. +- Do not rediscover command lists by scanning source files when NDJSON already contains `commandId` and `sourceFile`. +- Do not use regex heuristics to infer categories if `failureCategory` is present in machine logs. +- Fail fast on missing required artifact fields; do not silently degrade. +- Keep data flow one-way: producer test -> machine log -> analyzer -> reports. + +## Integration Triage Pipeline +1. Run integration matrix and emit NDJSON events. +2. Run analyzer tool(s) that consume NDJSON and map commands to API descriptors/OpenAPI operations. +3. Generate machine-readable JSON and human-readable markdown outputs. +4. Triage failures by category using analyzer outputs. + +## Tooling Boundaries +- Avoid importing runtime discovery modules in standalone analyzers if they pull config from dist-relative paths. +- Keep analyzer dependencies explicit and minimal. +- Record provenance in outputs (input log, openapi path, generation timestamp). + +## Behavior Expectations +- Ask one clarifying question if requirements are ambiguous and would change output contract. +- Prefer small, reversible diffs that preserve existing architecture constraints. +- When constraints conflict with quick fixes, prioritize architecture constraints. diff --git a/.gitignore b/.gitignore index 3ea8a77ec..a38b16f16 100644 --- a/.gitignore +++ b/.gitignore @@ -65,3 +65,9 @@ atlassian-ide-plugin.xml # Editor-based Rest Client .idea/httpRequests + +# integration testing artifacts +openapi.json +run-all-commands.ndjson +command-endpoint-map.json +command-endpoint-map.md diff --git a/docs/integration-tests-reactivation-plan.md b/docs/integration-tests-reactivation-plan.md new file mode 100644 index 000000000..bc2c94075 --- /dev/null +++ b/docs/integration-tests-reactivation-plan.md @@ -0,0 +1,273 @@ +# Integration Tests Reactivation Plan (No HTTP Mocking) + +## Goal +Run an integration command matrix for all CLI commands, not just selected skipped tests. +Use static code analysis to discover commands and synthesize runnable argument sets. +Reduce non-actionable failures so remaining failures indicate real command or contract issues. + +## Scope Update (2026-07-29) +- Previous scope (reactivate only two tests) is replaced. +- Primary scope is now run-all-commands integration coverage. +- Command discovery and invocation quality are now the main workstream. + +## Current Baseline (Latest Full Run) +- Total commands executed: 185 +- Successful: 100 +- Failed: 85 + +Top failure signals from testrun_manual.log: +- Missing required arg: 20 +- Missing required flag: 14 +- Exactly one of required options: 6 +- Interactive input required: 3 +- Nonexistent flag: 1 +- Resource precondition failures (not found/does not exist): 26 +- Contract-shape/runtime issues (`Invalid Version`, iterable/shape/type errors): remaining bucket + +Conclusion: +- Largest improvement potential is discovery and invocation synthesis. +- Next milestone is lowering argument-misuse failures to isolate real product issues. + +## Progress Update (2026-07-30) +- ARG_MISUSE failures are now at 0. +- Discovery hardening for argument and required-flag synthesis appears complete. +- Integration runs already surfaced one minor real command bug. +- Task 0 has been completed in parallel: a fully automated Mockoon environment is now running, and a separate repository was created to conserve and share this setup. + +## Constraints +- No nock and no per-test HTTP stubbing. +- HTTP must flow through `MITTWALD_API_BASE_URL`. +- Deterministic and CI-compatible runs only. + +## Mock Target Position +- Programmatic Mockoon runtime orchestration is active and automated. +- The environment is already integrated into full automated runs. +- A separate repository exists to preserve and share the Mockoon environment. + +## Immediate Next Task: Discovery Hardening +Owner: Code Agent +Status: In Progress (Argument Hardening Largely Complete) + +Actions: +1. Replace the current discovery flow in src/test/integration/command-discovery.ts with a new static schema-driven discovery flow that satisfies required args and required flags with type-aware defaults. (Done) +2. Add deterministic handling for mutually exclusive requirements (exactly-one groups). (Done) +3. Add interactive-command policy: + - provide non-interactive flags when supported, otherwise classify as INTERACTIVE_REQUIRED. +4. Add stale example detection and fallback when example flags are invalid. +5. Add command-specific invocation profiles for highest-failure domains. +6. Replace runner input wiring so run-all consumes only the new synthesized invocation payload from discovery (single contract, no dual path). +7. Emit failure taxonomy in run summary: + - ARG_MISUSE + - INTERACTIVE_REQUIRED + - RESOURCE_PRECONDITION + - CONTRACT_SHAPE + - COMMAND_BUG + +Acceptance Criteria: +- Significant reduction in ARG_MISUSE failures versus 2026-07-29 baseline. +- Interactive-only commands are deterministically handled or explicitly classified. +- Stale flag/example failures drop to zero. +- Residual failures are clearer and mostly actionable as real command or contract issues. + +Technical Design Blueprint (implementation-ready): + +Greenfield rule for this task: +- No backward compatibility layer is required for discovery internals. +- Discovery and the run-all integration test must share one explicit new contract in this workstream. + +Hard constraints: +- No dual-path execution (old and new discovery paths in parallel) in final implementation. +- No compatibility adapters for legacy discovery output. +- Remove or retire superseded discovery wiring once the new contract is in place. + +1. Discovery Output Model (new normalized schema) +- Expand discovery output from plain runnable args to a normalized per-command model: + - commandId + - sourceFile + - commandTokens + - parsedArgs: positional args with required, default, and inferred placeholder kind + - parsedFlags: flags with required, type, options, multiple, default, exactlyOne, exclusive, dependsOn + - interactiveSignals: static interactive hints from source scan + - invocationProfilesApplied: profile IDs applied to this command +- Define a single synthesized invocation payload consumed by the integration runner. + +2. Static Metadata Extraction Strategy (no help command execution) +- Resolve command metadata directly from source and composition patterns: + - inline static args/static flags + - spread imports and shared flag sets (for example project/app/resource flag sets) + - flag factory invocations (for example flag definitions with required and constraint options) +- Stop relying on help output parsing completely. +- Add extraction diagnostics to report unresolved spreads/factories explicitly (instead of silently degrading). + +3. Deterministic Arg/Flag Synthesis Engine +- Build runnable invocation in fixed passes: + 1) seed command tokens + 2) satisfy required positional args + 3) satisfy required flags + 4) resolve exactly-one and exclusive groups + 5) close dependency edges (dependsOn) + 6) apply profile overrides +- Type-aware placeholder mapping (deterministic defaults): + - uuid/id -> 00000000-0000-4000-8000-000000000000 + - email -> integration@example.com + - url/uri -> https://example.com + - duration/ttl/interval -> 1h (or command-specific profile value) + - enum options -> first safe option unless profile overrides + - file/directory -> deterministic local temp-safe path values + +4. Constraint Resolution Rules (precedence) +- Precedence order when constraints collide: + 1) command-specific profile rule + 2) explicit example-derived value (if validated) + 3) deterministic global heuristic + 4) fail classification as ARG_MISUSE (never randomize) +- Exactly-one policy: + - choose the option with the smallest dependency closure + - prefer non-interactive branch when one branch requires interaction + - prefer context-compatible branch (project scoped over org scoped when project context exists) +- Exclusive policy: + - if both appear after merge, keep higher-precedence value and drop the other deterministically. + +5. Interactive Command Policy +- Static detection signals include: + - addInput, addSelect, addConfirmation usage + - editor fallback behavior + - stdin-dependent branches +- Decision contract per command: + - NON_INTERACTIVE_RESOLVED: inject supported non-interactive flags/values + - INTERACTIVE_REQUIRED: classify and skip execution attempt +- Baseline targeted substitutions: + - destructive confirmation flows -> --force where supported + - consent prompts -> --consent where supported + - password prompts -> explicit password flags where supported + - passphrase prompts -> no-passphrase where supported + - project-type selections -> explicit override-type where supported + +6. Example Validation and Stale Example Fallback +- Validate parsed examples against extracted schema before selecting them: + - unknown flags + - missing required args/flags + - exactly-one/exclusive/dependsOn violations +- If invalid, mark source as stale-example and synthesize from schema/profile. +- Add stale example counters to run summary. + +7. Command-Specific Invocation Profile Schema +- Add a small declarative profile registry keyed by commandId or prefix. +- Profile fields: + - match: exact commandId or prefix + - requiredFlagDefaults: map of flag -> value + - requiredArgDefaults: map of arg -> value + - exactlyOneChoice: map of constraint group -> preferred member + - interactivePolicy: resolve | classify + - disableExampleSource: boolean + - notes: short rationale +- Initial high-failure domains to profile first: + - backup create / backup schedule create + - cronjob create + - extension install / extension list-installed + - sftp-user create / ssh-user create + - database mysql create / mysql user create + - ddev init + - login token + +8. Failure Taxonomy Contract for Run Summary +- Emit machine-countable categories per command failure: + - ARG_MISUSE + - INTERACTIVE_REQUIRED + - RESOURCE_PRECONDITION + - CONTRACT_SHAPE + - COMMAND_BUG +- Add deterministic signature rules to classify stderr/stdout patterns. +- Print summary table with counts, plus sample command IDs per category. + +9. Delivery Phases and Validation Gates +- Phase A: schema extractor and fixture tests for representative commands. +- Phase B: synthesis engine with constraint solver. +- Phase C: interactive policy and initial profile registry. +- Phase D: example validation and stale fallback. +- Phase E: runner wiring switch to the new payload as the only path; remove superseded discovery wiring. +- Phase F: taxonomy emission and baseline comparison report. + +10. Definition of Success for this task +- ARG_MISUSE category reduced significantly from baseline. +- INTERACTIVE_REQUIRED is explicit and deterministic (no prompt-time crashes). +- Nonexistent/stale example flag failures are zero. +- Remaining failures are primarily RESOURCE_PRECONDITION, CONTRACT_SHAPE, or COMMAND_BUG and actionable. + +## Backlog Tasks (Reordered) +### Task 0: Mockoon Runtime Orchestration (Completed) +- Programmatic startup/teardown is in place and actively used. +- Automated Mockoon-backed full runs are already operational. +- The setup is conserved in a separate shareable repository. + +### Task 4: Dedicated Integration Scripts +- Add `test:integration:commands` and optional Mockoon-backed variant. +- Keep unit and existing test flows unchanged. + +### Task 5: CI Hardening +- Stabilize retries, resource isolation, and cleanup. +- Promote run-all matrix to optional or gated CI step once discovery quality improves. + +### Task 7: Contract-Shape Guardrails +- Add preflight checks and endpoint-shape diagnostics so contract mismatches are isolated from discovery quality metrics. + +### Task 8: Create waiver mechanics, move config to JSON (Completed) +- Status: Implemented (2026-07-30) +- Config now lives in visible JSON artifacts: + - `src/test/integration/config/invocation-profiles.json` + - `src/test/integration/config/command-waivers.json` +- Runner now enforces strict waiver mechanics: + - a command is skipped only when an explicit waiver exists + - every skipped command must have a non-empty reason and category + - interactive-required commands without waiver are treated as failures + - stale waivers (waiver for undiscovered command) fail the run + +### Task 9: Agent Behavior Overhaul (Artifact-First) +- Status: Planned (2026-07-31) +- Purpose: prevent heuristic drift and force deterministic, auditable follow-up analysis. + +Hard rules for analysis tooling: +- The integration runner is the only producer of command discovery facts. +- Follow-up analyzers must consume machine artifacts; they must not rediscover commands from source tree scans. +- No regex reconstruction of command identity when machine events already provide commandId and sourceFile. +- Standalone analyzer tools must not import test discovery modules that load runtime config from dist-relative paths. +- Generated outputs must always include provenance fields (log path, schema inputs, timestamp, tool version). + +Canonical data flow: +1. `run-all-commands.test.ts` emits NDJSON events. +2. `command-start` events provide discovery metadata and sourceFile. +3. `command-result` events provide status and failureCategory. +4. analyzer joins command metadata + result category + descriptor mapping + OpenAPI operation data. +5. analyzer writes deterministic artifacts (json + markdown). + +Required NDJSON contract for analyzer inputs: +- `command-start`: + - commandId + - sourceFile + - parsedArgs + - parsedFlags + - interactiveSignals + - invocationProfilesApplied + - extractionDiagnostics +- `command-result`: + - commandId + - status + - failureCategory + +Acceptance criteria: +- Analyzer fails fast when required machine-log fields are missing. +- Analyzer behavior is stable across repeated runs with identical inputs. +- Endpoint mapping is descriptor-first and OpenAPI-backed, without command rediscovery fallbacks. +- Category slices (for example RESOURCE_PRECONDITION) are computed from `command-result` events only. + +Operator workflow (human or agent): +1. Run integration matrix and produce NDJSON log. +2. Run command-endpoint analyzer against that NDJSON log. +3. Filter by failure category and review endpoint coverage/deprecation metadata. +4. Triage into: product bug, contract-shape issue, precondition gap, or invocation profile gap. +5. Update profiles/waivers/tooling, then rerun end-to-end. + +## Definition Of Done +- Integration command matrix runs against configurable HTTP target without in-test mocking. +- Discovery quality is improved enough that failures are predominantly real issues, not invocation noise. +- Failure taxonomy is reported and used for follow-up prioritization. diff --git a/package.json b/package.json index ad8dde446..f1eef1f36 100644 --- a/package.json +++ b/package.json @@ -38,7 +38,9 @@ "test:format": "yarn lint && yarn format:prettier --check", "test:licenses": "yarn license-check --summary --unknown --failOn 'UNLICENSED;UNKNOWN'", "test:readme": "yarn generate:readme && git diff --exit-code README.md docs/*.md", - "test:unit": "NODE_NO_WARNINGS=1 yarn node --experimental-vm-modules $(yarn bin jest) ./src" + "test:unit": "NODE_NO_WARNINGS=1 yarn node --experimental-vm-modules $(yarn bin jest) ./src", + "tool:integration:generate-command-endpoint-map": "yarn compile && node dist/test/integration/tools/generate-command-endpoint-map.js", + "tool:integration:generate-resource-precondition-map": "yarn compile && node dist/test/integration/tools/generate-command-endpoint-map.js --category RESOURCE_PRECONDITION" }, "files": [ ".deps", diff --git a/src/commands/app/database/link.tsx b/src/commands/app/database/link.tsx index 927c623cf..826eaeb36 100644 --- a/src/commands/app/database/link.tsx +++ b/src/commands/app/database/link.tsx @@ -71,7 +71,7 @@ export default class Link extends ExecRenderBaseCommand { }); await process.runStep("linking database", async () => { - const response = await this.apiClient.app.linkDatabase({ + const response = await this.apiClient.app.linkDatabase({ // XXX: deprecated?! Should use UPDATE on app installation instead! appInstallationId, data: { databaseId, diff --git a/src/commands/backup/download.tsx b/src/commands/backup/download.tsx index 3fd1e318a..b6e2ef750 100644 --- a/src/commands/backup/download.tsx +++ b/src/commands/backup/download.tsx @@ -135,7 +135,7 @@ export class Download extends ExecRenderBaseCommand { } return null; - }, Duration.fromString("1h")); + }, Duration.fromString("1h")); // XXX: may i have a word here, too?! }, ); diff --git a/src/commands/conversation/show.test.ts b/src/commands/conversation/show.test.ts index f292efbc1..2f28d3d59 100644 --- a/src/commands/conversation/show.test.ts +++ b/src/commands/conversation/show.test.ts @@ -1,118 +1,52 @@ -import { runCommand } from "@oclif/test"; -import { MittwaldAPIV2 } from "@mittwald/api-client"; -import nock from "nock"; import { afterEach, beforeEach, describe, expect, it } from "@jest/globals"; - -type Conversation = MittwaldAPIV2.Components.Schemas.ConversationConversation; -type Message = MittwaldAPIV2.Components.Schemas.ConversationMessage; -type StatusUpdate = MittwaldAPIV2.Components.Schemas.ConversationStatusUpdate; +import { runDevCommand } from "../../test/integration/command.js"; +import { + configureIntegrationEnv, + restoreEnv, + snapshotEnv, +} from "../../test/integration/env.js"; + +function normalizeOutput(output: string): string { + return output + .replace(/\u001b\[[0-9;]*m/g, "") + .replace(/\r/g, "") + .trim(); +} describe("conversation:show", () => { - const conversationId = "186f8f22-aa0f-42bf-909d-757cb9d27b04"; - const userId = "6dbd84b5-74e0-43ed-8a81-b0b8a0405a47"; - const messageId = "10a59409-ff2d-478e-b07f-72c8f9f5b63f"; - const now = new Date(); - const user = { - userId, - clearName: "John Doe", - }; + const fallbackConversationId = "186f8f22-aa0f-42bf-909d-757cb9d27b04"; let originalEnv: NodeJS.ProcessEnv; beforeEach(() => { - originalEnv = { ...process.env }; - process.env["MITTWALD_API_TOKEN"] = "foo"; - - nock.disableNetConnect(); + originalEnv = snapshotEnv(); + configureIntegrationEnv("conversation:show"); }); afterEach(() => { - process.env = originalEnv; - nock.cleanAll(); - }); - - it("should test", () => { - expect(true).toBeTruthy(); + restoreEnv(originalEnv); }); - // skipped, to be fixed later - it.skip("shows a conversation and its messages", async () => { - const scope = nock("https://api.mittwald.de") - .get(`/v2/conversations/${conversationId}`) - .reply(200, { - conversationId, - shortId: "CONV-ID", - createdAt: now.toJSON(), - title: "Test conversation", - status: "open", - visibility: "shared", - mainUser: user, - } satisfies Conversation) - .get(`/v2/conversations/${conversationId}/messages`) - .reply(200, [ - { - conversationId, - type: "STATUS_UPDATE", - createdAt: now.toJSON(), - meta: { user }, - messageContent: "CONVERSATION_CREATED", - }, - { - messageId, - conversationId, - type: "MESSAGE", - createdAt: now.toJSON(), - createdBy: user, - messageContent: "Hello, World!", - }, - { - conversationId, - type: "STATUS_UPDATE", - createdAt: now.toJSON(), - meta: { user }, - messageContent: "STATUS_CLOSED", - }, - ] satisfies Array); + it("shows a conversation and its messages", async () => { + const conversationId = + process.env["MW_TEST_CONVERSATION_ID"] ?? fallbackConversationId; - console.log("foo"); + const { stdout, stderr, error, timedOut } = await runDevCommand( + ["conversation", "show", conversationId], + { + timeoutMs: 25_000, + }, + ); - const { stdout, stderr, error } = await runCommand([ - "conversation:show", - conversationId, - ]); + expect(timedOut).toBeFalsy(); - console.log("foo"); - - setTimeout(() => scope.done(), 5000); - - expect(stdout).toEqual(""); - expect(stderr).toEqual(""); expect(error).toBeUndefined(); - }); -}); - -/* - - api - .env({ MITTWALD_API_TOKEN: "foo" }) - .stdout() - .command(["conversation show", conversationId]) - .it("shows a conversation and its messages", (ctx) => { - expect(ctx.stdout.trim()).to.equal(`Conversation metadata -───────────────────── - -Title Test conversation -ID CONV-ID -Opened less than a minute ago by Unknown User -Status open -Messages -──────── + const output = normalizeOutput(`${stdout}\n${stderr}`); -CREATED, less than a minute ago - -John Doe, less than a minute ago -Hello, World! - -CLOSED, less than a minute ago`); - });*/ + expect(output).toContain("Conversation metadata"); + expect(output).toContain("Messages"); + expect(output).toMatch(/ID\s+\S+/); + expect(output).toMatch(/Status\s+\S+/i); + }, 30_000); +}); diff --git a/src/commands/database/mysql/create.test.ts b/src/commands/database/mysql/create.test.ts index 0f9d81283..6fa6da495 100644 --- a/src/commands/database/mysql/create.test.ts +++ b/src/commands/database/mysql/create.test.ts @@ -34,8 +34,7 @@ describe("database:mysql:create", () => { nock.cleanAll(); }); - // Skipped, to be fixed later - it.skip("creates a database and prints database and user name", async () => { + it("creates a database and prints database and user name", async () => { const scope = nock("https://api.mittwald.de"); scope.get(`/v2/projects/${projectId}`).reply(200, { @@ -86,7 +85,7 @@ describe("database:mysql:create", () => { }); // Skipped, to be fixed later - it.skip("retries fetching user until successful", async () => { + it("retries fetching user until successful", async () => { const scope = nock("https://api.mittwald.de"); scope.get(`/v2/projects/${projectId}`).reply(200, { diff --git a/src/test/integration/classification-catalog.ts b/src/test/integration/classification-catalog.ts new file mode 100644 index 000000000..fb7063c24 --- /dev/null +++ b/src/test/integration/classification-catalog.ts @@ -0,0 +1,225 @@ +import { readFile, writeFile } from "node:fs/promises"; +import path from "node:path"; +import type { WaiverCategory } from "./command-discovery/types.js"; + +export type FailureCategory = WaiverCategory; + +export type ClassificationEntrySource = "failure" | "waiver" | "skip"; + +export type CommandClassificationEntry = { + commandId: string; + category: FailureCategory; + source: ClassificationEntrySource; +}; + +export type CommandClassificationCatalog = { + schemaVersion: 1; + generatedAt: string; + source: { + kind: "run-all-summary" | "log-extract"; + path?: string; + }; + statistics: { + successful: number; + failed: number; + waivedSkipped: number; + total: number; + }; + entries: CommandClassificationEntry[]; +}; + +export const FAILURE_CATEGORIES: FailureCategory[] = [ + "ARG_MISUSE", + "INTERACTIVE_REQUIRED", + "RESOURCE_PRECONDITION", + "CONTRACT_SHAPE", + "COMMAND_BUG", +]; + +export function isFailureCategory(value: string): value is FailureCategory { + return FAILURE_CATEGORIES.includes(value as FailureCategory); +} + +export function parseFailureCategory(value: string): FailureCategory { + if (!isFailureCategory(value)) { + throw new Error( + `Invalid category '${value}'. Allowed categories: ${FAILURE_CATEGORIES.join(", ")}`, + ); + } + + return value; +} + +export function createFailureBuckets(): Record { + return { + ARG_MISUSE: [], + INTERACTIVE_REQUIRED: [], + RESOURCE_PRECONDITION: [], + CONTRACT_SHAPE: [], + COMMAND_BUG: [], + }; +} + +export function getDefaultClassificationCatalogPath(): string { + return path.resolve( + process.cwd(), + "src/test/integration/config/command-classifications.json", + ); +} + +export async function loadClassificationCatalog( + catalogPath = getDefaultClassificationCatalogPath(), +): Promise { + const raw = await readFile(catalogPath, "utf8"); + const parsed = JSON.parse(raw) as CommandClassificationCatalog; + return parsed; +} + +export async function saveClassificationCatalog( + catalog: CommandClassificationCatalog, + catalogPath = getDefaultClassificationCatalogPath(), +): Promise { + await writeFile(catalogPath, `${JSON.stringify(catalog, null, 2)}\n`, "utf8"); +} + +export function buildClassificationCatalogFromBuckets(input: { + failuresByCategory: Record; + waivedByCategory: Record; + statistics: { + successful: number; + failed: number; + waivedSkipped: number; + total: number; + }; + generatedAt?: string; +}): CommandClassificationCatalog { + const entryMap = new Map(); + + for (const category of FAILURE_CATEGORIES) { + for (const commandId of input.failuresByCategory[category]) { + entryMap.set(commandId, { + commandId, + category, + source: "failure", + }); + } + } + + for (const category of FAILURE_CATEGORIES) { + for (const commandId of input.waivedByCategory[category]) { + if (entryMap.has(commandId)) { + continue; + } + + entryMap.set(commandId, { + commandId, + category, + source: "waiver", + }); + } + } + + return { + schemaVersion: 1, + generatedAt: input.generatedAt ?? new Date().toISOString(), + source: { + kind: "run-all-summary", + }, + statistics: input.statistics, + entries: [...entryMap.values()].sort((a, b) => + a.commandId.localeCompare(b.commandId), + ), + }; +} + +export function extractClassificationCatalogFromRunLog(input: { + logContent: string; + logPath?: string; +}): CommandClassificationCatalog { + const entryMap = new Map(); + + const classifiedRegex = + /^\[(\d+)\/(\d+)\] classified (.+) as (ARG_MISUSE|INTERACTIVE_REQUIRED|RESOURCE_PRECONDITION|CONTRACT_SHAPE|COMMAND_BUG)$/m; + const waivedRegex = + /^\[(\d+)\/(\d+)\] waived (.+) \(category=(ARG_MISUSE|INTERACTIVE_REQUIRED|RESOURCE_PRECONDITION|CONTRACT_SHAPE|COMMAND_BUG)(?:;|\))/m; + const skippedInteractiveRegex = + /^\[(\d+)\/(\d+)\] skipped (.+) \(interactive required\)$/m; + + for (const line of input.logContent.split(/\r?\n/)) { + const classified = line.match(classifiedRegex); + if (classified) { + const commandId = classified[3].trim(); + const category = classified[4] as FailureCategory; + entryMap.set(commandId, { + commandId, + category, + source: "failure", + }); + continue; + } + + const waived = line.match(waivedRegex); + if (waived) { + const commandId = waived[3].trim(); + const category = waived[4] as FailureCategory; + entryMap.set(commandId, { + commandId, + category, + source: "waiver", + }); + continue; + } + + const skippedInteractive = line.match(skippedInteractiveRegex); + if (skippedInteractive) { + const commandId = skippedInteractive[3].trim(); + entryMap.set(commandId, { + commandId, + category: "INTERACTIVE_REQUIRED", + source: "skip", + }); + } + } + + const stats = parseStatistics(input.logContent); + + return { + schemaVersion: 1, + generatedAt: new Date().toISOString(), + source: { + kind: "log-extract", + path: input.logPath, + }, + statistics: stats, + entries: [...entryMap.values()].sort((a, b) => + a.commandId.localeCompare(b.commandId), + ), + }; +} + +function parseStatistics(logContent: string): { + successful: number; + failed: number; + waivedSkipped: number; + total: number; +} { + const statsRegex = + /\[run-all\] statistics: successful=(\d+), failed=(\d+), (?:waived-skipped|interactive-skipped)=(\d+), total=(\d+)/; + + const match = logContent.match(statsRegex); + if (!match) { + return { + successful: 0, + failed: 0, + waivedSkipped: 0, + total: 0, + }; + } + + return { + successful: Number.parseInt(match[1], 10), + failed: Number.parseInt(match[2], 10), + waivedSkipped: Number.parseInt(match[3], 10), + total: Number.parseInt(match[4], 10), + }; +} diff --git a/src/test/integration/command-discovery.ts b/src/test/integration/command-discovery.ts new file mode 100644 index 000000000..0501a39cb --- /dev/null +++ b/src/test/integration/command-discovery.ts @@ -0,0 +1,156 @@ +import { readdir, readFile } from "node:fs/promises"; +import path from "node:path"; +import { + type FailureCategory, + loadClassificationCatalog, +} from "./classification-catalog.js"; +import { + detectInteractiveSignals, + extractArgsSchema, + extractExampleCandidate, + extractFlagsSchema, +} from "./command-discovery/parsing.js"; +import { resolveProfiles, synthesizeInvocation } from "./command-discovery/synthesis.js"; +import type { DiscoveredCommand } from "./command-discovery/types.js"; + +export type { + DiscoveredCommand, + FlagValueType, + InteractiveSignal, + InvocationProfile, + ParsedArg, + ParsedFlag, + PlaceholderKind, + SynthesizedInvocation, + ValueSource, +} from "./command-discovery/types.js"; + +const COMMAND_FILE_EXTENSION_REGEX = /\.(ts|tsx)$/; +const NON_COMMAND_FILE_REGEX = /\.test\.(ts|tsx)$/; + +export type DiscoverCommandsOptions = { + commandsRoot?: string; + onProgress?: (message: string) => void; + categoryFilter?: FailureCategory; + classificationCatalogPath?: string; +}; + +export async function discoverRunnableCommands( + options: DiscoverCommandsOptions = {}, +): Promise { + const commandsRoot = + options.commandsRoot ?? path.resolve(process.cwd(), "src/commands"); + const onProgress = options.onProgress; + const categoryFilter = options.categoryFilter; + + const commandFiles = await collectCommandFiles(commandsRoot); + const discovered: DiscoveredCommand[] = []; + + onProgress?.( + `[discovery] found ${commandFiles.length} command source files under ${commandsRoot}`, + ); + + for (const [index, filePath] of commandFiles.entries()) { + const source = await readFile(filePath, "utf8"); + const relativePath = path.relative(commandsRoot, filePath); + const commandId = toCommandId(relativePath); + const commandTokens = commandId.split(" "); + const position = `${index + 1}/${commandFiles.length}`; + const extractionDiagnostics: string[] = []; + const profiles = resolveProfiles(commandId); + + onProgress?.(`[discovery:${position}] scanning ${commandId}`); + + const parsedArgs = extractArgsSchema(source, extractionDiagnostics); + const parsedFlags = extractFlagsSchema(source, extractionDiagnostics); + const interactiveSignals = detectInteractiveSignals(source); + const exampleCandidate = profiles.some((profile) => profile.disableExampleSource) + ? undefined + : extractExampleCandidate(source, commandId); + + const synthesizedInvocation = synthesizeInvocation({ + commandId, + commandTokens, + parsedArgs, + parsedFlags, + interactiveSignals, + exampleCandidate, + profiles, + }); + + discovered.push({ + commandId, + sourceFile: relativePath, + commandTokens, + parsedArgs, + parsedFlags, + interactiveSignals, + invocationProfilesApplied: profiles.map((profile) => profile.id), + extractionDiagnostics, + synthesizedInvocation, + }); + + onProgress?.( + `[discovery:${position}] ${commandId} -> ${synthesizedInvocation.argumentSource}${synthesizedInvocation.staleExample ? " (stale-example-fallback)" : ""}`, + ); + } + + const sorted = discovered.sort((a, b) => a.commandId.localeCompare(b.commandId)); + + if (!categoryFilter) { + onProgress?.(`[discovery] completed ${sorted.length} commands`); + return sorted; + } + + const classificationCatalog = await loadClassificationCatalog( + options.classificationCatalogPath, + ); + + const selectedCommandIds = new Set( + classificationCatalog.entries + .filter((entry) => entry.category === categoryFilter) + .map((entry) => entry.commandId), + ); + + const filtered = sorted.filter((command) => selectedCommandIds.has(command.commandId)); + + onProgress?.( + `[discovery] completed ${sorted.length} commands; category filter ${categoryFilter} => ${filtered.length}`, + ); + + return filtered; +} + +async function collectCommandFiles(rootDir: string): Promise { + const entries = await readdir(rootDir, { withFileTypes: true }); + const files = await Promise.all( + entries.map(async (entry) => { + const fullPath = path.join(rootDir, entry.name); + + if (entry.isDirectory()) { + return await collectCommandFiles(fullPath); + } + + if (!entry.isFile()) { + return []; + } + + if (!COMMAND_FILE_EXTENSION_REGEX.test(entry.name)) { + return []; + } + + if (NON_COMMAND_FILE_REGEX.test(entry.name)) { + return []; + } + + return [fullPath]; + }), + ); + + return files.flat(); +} + +function toCommandId(relativeFilePath: string): string { + const withoutExtension = relativeFilePath.replace(COMMAND_FILE_EXTENSION_REGEX, ""); + return withoutExtension.split(path.sep).join(" "); +} diff --git a/src/test/integration/command-discovery/config.ts b/src/test/integration/command-discovery/config.ts new file mode 100644 index 000000000..d8bc27935 --- /dev/null +++ b/src/test/integration/command-discovery/config.ts @@ -0,0 +1,194 @@ +import type { ParsedArg, ParsedFlag } from "./types.js"; + +export const DEFAULT_UUID = "00000000-0000-4000-8000-000000000000"; + +export const SHARED_FLAG_SCHEMAS: Record = { + processFlags: [ + { + name: "quiet", + required: false, + type: "boolean", + takesValue: false, + defaultValue: "false", + }, + ], + projectFlags: [ + { + name: "project-id", + required: false, + type: "string", + takesValue: true, + }, + ], + appInstallationFlags: [ + { + name: "installation-id", + required: false, + type: "string", + takesValue: true, + }, + ], + waitFlags: [ + { + name: "wait", + required: false, + type: "boolean", + takesValue: false, + }, + { + name: "wait-timeout", + required: false, + type: "string", + takesValue: true, + defaultValue: "10m", + }, + ], + ddevFlags: [ + { + name: "override-type", + required: false, + type: "string", + takesValue: true, + defaultValue: "auto", + options: ["auto"], + }, + { + name: "database-id", + required: false, + type: "string", + takesValue: true, + exclusive: ["without-database"], + }, + { + name: "without-database", + required: false, + type: "boolean", + takesValue: false, + exclusive: ["database-id"], + }, + ], + pathMappingFlags: [ + { + name: "path-to-app", + required: false, + type: "string", + takesValue: true, + multiple: true, + }, + { + name: "path-to-url", + required: false, + type: "string", + takesValue: true, + multiple: true, + }, + { + name: "path-to-container", + required: false, + type: "string", + takesValue: true, + multiple: true, + }, + ], +}; + +export const SHARED_ARG_SCHEMAS: Record = { + appInstallationArgs: [ + { + name: "installation-id", + required: true, + placeholderKind: "uuid", + }, + ], + backupArgs: [ + { + name: "backup-id", + required: true, + placeholderKind: "uuid", + }, + ], + mysqlArgs: [ + { + name: "database-id", + required: true, + placeholderKind: "uuid", + }, + ], + redisArgs: [ + { + name: "database-id", + required: true, + placeholderKind: "uuid", + }, + ], + dnsZoneArgs: [ + { + name: "dnszone-id", + required: true, + placeholderKind: "generic", + }, + ], + conversationArgs: [ + { + name: "conversation-id", + required: true, + placeholderKind: "uuid", + }, + ], + orgArgs: [ + { + name: "org-id", + required: true, + placeholderKind: "uuid", + }, + ], + domainArgs: [ + { + name: "domain-id", + required: true, + placeholderKind: "generic", + }, + ], + mailAddressArgs: [ + { + name: "mailaddress-id", + required: true, + placeholderKind: "generic", + }, + ], + mailDeliveryBoxArgs: [ + { + name: "maildeliverybox-id", + required: true, + placeholderKind: "uuid", + }, + ], + stackArgs: [ + { + name: "stack-id", + required: true, + placeholderKind: "uuid", + }, + ], +}; + +export const NAMED_FLAG_SCHEMAS: Record> = { + adminUserIdFlag: { + required: true, + type: "string", + takesValue: true, + }, + databasePurposeFlag: { + required: true, + type: "string", + takesValue: true, + options: ["primary", "cache", "custom"], + defaultValue: "primary", + }, + databasePurposeSelectorFlag: { + required: false, + type: "string", + takesValue: true, + options: ["primary", "cache", "custom"], + }, +}; diff --git a/src/test/integration/command-discovery/parsing.ts b/src/test/integration/command-discovery/parsing.ts new file mode 100644 index 000000000..6c9153ae0 --- /dev/null +++ b/src/test/integration/command-discovery/parsing.ts @@ -0,0 +1,930 @@ +import { + NAMED_FLAG_SCHEMAS, + SHARED_ARG_SCHEMAS, + SHARED_FLAG_SCHEMAS, +} from "./config.js"; +import type { + ExampleCandidate, + FlagValueType, + InteractiveSignal, + ParsedArg, + ParsedFlag, + PlaceholderKind, +} from "./types.js"; + +export function extractExampleCandidate( + source: string, + commandId: string, +): ExampleCandidate | undefined { + const examplesMatch = source.match(/static\s+examples\s*=\s*\[([\s\S]*?)\];/m); + if (!examplesMatch) { + return undefined; + } + + const block = examplesMatch[1]; + const commandStrings: string[] = []; + + const objectCommandRegex = + /command\s*:\s*("(?:[^"\\]|\\.)*"|'(?:[^'\\]|\\.)*'|`[\s\S]*?`)/g; + + let objectMatch = objectCommandRegex.exec(block); + while (objectMatch) { + const decoded = decodeStringLiteral(objectMatch[1]); + if (decoded) { + commandStrings.push(decoded); + } + + objectMatch = objectCommandRegex.exec(block); + } + + if (commandStrings.length === 0) { + const stringLiteralRegex = /("(?:[^"\\]|\\.)*"|'(?:[^'\\]|\\.)*'|`[\s\S]*?`)/g; + let stringMatch = stringLiteralRegex.exec(block); + while (stringMatch) { + const decoded = decodeStringLiteral(stringMatch[1]); + if (decoded && (decoded.includes("<%= command.id %>") || decoded.includes("mw "))) { + commandStrings.push(decoded); + } + + stringMatch = stringLiteralRegex.exec(block); + } + } + + for (const commandString of commandStrings) { + const args = parseExampleCommandToArgs(commandString, commandId); + if (!args) { + continue; + } + + const { positionalValues, flagValues } = parseInvocationParts(args.slice(commandId.split(" ").length)); + return { + args, + positionalValues, + flagValues, + }; + } + + return undefined; +} + +export function extractArgsSchema(source: string, diagnostics: string[]): ParsedArg[] { + const block = extractStaticObjectBlock(source, /static\s+args\s*=\s*{/m); + if (!block) { + return []; + } + + const entries = splitTopLevelEntries(block); + const args = new Map(); + + for (const entry of entries) { + const spread = entry.match(/^\.\.\.\s*([A-Za-z0-9_$.]+)\s*$/); + if (spread) { + const spreadName = spread[1].split(".").at(-1) ?? spread[1]; + const sharedArgs = SHARED_ARG_SCHEMAS[spreadName]; + if (sharedArgs) { + for (const sharedArg of sharedArgs) { + args.set(sharedArg.name, sharedArg); + } + continue; + } + + const localArgs = parseLocalArgObject(source, spreadName, diagnostics); + if (localArgs.length > 0) { + for (const localArg of localArgs) { + args.set(localArg.name, localArg); + } + continue; + } + + diagnostics.push(`args: unresolved spread '${spread[1]}'`); + continue; + } + + const split = splitObjectEntry(entry); + if (!split) { + continue; + } + + const config = extractFirstObjectLiteral(split.expression); + const required = readBooleanProp(config, "required") ?? false; + const defaultValue = readStringProp(config, "default"); + + args.set(split.key, { + name: split.key, + required, + defaultValue, + placeholderKind: inferPlaceholderKind(split.key), + }); + } + + if (args.size === 0) { + diagnostics.push("args: no statically extractable arg entries"); + } + + return [...args.values()]; +} + +export function extractFlagsSchema(source: string, diagnostics: string[]): ParsedFlag[] { + const block = extractStaticObjectBlock(source, /static\s+flags\s*=\s*{/m); + if (!block) { + diagnostics.push("flags: static flags block not found"); + return []; + } + + const entries = splitTopLevelEntries(block); + const flags = new Map(); + + for (const entry of entries) { + const factorySpreadFlags = parseFlagSpreadFactory(entry); + if (factorySpreadFlags.length > 0) { + for (const flag of factorySpreadFlags) { + flags.set(flag.name, flag); + } + continue; + } + + const spread = entry.match(/^\.\.\.\s*([A-Za-z0-9_$.]+)\s*$/); + if (spread) { + const spreadName = spread[1].split(".").at(-1) ?? spread[1]; + const shared = SHARED_FLAG_SCHEMAS[spreadName]; + if (shared) { + for (const flag of shared) { + flags.set(flag.name, flag); + } + continue; + } + + const localFlags = parseLocalFlagObject(source, spreadName, diagnostics); + if (localFlags.length > 0) { + for (const localFlag of localFlags) { + flags.set(localFlag.name, localFlag); + } + continue; + } + + diagnostics.push(`flags: unresolved spread '${spread[1]}'`); + continue; + } + + const split = splitObjectEntry(entry); + if (!split) { + diagnostics.push(`flags: could not parse entry '${entry.trim().slice(0, 80)}'`); + continue; + } + + const parsedFlag = parseFlagDefinition(split.key, split.expression); + if (!parsedFlag) { + diagnostics.push(`flags: unresolved factory for '${split.key}'`); + continue; + } + + flags.set(parsedFlag.name, parsedFlag); + } + + return [...flags.values()]; +} + +export function detectInteractiveSignals(source: string): InteractiveSignal[] { + const signals: InteractiveSignal[] = []; + const withSignal = (signal: InteractiveSignal, regex: RegExp) => { + if (regex.test(source)) { + signals.push(signal); + } + }; + + withSignal("addInput", /\.addInput\s*\(/); + withSignal("addSelect", /\.addSelect\s*\(/); + withSignal("addConfirmation", /\.addConfirmation\s*\(/); + withSignal("editorFallback", /editor|openEditor/i); + withSignal("stdinBranch", /stdin|process\.stdin/i); + + return [...new Set(signals)]; +} + +function parseFlagSpreadFactory(entry: string): ParsedFlag[] { + const expireFlagsMatch = entry.match( + /^\.\.\.\s*expireFlags\(\s*[^,]+,\s*(true|false)\s*\)\s*$/, + ); + + if (expireFlagsMatch) { + return [ + { + name: "expires", + required: expireFlagsMatch[1] === "true", + type: "string", + takesValue: true, + }, + ]; + } + + return []; +} + +function parseLocalArgObject( + source: string, + objectName: string, + diagnostics: string[], +): ParsedArg[] { + const block = extractConstObjectBlock(source, objectName); + if (!block) { + return []; + } + + const entries = splitTopLevelEntries(block); + const args = new Map(); + + for (const entry of entries) { + const spread = entry.match(/^\.\.\.\s*([A-Za-z0-9_$.]+)\s*$/); + if (spread) { + const spreadName = spread[1].split(".").at(-1) ?? spread[1]; + const shared = SHARED_ARG_SCHEMAS[spreadName]; + if (shared) { + for (const sharedArg of shared) { + args.set(sharedArg.name, sharedArg); + } + } + continue; + } + + const split = splitObjectEntry(entry); + if (!split) { + continue; + } + + const config = extractFirstObjectLiteral(split.expression); + const required = readBooleanProp(config, "required") ?? false; + const defaultValue = readStringProp(config, "default"); + + args.set(split.key, { + name: split.key, + required, + defaultValue, + placeholderKind: inferPlaceholderKind(split.key), + }); + } + + if (args.size === 0) { + diagnostics.push(`args: local spread '${objectName}' contained no extractable args`); + } + + return [...args.values()]; +} + +function parseLocalFlagObject( + source: string, + objectName: string, + diagnostics: string[], +): ParsedFlag[] { + const block = extractConstObjectBlock(source, objectName); + if (!block) { + return []; + } + + const entries = splitTopLevelEntries(block); + const flags = new Map(); + + for (const entry of entries) { + const spread = entry.match(/^\.\.\.\s*([A-Za-z0-9_$.]+)\s*$/); + if (spread) { + continue; + } + + const split = splitObjectEntry(entry); + if (!split) { + continue; + } + + const parsed = parseFlagDefinition(split.key, split.expression); + if (parsed) { + flags.set(parsed.name, parsed); + } + } + + if (flags.size === 0) { + diagnostics.push(`flags: local spread '${objectName}' contained no extractable flags`); + } + + return [...flags.values()]; +} + +function parseExampleCommandToArgs( + example: string, + commandId: string, +): string[] | undefined { + const rendered = example + .replace(/<%=\s*config\.bin\s*%>/g, "mw") + .replace(/<%=\s*command\.id\s*%>/g, commandId); + + const commandLine = rendered + .split("\n") + .map((line) => line.trim()) + .filter((line) => line.length > 0 && !line.startsWith("#")) + .find((line) => line.includes(commandId) || line.startsWith("mw ")); + + if (!commandLine) { + return undefined; + } + + const tokens = shellTokenize(commandLine.replace(/^\$\s*/, "")); + const normalizedTokens = tokens.filter((token) => token !== "mw"); + const commandTokens = commandId.split(" "); + const commandStart = findTokenSequenceIndex(normalizedTokens, commandTokens); + + if (commandStart === -1) { + return undefined; + } + + const rawInvocation = normalizedTokens.slice(commandStart); + return rawInvocation.map((token) => { + if (token.startsWith("<") && token.endsWith(">")) { + return makeTypedPlaceholderValue(token.slice(1, -1), "string", undefined); + } + + return token; + }); +} + +function findTokenSequenceIndex(haystack: string[], needle: string[]): number { + if (needle.length === 0 || haystack.length < needle.length) { + return -1; + } + + for (let i = 0; i <= haystack.length - needle.length; i += 1) { + const segment = haystack.slice(i, i + needle.length); + if (segment.every((token, idx) => token === needle[idx])) { + return i; + } + } + + return -1; +} + +function decodeStringLiteral(value: string): string | undefined { + const quote = value[0]; + if ((quote !== '"' && quote !== "'" && quote !== "`") || value.length < 2) { + return undefined; + } + + const inner = value.slice(1, -1); + return inner + .replace(/\\n/g, "\n") + .replace(/\\t/g, "\t") + .replace(/\\"/g, '"') + .replace(/\\'/g, "'") + .replace(/\\`/g, "`") + .replace(/\\\\/g, "\\"); +} + +function shellTokenize(value: string): string[] { + const matches = value.match(/"([^"\\]|\\.)*"|'([^'\\]|\\.)*'|\S+/g); + if (!matches) { + return []; + } + + return matches.map((token) => { + if ( + (token.startsWith('"') && token.endsWith('"')) || + (token.startsWith("'") && token.endsWith("'")) + ) { + return token.slice(1, -1); + } + + return token; + }); +} + +function makeTypedPlaceholderValue( + name: string, + type: FlagValueType, + options: string[] | undefined, +): string { + if (options && options.length > 0) { + return options[0]; + } + + const normalized = name + .replace(/[<>[\]]/g, "") + .replace(/[^A-Za-z0-9-]+/g, "-") + .replace(/^-+/, "") + .replace(/-+$/, "") + .toLowerCase(); + + if (normalized.includes("uuid") || normalized.endsWith("id") || normalized.includes("-id")) { + return "00000000-0000-4000-8000-000000000000"; + } + + if (normalized.includes("email")) { + return "integration@example.com"; + } + + if (normalized.includes("url") || normalized.includes("uri")) { + return "https://example.com"; + } + + if ( + normalized.includes("duration") || + normalized.includes("ttl") || + normalized.includes("interval") + ) { + return "1h"; + } + + if (normalized.includes("directory") || normalized.includes("path")) { + return "/tmp/mw-integration"; + } + + if (type === "file") { + return "/tmp/mw-integration.file"; + } + + if (type === "directory") { + return "/tmp/mw-integration"; + } + + if (normalized.includes("password")) { + return "integration-password"; + } + + if (normalized.includes("port")) { + return "12345"; + } + + return normalized.length > 0 ? `example-${normalized}` : "example-value"; +} + +function extractStaticObjectBlock(source: string, anchor: RegExp): string | undefined { + const match = anchor.exec(source); + if (!match) { + return undefined; + } + + const start = source.indexOf("{", match.index); + if (start === -1) { + return undefined; + } + + const end = findMatchingBraceIndex(source, start); + if (end === -1) { + return undefined; + } + + return source.slice(start + 1, end); +} + +function extractConstObjectBlock(source: string, objectName: string): string | undefined { + const anchor = new RegExp(`(?:const|let|var)\\s+${escapeRegExp(objectName)}\\s*=\\s*{`, "m"); + const match = anchor.exec(source); + if (!match) { + return undefined; + } + + const start = source.indexOf("{", match.index); + if (start === -1) { + return undefined; + } + + const end = findMatchingBraceIndex(source, start); + if (end === -1) { + return undefined; + } + + return source.slice(start + 1, end); +} + +function findMatchingBraceIndex(input: string, startIndex: number): number { + let depth = 0; + let quote: "'" | '"' | "`" | undefined; + let escaped = false; + + for (let i = startIndex; i < input.length; i += 1) { + const char = input[i]; + + if (quote) { + if (escaped) { + escaped = false; + continue; + } + + if (char === "\\") { + escaped = true; + continue; + } + + if (char === quote) { + quote = undefined; + } + + continue; + } + + if (char === '"' || char === "'" || char === "`") { + quote = char; + continue; + } + + if (char === "{") { + depth += 1; + continue; + } + + if (char === "}") { + depth -= 1; + if (depth === 0) { + return i; + } + } + } + + return -1; +} + +function splitTopLevelEntries(input: string): string[] { + const entries: string[] = []; + let start = 0; + let braceDepth = 0; + let parenDepth = 0; + let bracketDepth = 0; + let quote: "'" | '"' | "`" | undefined; + let escaped = false; + + for (let i = 0; i < input.length; i += 1) { + const char = input[i]; + + if (quote) { + if (escaped) { + escaped = false; + continue; + } + + if (char === "\\") { + escaped = true; + continue; + } + + if (char === quote) { + quote = undefined; + } + + continue; + } + + if (char === '"' || char === "'" || char === "`") { + quote = char; + continue; + } + + if (char === "{") { + braceDepth += 1; + continue; + } + + if (char === "}") { + braceDepth -= 1; + continue; + } + + if (char === "(") { + parenDepth += 1; + continue; + } + + if (char === ")") { + parenDepth -= 1; + continue; + } + + if (char === "[") { + bracketDepth += 1; + continue; + } + + if (char === "]") { + bracketDepth -= 1; + continue; + } + + if (char === "," && braceDepth === 0 && parenDepth === 0 && bracketDepth === 0) { + const part = input.slice(start, i).trim(); + if (part.length > 0) { + entries.push(part); + } + start = i + 1; + } + } + + const tail = input.slice(start).trim(); + if (tail.length > 0) { + entries.push(tail); + } + + return entries; +} + +function splitObjectEntry(entry: string): { key: string; expression: string } | undefined { + let quote: "'" | '"' | "`" | undefined; + let escaped = false; + let braceDepth = 0; + let parenDepth = 0; + let bracketDepth = 0; + + for (let i = 0; i < entry.length; i += 1) { + const char = entry[i]; + + if (quote) { + if (escaped) { + escaped = false; + continue; + } + + if (char === "\\") { + escaped = true; + continue; + } + + if (char === quote) { + quote = undefined; + } + + continue; + } + + if (char === '"' || char === "'" || char === "`") { + quote = char; + continue; + } + + if (char === "{") { + braceDepth += 1; + continue; + } + + if (char === "}") { + braceDepth -= 1; + continue; + } + + if (char === "(") { + parenDepth += 1; + continue; + } + + if (char === ")") { + parenDepth -= 1; + continue; + } + + if (char === "[") { + bracketDepth += 1; + continue; + } + + if (char === "]") { + bracketDepth -= 1; + continue; + } + + if (char === ":" && braceDepth === 0 && parenDepth === 0 && bracketDepth === 0) { + const keyRaw = entry.slice(0, i).trim(); + const expression = entry.slice(i + 1).trim(); + const key = keyRaw.replace(/^['"]/, "").replace(/['"]$/, ""); + if (!key || !expression) { + return undefined; + } + + return { key, expression }; + } + } + + return undefined; +} + +function parseFlagDefinition(name: string, expression: string): ParsedFlag | undefined { + const named = resolveNamedFlagSchemaFromExpression(expression); + if (named) { + return { + name, + ...named, + }; + } + + const type = detectFlagType(expression); + if (!type) { + return undefined; + } + + const config = extractFirstObjectLiteral(expression); + const required = readBooleanProp(config, "required") ?? false; + const multiple = readBooleanProp(config, "multiple") ?? false; + const options = readStringArrayProp(config, "options"); + const exactlyOne = readStringArrayProp(config, "exactlyOne"); + const exclusive = readStringArrayProp(config, "exclusive"); + const dependsOn = readStringArrayProp(config, "dependsOn"); + const defaultValue = readLiteralStringProp(config, "default"); + + return { + name, + required, + type, + takesValue: type !== "boolean", + multiple, + options, + defaultValue, + exactlyOne, + exclusive, + dependsOn, + }; +} + +function detectFlagType(expression: string): FlagValueType | undefined { + if (/Flags\.boolean\s*\(/.test(expression)) { + return "boolean"; + } + + if (/Flags\.integer\s*\(/.test(expression)) { + return "integer"; + } + + if (/Flags\.file\s*\(/.test(expression)) { + return "file"; + } + + if (/Flags\.directory\s*\(/.test(expression)) { + return "directory"; + } + + if (/Flags\.url\s*\(/.test(expression)) { + return "url"; + } + + if (/Flags\.(string|custom)\s*\(/.test(expression)) { + return "string"; + } + + if (/\.absoluteFlag\s*\(/.test(expression) || /\.relativeFlag\s*\(/.test(expression)) { + return "string"; + } + + // Fallback for wrapped/custom flag factories, e.g. `flagDefinitions.name({ required: true })`. + if (/^[A-Za-z0-9_.$\[\]"'-]+\s*\(/.test(expression)) { + return "string"; + } + + return undefined; +} + +function resolveNamedFlagSchemaFromExpression( + expression: string, +): Omit | undefined { + const normalized = expression.trim().replace(/\(\s*\)$/, ""); + const candidate = normalized.split(".").at(-1) ?? normalized; + return NAMED_FLAG_SCHEMAS[candidate]; +} + +function extractFirstObjectLiteral(expression: string): string { + const start = expression.indexOf("{"); + if (start === -1) { + return ""; + } + + const end = findMatchingBraceIndex(expression, start); + if (end === -1) { + return ""; + } + + return expression.slice(start, end + 1); +} + +function readBooleanProp(config: string, key: string): boolean | undefined { + if (!config) { + return undefined; + } + + const regex = new RegExp(`${escapeRegExp(key)}\\s*:\\s*(true|false)`); + const match = config.match(regex); + if (!match) { + return undefined; + } + + return match[1] === "true"; +} + +function readStringProp(config: string, key: string): string | undefined { + if (!config) { + return undefined; + } + + const regex = new RegExp(`${escapeRegExp(key)}\\s*:\\s*(["'])(.*?)\\1`, "s"); + const match = config.match(regex); + return match?.[2]; +} + +function readLiteralStringProp(config: string, key: string): string | undefined { + const stringValue = readStringProp(config, key); + if (stringValue !== undefined) { + return stringValue; + } + + const boolMatch = config.match(new RegExp(`${escapeRegExp(key)}\\s*:\\s*(true|false)`)); + if (boolMatch) { + return boolMatch[1]; + } + + const numberMatch = config.match(new RegExp(`${escapeRegExp(key)}\\s*:\\s*([0-9]+(?:\\.[0-9]+)?)`)); + if (numberMatch) { + return numberMatch[1]; + } + + return undefined; +} + +function readStringArrayProp(config: string, key: string): string[] | undefined { + if (!config) { + return undefined; + } + + const regex = new RegExp(`${escapeRegExp(key)}\\s*:\\s*\\[([^\\]]*)\\]`, "s"); + const match = config.match(regex); + if (!match) { + return undefined; + } + + return match[1] + .split(",") + .map((entry) => entry.trim().replace(/^['"]/, "").replace(/['"]$/, "")) + .filter((entry) => entry.length > 0); +} + +function inferPlaceholderKind(name: string): PlaceholderKind { + const normalized = name.toLowerCase(); + if (normalized.includes("uuid") || normalized.endsWith("id") || normalized.includes("-id")) { + return "uuid"; + } + if (normalized.includes("email")) { + return "email"; + } + if (normalized.includes("url") || normalized.includes("uri")) { + return "url"; + } + if (normalized.includes("duration") || normalized.includes("ttl") || normalized.includes("interval")) { + return "duration"; + } + if (normalized.includes("directory")) { + return "directory"; + } + if (normalized.includes("file")) { + return "file"; + } + if (normalized.includes("password") || normalized.includes("passphrase") || normalized.includes("token")) { + return "password"; + } + if (normalized.includes("port")) { + return "port"; + } + return "generic"; +} + +function parseInvocationParts(args: string[]): { + positionalValues: string[]; + flagValues: Map; +} { + const positionalValues: string[] = []; + const flagValues = new Map(); + + for (let i = 0; i < args.length; i += 1) { + const token = args[i]; + if (!token.startsWith("--")) { + positionalValues.push(token); + continue; + } + + const withoutPrefix = token.slice(2); + const eqIndex = withoutPrefix.indexOf("="); + let name = withoutPrefix; + let value: string | undefined; + + if (eqIndex >= 0) { + name = withoutPrefix.slice(0, eqIndex); + value = withoutPrefix.slice(eqIndex + 1); + } else { + const nextToken = args[i + 1]; + if (nextToken && !nextToken.startsWith("--")) { + value = nextToken; + i += 1; + } + } + + const values = flagValues.get(name) ?? []; + if (value === undefined) { + values.push("true"); + } else { + values.push(value); + } + flagValues.set(name, values); + } + + return { positionalValues, flagValues }; +} + +function escapeRegExp(value: string): string { + return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); +} diff --git a/src/test/integration/command-discovery/synthesis.ts b/src/test/integration/command-discovery/synthesis.ts new file mode 100644 index 000000000..879d2b0a3 --- /dev/null +++ b/src/test/integration/command-discovery/synthesis.ts @@ -0,0 +1,686 @@ +import { loadInvocationProfiles } from "../config/loader.js"; +import { DEFAULT_UUID } from "./config.js"; +import type { + ExampleCandidate, + InteractiveSignal, + InvocationProfile, + ParsedArg, + ParsedFlag, + PlaceholderKind, + ResolvedFlagValue, + SynthesizedInvocation, + ValueSource, +} from "./types.js"; + +export function resolveProfiles(commandId: string): InvocationProfile[] { + const invocationProfiles = loadInvocationProfiles(); + return invocationProfiles.filter((profile) => { + if (profile.match.exact && profile.match.exact === commandId) { + return true; + } + + if (profile.match.prefix && commandId.startsWith(`${profile.match.prefix} `)) { + return true; + } + + return false; + }); +} + +export function synthesizeInvocation(input: { + commandId: string; + commandTokens: string[]; + parsedArgs: ParsedArg[]; + parsedFlags: ParsedFlag[]; + interactiveSignals: InteractiveSignal[]; + exampleCandidate: ExampleCandidate | undefined; + profiles: InvocationProfile[]; +}): SynthesizedInvocation { + const { + commandId, + commandTokens, + parsedArgs, + parsedFlags, + interactiveSignals, + exampleCandidate, + profiles, + } = input; + + const staleExampleReasons: string[] = []; + let validatedExample: ExampleCandidate | undefined; + if (exampleCandidate) { + const validationErrors = validateExampleCandidate(exampleCandidate, parsedArgs, parsedFlags); + if (validationErrors.length === 0) { + validatedExample = exampleCandidate; + } else { + staleExampleReasons.push(...validationErrors.map((reason) => `stale-example: ${reason}`)); + } + } + + const selectedFlags = new Map(); + let strongestSource: ValueSource = "heuristic"; + + const positionalValues = parsedArgs.map((arg, index) => { + const profileValue = getProfileArgValue(profiles, arg.name); + if (profileValue !== undefined) { + strongestSource = selectStrongerSource(strongestSource, "profile"); + return profileValue; + } + + const exampleValue = validatedExample?.positionalValues[index]; + if (exampleValue !== undefined) { + strongestSource = selectStrongerSource(strongestSource, "example"); + return exampleValue; + } + + if (arg.defaultValue !== undefined) { + return arg.defaultValue; + } + + return defaultValueForPlaceholderKind(arg.placeholderKind, arg.name); + }); + + for (const flag of parsedFlags) { + if (!flag.required) { + continue; + } + + const fromProfile = getProfileFlagValue(profiles, flag.name); + if (fromProfile !== undefined) { + setFlagValue(selectedFlags, flag.name, normalizeFlagValue(fromProfile), "profile"); + strongestSource = selectStrongerSource(strongestSource, "profile"); + continue; + } + + const fromExample = validatedExample?.flagValues.get(flag.name); + if (fromExample && fromExample.length > 0) { + setFlagValue(selectedFlags, flag.name, fromExample, "example"); + strongestSource = selectStrongerSource(strongestSource, "example"); + continue; + } + + const heuristic = buildHeuristicFlagValue(flag); + setFlagValue(selectedFlags, flag.name, heuristic, "heuristic"); + } + + resolveExactlyOneGroups(commandId, parsedFlags, selectedFlags, profiles, interactiveSignals); + resolveDependencies(parsedFlags, selectedFlags, profiles); + resolveExclusiveGroups(parsedFlags, selectedFlags); + applyProfileOverrides(parsedFlags, selectedFlags, profiles); + const interactiveDecision = decideInteractivePolicy( + commandId, + parsedFlags, + selectedFlags, + interactiveSignals, + profiles, + ); + + const invocation = renderInvocation(commandTokens, positionalValues, parsedFlags, selectedFlags); + return { + args: invocation, + argumentSource: strongestSource, + interactiveDecision, + staleExample: staleExampleReasons.length > 0, + staleExampleReasons, + }; +} + +function validateExampleCandidate( + example: ExampleCandidate, + argsSchema: ParsedArg[], + flagSchema: ParsedFlag[], +): string[] { + const errors: string[] = []; + const flagNames = new Set(flagSchema.map((flag) => flag.name)); + + for (const flagName of example.flagValues.keys()) { + if (!flagNames.has(flagName)) { + errors.push(`unknown flag --${flagName}`); + } + } + + const requiredArgsCount = argsSchema.filter((arg) => arg.required).length; + if (example.positionalValues.length < requiredArgsCount) { + errors.push("missing required positional arguments"); + } + + for (const flag of flagSchema) { + if (flag.required && !example.flagValues.has(flag.name)) { + errors.push(`missing required flag --${flag.name}`); + } + + if (flag.dependsOn && example.flagValues.has(flag.name)) { + for (const dependency of flag.dependsOn) { + if (!example.flagValues.has(dependency)) { + errors.push(`--${flag.name} depends on --${dependency}`); + } + } + } + + if (flag.exclusive) { + for (const conflicting of flag.exclusive) { + if (example.flagValues.has(flag.name) && example.flagValues.has(conflicting)) { + errors.push(`--${flag.name} is exclusive with --${conflicting}`); + } + } + } + } + + for (const group of collectExactlyOneGroups(flagSchema)) { + const count = group.members.filter((name) => example.flagValues.has(name)).length; + if (count !== 1) { + errors.push(`exactly one of [${group.members.join(", ")}] must be set`); + } + } + + return errors; +} + +function collectExactlyOneGroups(flagSchema: ParsedFlag[]): Array<{ key: string; members: string[] }> { + const groups = new Map(); + + for (const flag of flagSchema) { + if (!flag.exactlyOne || flag.exactlyOne.length < 2) { + continue; + } + + const members = [...new Set(flag.exactlyOne)].sort(); + const key = members.join("|"); + groups.set(key, members); + } + + return [...groups.entries()].map(([key, members]) => ({ key, members })); +} + +function resolveExactlyOneGroups( + commandId: string, + flagSchema: ParsedFlag[], + selectedFlags: Map, + profiles: InvocationProfile[], + interactiveSignals: InteractiveSignal[], +): void { + for (const group of collectExactlyOneGroups(flagSchema)) { + const selectedMembers = group.members.filter((member) => selectedFlags.has(member)); + + if (selectedMembers.length === 1) { + continue; + } + + const preferredByProfile = getProfileExactlyOneChoice(profiles, group.key); + if (preferredByProfile && group.members.includes(preferredByProfile)) { + selectedFlags.set(preferredByProfile, { + values: [makeTypedPlaceholderValue(preferredByProfile, "string", undefined)], + source: "profile", + }); + for (const member of group.members) { + if (member !== preferredByProfile) { + selectedFlags.delete(member); + } + } + continue; + } + + const chosen = chooseExactlyOneMember( + commandId, + group.members, + flagSchema, + interactiveSignals, + ); + + const existing = selectedFlags.get(chosen); + if (!existing) { + selectedFlags.set(chosen, { + values: [makeTypedPlaceholderValue(chosen, "string", undefined)], + source: "heuristic", + }); + } + + for (const member of group.members) { + if (member !== chosen) { + selectedFlags.delete(member); + } + } + } +} + +function chooseExactlyOneMember( + commandId: string, + members: string[], + flagSchema: ParsedFlag[], + interactiveSignals: InteractiveSignal[], +): string { + if (members.includes("project-id")) { + return "project-id"; + } + + const scored = members.map((member) => { + const closure = dependencyClosureSize(member, flagSchema); + const nonInteractiveBonus = scoreNonInteractiveMember(member, interactiveSignals); + return { + member, + score: closure - nonInteractiveBonus, + }; + }); + + scored.sort((a, b) => { + if (a.score !== b.score) { + return a.score - b.score; + } + + return a.member.localeCompare(b.member); + }); + + return scored[0]?.member ?? members[0] ?? commandId; +} + +function scoreNonInteractiveMember(member: string, interactiveSignals: InteractiveSignal[]): number { + if (member === "consent" && interactiveSignals.includes("addConfirmation")) { + return 3; + } + + if (member === "force" && interactiveSignals.includes("addConfirmation")) { + return 3; + } + + if (member === "password" && interactiveSignals.includes("addInput")) { + return 3; + } + + if (member === "override-type" && interactiveSignals.includes("addSelect")) { + return 2; + } + + return 0; +} + +function dependencyClosureSize(member: string, flagSchema: ParsedFlag[]): number { + const visited = new Set(); + + const visit = (flagName: string): void => { + if (visited.has(flagName)) { + return; + } + + visited.add(flagName); + const flag = flagSchema.find((candidate) => candidate.name === flagName); + for (const dependency of flag?.dependsOn ?? []) { + visit(dependency); + } + }; + + visit(member); + return visited.size; +} + +function resolveDependencies( + flagSchema: ParsedFlag[], + selectedFlags: Map, + profiles: InvocationProfile[], +): void { + let changed = true; + + while (changed) { + changed = false; + + for (const flag of flagSchema) { + if (!selectedFlags.has(flag.name)) { + continue; + } + + for (const dependency of flag.dependsOn ?? []) { + if (selectedFlags.has(dependency)) { + continue; + } + + const dependencySpec = flagSchema.find((candidate) => candidate.name === dependency); + const profileValue = getProfileFlagValue(profiles, dependency); + if (profileValue !== undefined) { + setFlagValue(selectedFlags, dependency, normalizeFlagValue(profileValue), "profile"); + changed = true; + continue; + } + + if (!dependencySpec) { + setFlagValue(selectedFlags, dependency, ["true"], "heuristic"); + changed = true; + continue; + } + + setFlagValue(selectedFlags, dependency, buildHeuristicFlagValue(dependencySpec), "heuristic"); + changed = true; + } + } + } +} + +function resolveExclusiveGroups( + flagSchema: ParsedFlag[], + selectedFlags: Map, +): void { + for (const flag of flagSchema) { + const selected = selectedFlags.get(flag.name); + if (!selected || !flag.exclusive) { + continue; + } + + for (const otherName of flag.exclusive) { + const other = selectedFlags.get(otherName); + if (!other) { + continue; + } + + if (compareSourcePrecedence(selected.source, other.source) >= 0) { + selectedFlags.delete(otherName); + } else { + selectedFlags.delete(flag.name); + } + } + } +} + +function applyProfileOverrides( + flagSchema: ParsedFlag[], + selectedFlags: Map, + profiles: InvocationProfile[], +): void { + for (const profile of profiles) { + for (const [flagName, profileValue] of Object.entries(profile.requiredFlagDefaults ?? {})) { + const spec = flagSchema.find((flag) => flag.name === flagName); + if (!spec) { + continue; + } + + setFlagValue(selectedFlags, flagName, normalizeFlagValue(profileValue), "profile"); + } + } +} + +function decideInteractivePolicy( + commandId: string, + flagSchema: ParsedFlag[], + selectedFlags: Map, + interactiveSignals: InteractiveSignal[], + profiles: InvocationProfile[], +): "NON_INTERACTIVE_RESOLVED" | "INTERACTIVE_REQUIRED" { + if (interactiveSignals.length === 0) { + return "NON_INTERACTIVE_RESOLVED"; + } + + const policy = profiles.find((profile) => profile.interactivePolicy)?.interactivePolicy; + if (policy === "classify") { + return "INTERACTIVE_REQUIRED"; + } + + const unresolved = resolveInteractiveSignals(commandId, flagSchema, selectedFlags, interactiveSignals); + return unresolved.length === 0 ? "NON_INTERACTIVE_RESOLVED" : "INTERACTIVE_REQUIRED"; +} + +function resolveInteractiveSignals( + commandId: string, + flagSchema: ParsedFlag[], + selectedFlags: Map, + interactiveSignals: InteractiveSignal[], +): InteractiveSignal[] { + const unresolved: InteractiveSignal[] = []; + + const hasFlag = (name: string): boolean => flagSchema.some((flag) => flag.name === name); + + for (const signal of interactiveSignals) { + if (signal === "addConfirmation") { + if (hasFlag("force")) { + setFlagValue(selectedFlags, "force", ["true"], "heuristic"); + continue; + } + + if (hasFlag("consent")) { + setFlagValue(selectedFlags, "consent", ["true"], "heuristic"); + continue; + } + + unresolved.push(signal); + continue; + } + + if (signal === "addInput") { + if (hasFlag("password")) { + setFlagValue(selectedFlags, "password", ["integration-password"], "heuristic"); + continue; + } + + if (hasFlag("user-password")) { + setFlagValue(selectedFlags, "user-password", ["integration-password"], "heuristic"); + continue; + } + + unresolved.push(signal); + continue; + } + + if (signal === "addSelect") { + if (hasFlag("override-type")) { + setFlagValue(selectedFlags, "override-type", ["auto"], "heuristic"); + continue; + } + + unresolved.push(signal); + continue; + } + + unresolved.push(signal); + } + + if (commandId === "login token") { + return [...new Set([...unresolved, "addInput"])] as InteractiveSignal[]; + } + + return [...new Set(unresolved)]; +} + +function renderInvocation( + commandTokens: string[], + positionalValues: string[], + parsedFlags: ParsedFlag[], + selectedFlags: Map, +): string[] { + const args = [...commandTokens, ...positionalValues]; + + const orderedFlags = parsedFlags + .filter((flag) => selectedFlags.has(flag.name)) + .sort((a, b) => a.name.localeCompare(b.name)); + + for (const flag of orderedFlags) { + const resolved = selectedFlags.get(flag.name); + if (!resolved) { + continue; + } + + if (!flag.takesValue) { + args.push(`--${flag.name}`); + continue; + } + + for (const value of resolved.values) { + args.push(`--${flag.name}`); + args.push(value); + } + } + + return args; +} + +function setFlagValue( + map: Map, + flagName: string, + values: string[], + source: ValueSource, +): void { + const existing = map.get(flagName); + if (!existing) { + map.set(flagName, { values, source }); + return; + } + + if (compareSourcePrecedence(source, existing.source) >= 0) { + map.set(flagName, { values, source }); + } +} + +function normalizeFlagValue(value: string | boolean): string[] { + if (typeof value === "boolean") { + return [value ? "true" : "false"]; + } + + return [value]; +} + +function buildHeuristicFlagValue(flag: ParsedFlag): string[] { + if (!flag.takesValue) { + return ["true"]; + } + + if (flag.defaultValue !== undefined) { + return [flag.defaultValue]; + } + + return [makeTypedPlaceholderValue(flag.name, flag.type, flag.options)]; +} + +function getProfileArgValue(profiles: InvocationProfile[], argName: string): string | undefined { + for (const profile of profiles) { + const value = profile.requiredArgDefaults?.[argName]; + if (value !== undefined) { + return value; + } + } + + return undefined; +} + +function getProfileFlagValue( + profiles: InvocationProfile[], + flagName: string, +): string | boolean | undefined { + for (const profile of profiles) { + const value = profile.requiredFlagDefaults?.[flagName]; + if (value !== undefined) { + return value; + } + } + + return undefined; +} + +function getProfileExactlyOneChoice( + profiles: InvocationProfile[], + groupKey: string, +): string | undefined { + for (const profile of profiles) { + const choice = profile.exactlyOneChoice?.[groupKey]; + if (choice !== undefined) { + return choice; + } + } + + return undefined; +} + +function compareSourcePrecedence(a: ValueSource, b: ValueSource): number { + const precedence: Record = { + heuristic: 1, + example: 2, + profile: 3, + }; + + return precedence[a] - precedence[b]; +} + +function selectStrongerSource(current: ValueSource, candidate: ValueSource): ValueSource { + return compareSourcePrecedence(candidate, current) >= 0 ? candidate : current; +} + +function defaultValueForPlaceholderKind(kind: PlaceholderKind, name: string): string { + if (kind === "uuid") { + return DEFAULT_UUID; + } + + if (kind === "email") { + return "integration@example.com"; + } + + if (kind === "url") { + return "https://example.com"; + } + + if (kind === "duration") { + return "1h"; + } + + if (kind === "directory") { + return "/tmp/mw-integration"; + } + + if (kind === "file") { + return "/tmp/mw-integration.file"; + } + + if (kind === "password") { + return "integration-password"; + } + + if (kind === "port") { + return "12345"; + } + + return makeTypedPlaceholderValue(name, "string", undefined); +} + +function makeTypedPlaceholderValue( + name: string, + _type: string, + options: string[] | undefined, +): string { + if (options && options.length > 0) { + return options[0]; + } + + const normalized = name + .replace(/[<>[\]]/g, "") + .replace(/[^A-Za-z0-9-]+/g, "-") + .replace(/^-+/, "") + .replace(/-+$/, "") + .toLowerCase(); + + if (normalized.includes("uuid") || normalized.endsWith("id") || normalized.includes("-id")) { + return DEFAULT_UUID; + } + + if (normalized.includes("email")) { + return "integration@example.com"; + } + + if (normalized.includes("url") || normalized.includes("uri")) { + return "https://example.com"; + } + + if ( + normalized.includes("duration") || + normalized.includes("ttl") || + normalized.includes("interval") + ) { + return "1h"; + } + + if (normalized.includes("directory") || normalized.includes("path")) { + return "/tmp/mw-integration"; + } + + if (normalized.includes("password")) { + return "integration-password"; + } + + if (normalized.includes("port")) { + return "12345"; + } + + return normalized.length > 0 ? `example-${normalized}` : "example-value"; +} diff --git a/src/test/integration/command-discovery/types.ts b/src/test/integration/command-discovery/types.ts new file mode 100644 index 000000000..a7e4949f5 --- /dev/null +++ b/src/test/integration/command-discovery/types.ts @@ -0,0 +1,106 @@ +export type FlagValueType = + | "boolean" + | "string" + | "integer" + | "file" + | "directory" + | "url" + | "custom"; + +export type ValueSource = "profile" | "example" | "heuristic"; + +export type InteractiveSignal = + | "addInput" + | "addSelect" + | "addConfirmation" + | "editorFallback" + | "stdinBranch"; + +export type PlaceholderKind = + | "uuid" + | "email" + | "url" + | "duration" + | "file" + | "directory" + | "password" + | "port" + | "generic"; + +export type ParsedArg = { + name: string; + required: boolean; + defaultValue?: string; + placeholderKind: PlaceholderKind; +}; + +export type ParsedFlag = { + name: string; + required: boolean; + type: FlagValueType; + takesValue: boolean; + options?: string[]; + multiple?: boolean; + defaultValue?: string; + exactlyOne?: string[]; + exclusive?: string[]; + dependsOn?: string[]; +}; + +export type SynthesizedInvocation = { + args: string[]; + argumentSource: ValueSource; + interactiveDecision: "NON_INTERACTIVE_RESOLVED" | "INTERACTIVE_REQUIRED"; + staleExample: boolean; + staleExampleReasons: string[]; +}; + +export type DiscoveredCommand = { + commandId: string; + sourceFile: string; + commandTokens: string[]; + parsedArgs: ParsedArg[]; + parsedFlags: ParsedFlag[]; + interactiveSignals: InteractiveSignal[]; + invocationProfilesApplied: string[]; + extractionDiagnostics: string[]; + synthesizedInvocation: SynthesizedInvocation; +}; + +export type ExampleCandidate = { + args: string[]; + positionalValues: string[]; + flagValues: Map; +}; + +export type ResolvedFlagValue = { + values: string[]; + source: ValueSource; +}; + +export type InvocationProfile = { + id: string; + match: { exact?: string; prefix?: string }; + requiredFlagDefaults?: Record; + requiredArgDefaults?: Record; + exactlyOneChoice?: Record; + interactivePolicy?: "resolve" | "classify"; + disableExampleSource?: boolean; + notes?: string; +}; + +export type WaiverCategory = + | "ARG_MISUSE" + | "INTERACTIVE_REQUIRED" + | "RESOURCE_PRECONDITION" + | "CONTRACT_SHAPE" + | "COMMAND_BUG"; + +export type CommandWaiver = { + id: string; + commandId: string; + category: WaiverCategory; + reason: string; + issue?: string; + expiresOn?: string; +}; diff --git a/src/test/integration/command.ts b/src/test/integration/command.ts new file mode 100644 index 000000000..2d0edb1bb --- /dev/null +++ b/src/test/integration/command.ts @@ -0,0 +1,127 @@ +import { spawn } from "node:child_process"; + +export type DevCommandResult = { + stdout: string; + stderr: string; + exitCode: number | null; + signal: NodeJS.Signals | null; + error?: Error; + timedOut?: boolean; +}; + +export type RunDevCommandOptions = { + cwd?: string; + env?: NodeJS.ProcessEnv; + timeoutMs?: number; +}; + +export async function runDevCommand( + args: string[], + options: RunDevCommandOptions = {}, +): Promise { + return await new Promise((resolve) => { + const timeoutMs = options.timeoutMs ?? 30_000; + + const child = spawn( + "yarn", + [ + "node", + "--import", + "tsx", + "--no-warnings=ExperimentalWarning", + "./bin/dev.js", + ...args, + ], + { + cwd: options.cwd ?? process.cwd(), + env: options.env ?? process.env, + stdio: ["ignore", "pipe", "pipe"], + }, + ); + + let stdout = ""; + let stderr = ""; + let settled = false; + let didTimeOut = false; + + const finish = (result: DevCommandResult): void => { + if (settled) { + return; + } + + settled = true; + if (timeoutHandle) { + clearTimeout(timeoutHandle); + } + if (forceKillHandle) { + clearTimeout(forceKillHandle); + } + + resolve(result); + }; + + let forceKillHandle: NodeJS.Timeout | undefined; + const timeoutHandle: NodeJS.Timeout | undefined = + timeoutMs > 0 + ? setTimeout(() => { + didTimeOut = true; + child.kill("SIGTERM"); + + // Give graceful termination a short window before hard-killing. + forceKillHandle = setTimeout(() => { + child.kill("SIGKILL"); + }, 2_000); + forceKillHandle.unref?.(); + }, timeoutMs) + : undefined; + + timeoutHandle?.unref?.(); + + child.stdout.on("data", (chunk) => { + stdout += chunk.toString(); + }); + + child.stderr.on("data", (chunk) => { + stderr += chunk.toString(); + }); + + child.on("error", (error) => { + finish({ + stdout, + stderr, + exitCode: null, + signal: null, + timedOut: didTimeOut, + error, + }); + }); + + child.on("close", (exitCode, signal) => { + if (didTimeOut) { + finish({ + stdout, + stderr, + exitCode, + signal, + timedOut: true, + error: new Error(`dev.js subprocess timed out after ${timeoutMs}ms`), + }); + return; + } + + if (exitCode === 0) { + finish({ stdout, stderr, exitCode, signal, timedOut: false }); + return; + } + + finish({ + stdout, + stderr, + exitCode, + signal, + timedOut: false, + error: new Error(`dev.js subprocess exited with code ${exitCode}`), + }); + }); + }); +} \ No newline at end of file diff --git a/src/test/integration/config/command-classifications.json b/src/test/integration/config/command-classifications.json new file mode 100644 index 000000000..8ea8a2a55 --- /dev/null +++ b/src/test/integration/config/command-classifications.json @@ -0,0 +1,350 @@ +{ + "schemaVersion": 1, + "generatedAt": "2026-07-31T09:25:15.388Z", + "source": { + "kind": "run-all-summary" + }, + "statistics": { + "successful": 118, + "failed": 57, + "waivedSkipped": 10, + "total": 185 + }, + "entries": [ + { + "commandId": "app create node", + "category": "CONTRACT_SHAPE", + "source": "failure" + }, + { + "commandId": "app create php", + "category": "CONTRACT_SHAPE", + "source": "failure" + }, + { + "commandId": "app create php-worker", + "category": "CONTRACT_SHAPE", + "source": "failure" + }, + { + "commandId": "app create python", + "category": "CONTRACT_SHAPE", + "source": "failure" + }, + { + "commandId": "app create static", + "category": "CONTRACT_SHAPE", + "source": "failure" + }, + { + "commandId": "app database link", + "category": "RESOURCE_PRECONDITION", + "source": "failure" + }, + { + "commandId": "app database replace", + "category": "RESOURCE_PRECONDITION", + "source": "failure" + }, + { + "commandId": "app dependency update", + "category": "COMMAND_BUG", + "source": "failure" + }, + { + "commandId": "app dependency versions", + "category": "RESOURCE_PRECONDITION", + "source": "failure" + }, + { + "commandId": "app download", + "category": "COMMAND_BUG", + "source": "failure" + }, + { + "commandId": "app exec", + "category": "COMMAND_BUG", + "source": "failure" + }, + { + "commandId": "app get", + "category": "COMMAND_BUG", + "source": "failure" + }, + { + "commandId": "app install contao", + "category": "CONTRACT_SHAPE", + "source": "failure" + }, + { + "commandId": "app install joomla", + "category": "CONTRACT_SHAPE", + "source": "failure" + }, + { + "commandId": "app install matomo", + "category": "COMMAND_BUG", + "source": "failure" + }, + { + "commandId": "app install nextcloud", + "category": "CONTRACT_SHAPE", + "source": "failure" + }, + { + "commandId": "app install shopware5", + "category": "CONTRACT_SHAPE", + "source": "failure" + }, + { + "commandId": "app install shopware6", + "category": "COMMAND_BUG", + "source": "failure" + }, + { + "commandId": "app install typo3", + "category": "COMMAND_BUG", + "source": "failure" + }, + { + "commandId": "app install wordpress", + "category": "COMMAND_BUG", + "source": "failure" + }, + { + "commandId": "app list-upgrade-candidates", + "category": "COMMAND_BUG", + "source": "failure" + }, + { + "commandId": "app open", + "category": "COMMAND_BUG", + "source": "failure" + }, + { + "commandId": "app ssh", + "category": "COMMAND_BUG", + "source": "failure" + }, + { + "commandId": "app upgrade", + "category": "INTERACTIVE_REQUIRED", + "source": "waiver" + }, + { + "commandId": "app upload", + "category": "RESOURCE_PRECONDITION", + "source": "failure" + }, + { + "commandId": "app version-info", + "category": "RESOURCE_PRECONDITION", + "source": "failure" + }, + { + "commandId": "app versions", + "category": "COMMAND_BUG", + "source": "failure" + }, + { + "commandId": "backup download", + "category": "COMMAND_BUG", + "source": "failure" + }, + { + "commandId": "container cp", + "category": "RESOURCE_PRECONDITION", + "source": "failure" + }, + { + "commandId": "container delete", + "category": "RESOURCE_PRECONDITION", + "source": "failure" + }, + { + "commandId": "container exec", + "category": "RESOURCE_PRECONDITION", + "source": "failure" + }, + { + "commandId": "container logs", + "category": "INTERACTIVE_REQUIRED", + "source": "waiver" + }, + { + "commandId": "container port-forward", + "category": "RESOURCE_PRECONDITION", + "source": "failure" + }, + { + "commandId": "container recreate", + "category": "RESOURCE_PRECONDITION", + "source": "failure" + }, + { + "commandId": "container restart", + "category": "RESOURCE_PRECONDITION", + "source": "failure" + }, + { + "commandId": "container run", + "category": "RESOURCE_PRECONDITION", + "source": "failure" + }, + { + "commandId": "container ssh", + "category": "RESOURCE_PRECONDITION", + "source": "failure" + }, + { + "commandId": "container start", + "category": "RESOURCE_PRECONDITION", + "source": "failure" + }, + { + "commandId": "container stop", + "category": "RESOURCE_PRECONDITION", + "source": "failure" + }, + { + "commandId": "container update", + "category": "RESOURCE_PRECONDITION", + "source": "failure" + }, + { + "commandId": "conversation create", + "category": "COMMAND_BUG", + "source": "failure" + }, + { + "commandId": "conversation reply", + "category": "COMMAND_BUG", + "source": "failure" + }, + { + "commandId": "cronjob execution logs", + "category": "INTERACTIVE_REQUIRED", + "source": "waiver" + }, + { + "commandId": "database mysql dump", + "category": "RESOURCE_PRECONDITION", + "source": "failure" + }, + { + "commandId": "database mysql import", + "category": "INTERACTIVE_REQUIRED", + "source": "waiver" + }, + { + "commandId": "database mysql phpmyadmin", + "category": "RESOURCE_PRECONDITION", + "source": "failure" + }, + { + "commandId": "database mysql port-forward", + "category": "RESOURCE_PRECONDITION", + "source": "failure" + }, + { + "commandId": "database mysql shell", + "category": "INTERACTIVE_REQUIRED", + "source": "waiver" + }, + { + "commandId": "database mysql upgrade", + "category": "INTERACTIVE_REQUIRED", + "source": "waiver" + }, + { + "commandId": "database mysql user delete", + "category": "RESOURCE_PRECONDITION", + "source": "failure" + }, + { + "commandId": "ddev init", + "category": "INTERACTIVE_REQUIRED", + "source": "waiver" + }, + { + "commandId": "ddev render-config", + "category": "CONTRACT_SHAPE", + "source": "failure" + }, + { + "commandId": "domain dnszone get", + "category": "RESOURCE_PRECONDITION", + "source": "failure" + }, + { + "commandId": "domain dnszone update", + "category": "RESOURCE_PRECONDITION", + "source": "failure" + }, + { + "commandId": "domain get", + "category": "RESOURCE_PRECONDITION", + "source": "failure" + }, + { + "commandId": "experimental deploy", + "category": "RESOURCE_PRECONDITION", + "source": "failure" + }, + { + "commandId": "login token", + "category": "INTERACTIVE_REQUIRED", + "source": "waiver" + }, + { + "commandId": "mail address update", + "category": "RESOURCE_PRECONDITION", + "source": "failure" + }, + { + "commandId": "org delete", + "category": "RESOURCE_PRECONDITION", + "source": "failure" + }, + { + "commandId": "sftp-user create", + "category": "RESOURCE_PRECONDITION", + "source": "failure" + }, + { + "commandId": "ssh-user create", + "category": "RESOURCE_PRECONDITION", + "source": "failure" + }, + { + "commandId": "stack delete", + "category": "COMMAND_BUG", + "source": "failure" + }, + { + "commandId": "stack deploy", + "category": "INTERACTIVE_REQUIRED", + "source": "waiver" + }, + { + "commandId": "stack set-update-schedule", + "category": "RESOURCE_PRECONDITION", + "source": "failure" + }, + { + "commandId": "stack unset-update-schedule", + "category": "RESOURCE_PRECONDITION", + "source": "failure" + }, + { + "commandId": "user ssh-key create", + "category": "INTERACTIVE_REQUIRED", + "source": "waiver" + }, + { + "commandId": "volume delete", + "category": "RESOURCE_PRECONDITION", + "source": "failure" + } + ] +} diff --git a/src/test/integration/config/command-waivers.json b/src/test/integration/config/command-waivers.json new file mode 100644 index 000000000..9e5afd48d --- /dev/null +++ b/src/test/integration/config/command-waivers.json @@ -0,0 +1,72 @@ +[ + { + "id": "interactive-app-upgrade", + "commandId": "app upgrade", + "category": "INTERACTIVE_REQUIRED", + "reason": "Upgrade target selection is interactive and has no stable non-interactive override yet.", + "issue": "defer-interactive-support" + }, + { + "id": "interactive-container-logs", + "commandId": "container logs", + "category": "INTERACTIVE_REQUIRED", + "reason": "Log follow behavior depends on interactive terminal controls in current implementation.", + "issue": "defer-interactive-support" + }, + { + "id": "interactive-cronjob-execution-logs", + "commandId": "cronjob execution logs", + "category": "INTERACTIVE_REQUIRED", + "reason": "Execution log access path prompts or expects interactive I/O in this test setup.", + "issue": "defer-interactive-support" + }, + { + "id": "interactive-database-mysql-import", + "commandId": "database mysql import", + "category": "INTERACTIVE_REQUIRED", + "reason": "Import flow expects interactive input or file prompt handling not available in the simple renderer.", + "issue": "defer-interactive-support" + }, + { + "id": "interactive-database-mysql-shell", + "commandId": "database mysql shell", + "category": "INTERACTIVE_REQUIRED", + "reason": "MySQL shell requires password prompt interaction and cannot run headless yet.", + "issue": "defer-interactive-support" + }, + { + "id": "interactive-database-mysql-upgrade", + "commandId": "database mysql upgrade", + "category": "INTERACTIVE_REQUIRED", + "reason": "Upgrade confirmation and version choice currently requires interactive input.", + "issue": "defer-interactive-support" + }, + { + "id": "interactive-ddev-init", + "commandId": "ddev init", + "category": "INTERACTIVE_REQUIRED", + "reason": "Project type and configuration selection still enters interactive decision branches.", + "issue": "defer-interactive-support" + }, + { + "id": "interactive-login-token", + "commandId": "login token", + "category": "INTERACTIVE_REQUIRED", + "reason": "Token acquisition flow is intentionally interactive for secure input handling.", + "issue": "defer-interactive-support" + }, + { + "id": "interactive-stack-deploy", + "commandId": "stack deploy", + "category": "INTERACTIVE_REQUIRED", + "reason": "Deploy flow requires interactive input in current command implementation.", + "issue": "defer-interactive-support" + }, + { + "id": "interactive-user-ssh-key-create", + "commandId": "user ssh-key create", + "category": "INTERACTIVE_REQUIRED", + "reason": "SSH key creation relies on interactive prompts for key source and confirmation.", + "issue": "defer-interactive-support" + } +] diff --git a/src/test/integration/config/invocation-profiles.json b/src/test/integration/config/invocation-profiles.json new file mode 100644 index 000000000..3f3f330bf --- /dev/null +++ b/src/test/integration/config/invocation-profiles.json @@ -0,0 +1,218 @@ +[ + { + "id": "backup-create", + "match": { "exact": "backup create" }, + "requiredFlagDefaults": { + "expires": "30d" + }, + "notes": "Backup expiration must be set explicitly for deterministic runs." + }, + { + "id": "backup-schedule-create", + "match": { "exact": "backup schedule create" }, + "requiredFlagDefaults": { + "schedule": "0 * * * *", + "ttl": "7d" + } + }, + { + "id": "cronjob-create", + "match": { "exact": "cronjob create" }, + "requiredFlagDefaults": { + "description": "integration-cronjob", + "interval": "0 * * * *", + "url": "https://example.com/cronjob" + }, + "exactlyOneChoice": { + "command|url": "url" + } + }, + { + "id": "extension-install", + "match": { "exact": "extension install" }, + "requiredArgDefaults": { + "extension-id": "example-extension-id" + }, + "requiredFlagDefaults": { + "consent": true, + "project-id": "00000000-0000-4000-8000-000000000000" + }, + "exactlyOneChoice": { + "org-id|project-id": "project-id" + } + }, + { + "id": "extension-list-installed", + "match": { "exact": "extension list-installed" }, + "requiredFlagDefaults": { + "project-id": "00000000-0000-4000-8000-000000000000" + }, + "exactlyOneChoice": { + "org-id|project-id": "project-id" + } + }, + { + "id": "sftp-user-create", + "match": { "exact": "sftp-user create" }, + "requiredFlagDefaults": { + "description": "integration-sftp-user", + "directories": "/", + "password": "integration-password" + }, + "exactlyOneChoice": { + "password|public-key": "password" + } + }, + { + "id": "ssh-user-create", + "match": { "exact": "ssh-user create" }, + "requiredFlagDefaults": { + "description": "integration-ssh-user", + "password": "integration-password" + } + }, + { + "id": "database-mysql-create", + "match": { "exact": "database mysql create" }, + "requiredFlagDefaults": { + "description": "integration-mysql-db", + "version": "8.0", + "user-password": "integration-password" + } + }, + { + "id": "database-mysql-user-create", + "match": { "exact": "database mysql user create" }, + "requiredFlagDefaults": { + "database-id": "00000000-0000-4000-8000-000000000000", + "access-level": "full", + "description": "integration-mysql-user", + "password": "integration-password" + } + }, + { + "id": "database-mysql-shell", + "match": { "exact": "database mysql shell" }, + "interactivePolicy": "classify" + }, + { + "id": "app-database-link", + "match": { "exact": "app database link" }, + "requiredFlagDefaults": { + "database-id": "00000000-0000-4000-8000-000000000000", + "admin-user-id": "00000000-0000-4000-8000-000000000000", + "purpose": "primary" + } + }, + { + "id": "app-database-replace", + "match": { "exact": "app database replace" }, + "requiredFlagDefaults": { + "new-database-id": "00000000-0000-4000-8000-000000000000", + "admin-user-id": "00000000-0000-4000-8000-000000000000" + } + }, + { + "id": "org-invite", + "match": { "exact": "org invite" }, + "requiredFlagDefaults": { + "email": "integration@example.com" + } + }, + { + "id": "user-api-token-create", + "match": { "exact": "user api-token create" }, + "requiredFlagDefaults": { + "description": "integration-api-token", + "roles": "api_read" + } + }, + { + "id": "ssh-user-update", + "match": { "exact": "ssh-user update" }, + "requiredFlagDefaults": { + "password": "integration-password" + }, + "exactlyOneChoice": { + "password|public-key": "password" + } + }, + { + "id": "domain-dnszone-update", + "match": { "exact": "domain dnszone update" }, + "requiredArgDefaults": { + "record-set": "a" + }, + "requiredFlagDefaults": { + "record": "203.0.113.10" + } + }, + { + "id": "domain-get", + "match": { "exact": "domain get" }, + "requiredArgDefaults": { + "domain-id": "example.com" + } + }, + { + "id": "domain-dnszone-get", + "match": { "exact": "domain dnszone get" }, + "requiredArgDefaults": { + "dnszone-id": "example.com" + } + }, + { + "id": "domain-virtualhost-update", + "match": { "exact": "domain virtualhost update" }, + "requiredFlagDefaults": { + "path-to-url": "/:https://example.com" + } + }, + { + "id": "mail-address-update", + "match": { "exact": "mail address update" }, + "requiredArgDefaults": { + "mailaddress-id": "integration@example.com" + } + }, + { + "id": "mail-deliverybox-update", + "match": { "exact": "mail deliverybox update" }, + "requiredArgDefaults": { + "maildeliverybox-id": "00000000-0000-4000-8000-000000000000" + } + }, + { + "id": "ddev-init", + "match": { "exact": "ddev init" }, + "requiredFlagDefaults": { + "override-type": "auto", + "project-name": "integration-ddev" + } + }, + { + "id": "ddev-render-config", + "match": { "exact": "ddev render-config" }, + "requiredFlagDefaults": { + "override-type": "php" + } + }, + { + "id": "login-token", + "match": { "exact": "login token" }, + "interactivePolicy": "classify", + "disableExampleSource": true + }, + { + "id": "conversation-create", + "match": { "exact": "conversation create" }, + "interactivePolicy": "classify", + "disableExampleSource": true + }, + { + "id": "conversation-reply", + "match": { "exact": "conversation reply" }, + "interactivePolicy": "classify", + "disableExampleSource": true + } +] diff --git a/src/test/integration/config/loader.ts b/src/test/integration/config/loader.ts new file mode 100644 index 000000000..4e5432b0e --- /dev/null +++ b/src/test/integration/config/loader.ts @@ -0,0 +1,275 @@ +import { readFileSync } from "node:fs"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import type { CommandWaiver, InvocationProfile, WaiverCategory } from "../command-discovery/types.js"; + +const CONFIG_DIR = path.dirname(fileURLToPath(import.meta.url)); +const INVOCATION_PROFILES_PATH = path.join(CONFIG_DIR, "invocation-profiles.json"); +const COMMAND_WAIVERS_PATH = path.join(CONFIG_DIR, "command-waivers.json"); + +const WAIVER_CATEGORIES: Set = new Set([ + "ARG_MISUSE", + "INTERACTIVE_REQUIRED", + "RESOURCE_PRECONDITION", + "CONTRACT_SHAPE", + "COMMAND_BUG", +]); + +let invocationProfilesCache: InvocationProfile[] | undefined; +let commandWaiversCache: CommandWaiver[] | undefined; + +export function loadInvocationProfiles(): InvocationProfile[] { + if (invocationProfilesCache) { + return invocationProfilesCache; + } + + const raw = readJsonFile(INVOCATION_PROFILES_PATH, "invocation profiles"); + if (!Array.isArray(raw)) { + throw new Error("[integration-config] invocation profiles must be an array."); + } + + invocationProfilesCache = raw.map((value, index) => + validateInvocationProfile(value, index), + ); + + return invocationProfilesCache; +} + +export function loadCommandWaivers(): CommandWaiver[] { + if (commandWaiversCache) { + return commandWaiversCache; + } + + const raw = readJsonFile(COMMAND_WAIVERS_PATH, "command waivers"); + if (!Array.isArray(raw)) { + throw new Error("[integration-config] command waivers must be an array."); + } + + const validated = raw.map((value, index) => validateCommandWaiver(value, index)); + + const ids = new Set(); + const commandIds = new Set(); + for (const waiver of validated) { + if (ids.has(waiver.id)) { + throw new Error(`[integration-config] duplicate waiver id '${waiver.id}'.`); + } + + if (commandIds.has(waiver.commandId)) { + throw new Error( + `[integration-config] duplicate waiver commandId '${waiver.commandId}'.`, + ); + } + + ids.add(waiver.id); + commandIds.add(waiver.commandId); + } + + commandWaiversCache = validated; + return commandWaiversCache; +} + +function readJsonFile(filePath: string, label: string): unknown { + try { + const content = readFileSync(filePath, "utf8"); + return JSON.parse(content); + } catch (error) { + throw new Error( + `[integration-config] failed to load ${label} at ${filePath}: ${(error as Error).message}`, + ); + } +} + +function validateInvocationProfile(value: unknown, index: number): InvocationProfile { + const record = asRecord(value, `invocation profile at index ${index}`); + const id = asNonEmptyString(record.id, `${profileLabel(index)}.id`); + + const matchRaw = asRecord(record.match, `${profileLabel(index)}.match`); + const exact = asOptionalString(matchRaw.exact, `${profileLabel(index)}.match.exact`); + const prefix = asOptionalString(matchRaw.prefix, `${profileLabel(index)}.match.prefix`); + if (!exact && !prefix) { + throw new Error( + `[integration-config] ${profileLabel(index)}.match requires 'exact' or 'prefix'.`, + ); + } + + const requiredFlagDefaults = asOptionalStringBooleanMap( + record.requiredFlagDefaults, + `${profileLabel(index)}.requiredFlagDefaults`, + ); + const requiredArgDefaults = asOptionalStringMap( + record.requiredArgDefaults, + `${profileLabel(index)}.requiredArgDefaults`, + ); + const exactlyOneChoice = asOptionalStringMap( + record.exactlyOneChoice, + `${profileLabel(index)}.exactlyOneChoice`, + ); + + const interactivePolicy = asOptionalInteractivePolicy( + record.interactivePolicy, + `${profileLabel(index)}.interactivePolicy`, + ); + + const disableExampleSource = asOptionalBoolean( + record.disableExampleSource, + `${profileLabel(index)}.disableExampleSource`, + ); + + const notes = asOptionalString(record.notes, `${profileLabel(index)}.notes`); + + return { + id, + match: { + ...(exact ? { exact } : {}), + ...(prefix ? { prefix } : {}), + }, + ...(requiredFlagDefaults ? { requiredFlagDefaults } : {}), + ...(requiredArgDefaults ? { requiredArgDefaults } : {}), + ...(exactlyOneChoice ? { exactlyOneChoice } : {}), + ...(interactivePolicy ? { interactivePolicy } : {}), + ...(disableExampleSource !== undefined ? { disableExampleSource } : {}), + ...(notes ? { notes } : {}), + }; +} + +function validateCommandWaiver(value: unknown, index: number): CommandWaiver { + const record = asRecord(value, `command waiver at index ${index}`); + const id = asNonEmptyString(record.id, `${waiverLabel(index)}.id`); + const commandId = asNonEmptyString( + record.commandId, + `${waiverLabel(index)}.commandId`, + ); + const category = asNonEmptyString( + record.category, + `${waiverLabel(index)}.category`, + ) as WaiverCategory; + + if (!WAIVER_CATEGORIES.has(category)) { + throw new Error( + `[integration-config] ${waiverLabel(index)}.category must be one of ${[ + ...WAIVER_CATEGORIES, + ].join(", ")}.`, + ); + } + + const reason = asNonEmptyString(record.reason, `${waiverLabel(index)}.reason`); + const issue = asOptionalString(record.issue, `${waiverLabel(index)}.issue`); + const expiresOn = asOptionalString(record.expiresOn, `${waiverLabel(index)}.expiresOn`); + + return { + id, + commandId, + category, + reason, + ...(issue ? { issue } : {}), + ...(expiresOn ? { expiresOn } : {}), + }; +} + +function asRecord(value: unknown, label: string): Record { + if (!value || typeof value !== "object" || Array.isArray(value)) { + throw new Error(`[integration-config] ${label} must be an object.`); + } + + return value as Record; +} + +function asNonEmptyString(value: unknown, label: string): string { + if (typeof value !== "string" || value.trim().length === 0) { + throw new Error(`[integration-config] ${label} must be a non-empty string.`); + } + + return value.trim(); +} + +function asOptionalString(value: unknown, label: string): string | undefined { + if (value === undefined) { + return undefined; + } + + if (typeof value !== "string") { + throw new Error(`[integration-config] ${label} must be a string when provided.`); + } + + return value; +} + +function asOptionalBoolean(value: unknown, label: string): boolean | undefined { + if (value === undefined) { + return undefined; + } + + if (typeof value !== "boolean") { + throw new Error(`[integration-config] ${label} must be a boolean when provided.`); + } + + return value; +} + +function asOptionalInteractivePolicy( + value: unknown, + label: string, +): "resolve" | "classify" | undefined { + if (value === undefined) { + return undefined; + } + + if (value !== "resolve" && value !== "classify") { + throw new Error( + `[integration-config] ${label} must be 'resolve' or 'classify' when provided.`, + ); + } + + return value; +} + +function asOptionalStringMap( + value: unknown, + label: string, +): Record | undefined { + if (value === undefined) { + return undefined; + } + + const record = asRecord(value, label); + const result: Record = {}; + for (const [key, entry] of Object.entries(record)) { + if (typeof entry !== "string") { + throw new Error(`[integration-config] ${label}.${key} must be a string.`); + } + + result[key] = entry; + } + + return result; +} + +function asOptionalStringBooleanMap( + value: unknown, + label: string, +): Record | undefined { + if (value === undefined) { + return undefined; + } + + const record = asRecord(value, label); + const result: Record = {}; + + for (const [key, entry] of Object.entries(record)) { + if (typeof entry !== "string" && typeof entry !== "boolean") { + throw new Error(`[integration-config] ${label}.${key} must be a string or boolean.`); + } + + result[key] = entry; + } + + return result; +} + +function profileLabel(index: number): string { + return `invocation-profiles[${index}]`; +} + +function waiverLabel(index: number): string { + return `command-waivers[${index}]`; +} diff --git a/src/test/integration/env.ts b/src/test/integration/env.ts new file mode 100644 index 000000000..3113fdbdc --- /dev/null +++ b/src/test/integration/env.ts @@ -0,0 +1,32 @@ +export type EnvSnapshot = NodeJS.ProcessEnv; + +export function snapshotEnv(): EnvSnapshot { + return { ...process.env }; +} + +export function restoreEnv(snapshot: EnvSnapshot): void { + process.env = snapshot; +} + +export function requireIntegrationEnv( + envVars: string[], + context: string, +): void { + const missing = envVars.filter((envVar) => { + const value = process.env[envVar]; + return value === undefined || value.trim() === ""; + }); + + if (missing.length === 0) { + return; + } + + throw new Error( + `[integration:${context}] Missing required environment variables: ${missing.join(", ")}. ` + + "Set them before running this test.", + ); +} + +export function configureIntegrationEnv(context: string): void { + requireIntegrationEnv(["MITTWALD_API_TOKEN", "MITTWALD_API_BASE_URL"], context); +} diff --git a/src/test/integration/run-all-commands.test.ts b/src/test/integration/run-all-commands.test.ts new file mode 100644 index 000000000..1ef49173a --- /dev/null +++ b/src/test/integration/run-all-commands.test.ts @@ -0,0 +1,592 @@ +import { afterEach, beforeEach, describe, expect, it, jest } from "@jest/globals"; +import { appendFile, mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { + buildClassificationCatalogFromBuckets, + parseFailureCategory, + saveClassificationCatalog, +} from "./classification-catalog.js"; +import { runDevCommand } from "./command.js"; +import { discoverRunnableCommands } from "./command-discovery.js"; +import type { CommandWaiver, WaiverCategory } from "./command-discovery/types.js"; +import { loadCommandWaivers } from "./config/loader.js"; +import { configureIntegrationEnv, requireIntegrationEnv, restoreEnv, snapshotEnv } from "./env.js"; + +jest.setTimeout(20 * 60 * 1000); + +type FailureCategory = WaiverCategory; + +type MachineLogEntry = Record; + +const FAILURE_CATEGORIES: FailureCategory[] = [ + "ARG_MISUSE", + "INTERACTIVE_REQUIRED", + "RESOURCE_PRECONDITION", + "CONTRACT_SHAPE", + "COMMAND_BUG", +]; + +function createFailureBuckets(): Record { + return { + ARG_MISUSE: [], + INTERACTIVE_REQUIRED: [], + RESOURCE_PRECONDITION: [], + CONTRACT_SHAPE: [], + COMMAND_BUG: [], + }; +} + +function classifyFailure(output: { stderr: string; stdout: string }): FailureCategory { + const text = `${output.stderr}\n${output.stdout}`.toLowerCase(); + + if ( + /missing\s+(?:\d+\s+)?required arg|missing\s+(?:\d+\s+)?required flag|exactly one of|required options|unexpected argument|unknown flag|nonexistent flag|invalid flag|flag .* expects|no .* id given|you need to specify at least one/i.test( + text, + ) + ) { + return "ARG_MISUSE"; + } + + if ( + /prompt|interactive|addinput|addselect|addconfirmation|overwrite\?|token file already exists|tty/i.test( + text, + ) + ) { + return "INTERACTIVE_REQUIRED"; + } + + if ( + /not found|does not exist|no .* found|resource.*missing|404|forbidden|unauthorized|no project found|failed to connect|could not resolve hostname|name or service not known|no main user found|main mysql user can not be deleted manually/i.test( + text, + ) + ) { + return "RESOURCE_PRECONDITION"; + } + + if ( + /invalid version|not iterable|cannot read properties|undefined.*data|validation|invalid type|schema/i.test( + text, + ) + ) { + return "CONTRACT_SHAPE"; + } + + return "COMMAND_BUG"; +} + +function parseInvocationPartsFromArgs( + args: string[], + commandTokenCount: number, +): { positionalValues: string[]; flagValues: Map } { + const positionalValues: string[] = []; + const flagValues = new Map(); + + const invocationArgs = args.slice(commandTokenCount); + for (let i = 0; i < invocationArgs.length; i += 1) { + const token = invocationArgs[i]; + if (!token.startsWith("--")) { + positionalValues.push(token); + continue; + } + + const withoutPrefix = token.slice(2); + const eqIndex = withoutPrefix.indexOf("="); + let name = withoutPrefix; + let value: string | undefined; + + if (eqIndex >= 0) { + name = withoutPrefix.slice(0, eqIndex); + value = withoutPrefix.slice(eqIndex + 1); + } else { + const nextToken = invocationArgs[i + 1]; + if (nextToken && !nextToken.startsWith("--")) { + value = nextToken; + i += 1; + } + } + + const values = flagValues.get(name) ?? []; + values.push(value ?? "true"); + flagValues.set(name, values); + } + + return { positionalValues, flagValues }; +} + +function validateInvocationCompleteness(command: Awaited>[number]): string[] { + const issues: string[] = []; + const { positionalValues, flagValues } = parseInvocationPartsFromArgs( + command.synthesizedInvocation.args, + command.commandTokens.length, + ); + + command.parsedArgs.forEach((arg, index) => { + if (!arg.required) { + return; + } + if (positionalValues[index] === undefined) { + issues.push(`missing required arg ${arg.name}`); + } + }); + + for (const flag of command.parsedFlags) { + if (flag.required && !flagValues.has(flag.name)) { + issues.push(`missing required flag --${flag.name}`); + } + } + + const exactlyOneGroups = new Map(); + for (const flag of command.parsedFlags) { + if (!flag.exactlyOne || flag.exactlyOne.length < 2) { + continue; + } + const members = [...new Set(flag.exactlyOne)].sort(); + exactlyOneGroups.set(members.join("|"), members); + } + + for (const members of exactlyOneGroups.values()) { + const selected = members.filter((member) => flagValues.has(member)); + if (selected.length !== 1) { + issues.push(`exactly-one unresolved [${members.join(",")}]`); + } + } + + return issues; +} + +function logFailureTaxonomySummary( + failuresByCategory: Record, +): void { + logProgress("[run-all] failure taxonomy summary:"); + + for (const category of FAILURE_CATEGORIES) { + const commands = failuresByCategory[category]; + const sample = commands.slice(0, 5).join(", "); + logProgress( + `[run-all] ${category.padEnd(22, " ")} count=${String(commands.length).padStart(3, " ")} sample=${sample || "-"}`, + ); + } +} + +function mapCommandWaivers(waivers: CommandWaiver[]): { + waiversByCommandId: Map; + duplicates: string[]; +} { + const waiversByCommandId = new Map(); + const duplicates: string[] = []; + + for (const waiver of waivers) { + if (waiversByCommandId.has(waiver.commandId)) { + duplicates.push(waiver.commandId); + continue; + } + + waiversByCommandId.set(waiver.commandId, waiver); + } + + return { waiversByCommandId, duplicates }; +} + +function logWaiverSummary(waivedByCategory: Record): void { + logProgress("[run-all] waiver summary:"); + + for (const category of FAILURE_CATEGORIES) { + const commands = waivedByCategory[category]; + const sample = commands.slice(0, 5).join(", "); + logProgress( + `[run-all] ${category.padEnd(22, " ")} count=${String(commands.length).padStart(3, " ")} sample=${sample || "-"}`, + ); + } +} + +function logProgress(message: string): void { + process.stderr.write(`${message}\n`); +} + +function formatOutputBlock(output: string): string { + const trimmed = output.trim(); + return trimmed.length > 0 ? trimmed : ""; +} + +function logCommandFailureOutput( + position: string, + commandId: string, + result: { stdout: string; stderr: string }, +): void { + logProgress(`[${position}] diagnostics ${commandId}: stderr >>>`); + logProgress(formatOutputBlock(result.stderr)); + logProgress(`[${position}] diagnostics ${commandId}: stdout >>>`); + logProgress(formatOutputBlock(result.stdout)); + logProgress(`[${position}] diagnostics ${commandId}: <<<`); +} + +async function initializeMachineLogFile(filePath: string): Promise { + await mkdir(path.dirname(filePath), { recursive: true }); + await writeFile(filePath, "", "utf-8"); +} + +async function appendMachineLogEntry( + filePath: string, + entry: MachineLogEntry, +): Promise { + const line = JSON.stringify({ + timestamp: new Date().toISOString(), + ...entry, + }); + await appendFile(filePath, `${line}\n`, "utf-8"); +} + +async function seedProjectContext(projectId: string): Promise { + const configDir = process.env.MW_CONFIG_DIR; + + if (!configDir) { + throw new Error( + "[integration:run-all-commands] MW_CONFIG_DIR was not set before seeding project context.", + ); + } + + const contextFile = path.join(configDir, "context.json"); + + await mkdir(configDir, { recursive: true }); + await writeFile( + contextFile, + JSON.stringify({ + "project-id": projectId, + "server-id": "6b4f48f5-d80c-4d20-9db8-fecf4c9e6221", + "installation-id": "f7b47c12-7d11-4f3a-b9bc-1b3c706e1d55", + "org-id": "88e8d927-7db4-42ef-ae02-f8a7ef0b4d77", + }), + "utf-8", + ); +} + +describe("integration: run all commands", () => { + let originalEnv: NodeJS.ProcessEnv; + let tempConfigDir: string; + + beforeEach(async () => { + originalEnv = snapshotEnv(); + tempConfigDir = await mkdtemp(path.join(tmpdir(), "mw-int-config-")); + process.env.MW_CONFIG_DIR = tempConfigDir; + }); + + afterEach(async () => { + restoreEnv(originalEnv); + await rm(tempConfigDir, { recursive: true, force: true }); + }); + + it("discovers and executes every command once", async () => { + configureIntegrationEnv("run-all-commands"); + requireIntegrationEnv(["MW_TEST_PROJECT_ID"], "run-all-commands"); + + const categoryFilterRaw = process.env.MW_TEST_CATEGORY?.trim(); + const categoryFilter = categoryFilterRaw + ? parseFailureCategory(categoryFilterRaw) + : undefined; + const classificationCatalogPath = + process.env.MW_TEST_CLASSIFICATION_CATALOG_PATH?.trim() || undefined; + const machineLogPath = + process.env.MW_TEST_MACHINE_LOG_PATH?.trim() || + path.resolve("run-all-commands.ndjson"); + + await initializeMachineLogFile(machineLogPath); + logProgress(`[run-all] machine log path=${machineLogPath}`); + + const projectId = process.env.MW_TEST_PROJECT_ID!.trim(); + await seedProjectContext(projectId); + logProgress( + `[run-all] using context project-id from MW_TEST_PROJECT_ID (${projectId}); MW_CONFIG_DIR=${process.env.MW_CONFIG_DIR}`, + ); + + logProgress("[run-all] starting command discovery"); + const commands = await discoverRunnableCommands({ + onProgress: logProgress, + categoryFilter, + classificationCatalogPath, + }); + expect(commands.length).toBeGreaterThan(0); + + if (categoryFilter) { + logProgress( + `[run-all] category filter active: ${categoryFilter}${classificationCatalogPath ? ` (catalog=${classificationCatalogPath})` : ""}`, + ); + } + + const waivers = loadCommandWaivers(); + const { waiversByCommandId, duplicates } = mapCommandWaivers(waivers); + + await appendMachineLogEntry(machineLogPath, { + event: "run-start", + categoryFilter: categoryFilter ?? null, + classificationCatalogPath: classificationCatalogPath ?? null, + projectId, + commandCount: commands.length, + waiverCount: waivers.length, + }); + + logProgress(`[run-all] discovered ${commands.length} commands to execute`); + logProgress(`[run-all] loaded ${waivers.length} waiver entries`); + + const staleExampleCommands = commands.filter( + (command) => command.synthesizedInvocation.staleExample, + ); + const extractionDiagnostics = commands + .flatMap((command) => command.extractionDiagnostics) + .length; + logProgress( + `[run-all] stale examples detected=${staleExampleCommands.length}; extraction diagnostics=${extractionDiagnostics}`, + ); + + const infrastructureFailures: string[] = []; + const failuresByCategory = createFailureBuckets(); + const waivedByCategory = createFailureBuckets(); + let successfulCommands = 0; + let failedCommands = 0; + let waivedSkippedCommands = 0; + + if (!categoryFilter) { + if (duplicates.length > 0) { + infrastructureFailures.push( + `[waivers] duplicate waiver commandId entries: ${duplicates.join(", ")}`, + ); + } + + const discoveredCommandIds = new Set(commands.map((command) => command.commandId)); + for (const waiver of waivers) { + if (!discoveredCommandIds.has(waiver.commandId)) { + infrastructureFailures.push( + `[waivers] command '${waiver.commandId}' has a waiver but is not part of current discovery output`, + ); + } + } + } else { + logProgress("[waivers] strict waiver integrity checks skipped (category filter active)"); + } + + for (const [index, command] of commands.entries()) { + const position = `${index + 1}/${commands.length}`; + const invocation = command.synthesizedInvocation; + const waiver = waiversByCommandId.get(command.commandId); + const commandStartedAt = Date.now(); + + await seedProjectContext(projectId); + logProgress( + `[${position}] running ${command.commandId} (source=${invocation.argumentSource}; interactive=${invocation.interactiveDecision}; re-seeded project context)`, + ); + + await appendMachineLogEntry(machineLogPath, { + event: "command-start", + index: index + 1, + total: commands.length, + position, + commandId: command.commandId, + sourceFile: command.sourceFile, + commandTokens: command.commandTokens, + parsedArgs: command.parsedArgs, + parsedFlags: command.parsedFlags, + interactiveSignals: command.interactiveSignals, + invocationProfilesApplied: command.invocationProfilesApplied, + extractionDiagnostics: command.extractionDiagnostics, + invocationArgs: invocation.args, + argumentSource: invocation.argumentSource, + interactiveDecision: invocation.interactiveDecision, + }); + + if (waiver) { + waivedSkippedCommands += 1; + waivedByCategory[waiver.category].push(command.commandId); + logProgress( + `[${position}] waived ${command.commandId} (category=${waiver.category}; reason=${waiver.reason}${waiver.issue ? `; issue=${waiver.issue}` : ""})`, + ); + await appendMachineLogEntry(machineLogPath, { + event: "command-result", + index: index + 1, + total: commands.length, + position, + commandId: command.commandId, + status: "waived", + durationMs: Date.now() - commandStartedAt, + waiver, + }); + continue; + } + + if (invocation.interactiveDecision === "INTERACTIVE_REQUIRED") { + failedCommands += 1; + failuresByCategory.INTERACTIVE_REQUIRED.push(command.commandId); + infrastructureFailures.push( + `[waivers] ${command.commandId} was classified INTERACTIVE_REQUIRED but has no waiver entry`, + ); + logProgress( + `[${position}] classified ${command.commandId} as INTERACTIVE_REQUIRED (missing waiver entry)`, + ); + await appendMachineLogEntry(machineLogPath, { + event: "command-result", + index: index + 1, + total: commands.length, + position, + commandId: command.commandId, + status: "failed", + failureCategory: "INTERACTIVE_REQUIRED", + durationMs: Date.now() - commandStartedAt, + details: "classified INTERACTIVE_REQUIRED but no waiver entry exists", + }); + continue; + } + + const staticInvocationIssues = validateInvocationCompleteness(command); + if (staticInvocationIssues.length > 0) { + failedCommands += 1; + failuresByCategory.ARG_MISUSE.push(command.commandId); + logProgress( + `[${position}] preflight ${command.commandId} classified as ARG_MISUSE (${staticInvocationIssues.join("; ")})`, + ); + await appendMachineLogEntry(machineLogPath, { + event: "command-result", + index: index + 1, + total: commands.length, + position, + commandId: command.commandId, + status: "failed", + failureCategory: "ARG_MISUSE", + durationMs: Date.now() - commandStartedAt, + preflightIssues: staticInvocationIssues, + }); + continue; + } + + const result = await runDevCommand(invocation.args, { + timeoutMs: 30_000, + }); + + if (result.timedOut) { + failedCommands += 1; + failuresByCategory.COMMAND_BUG.push(command.commandId); + logProgress(`[${position}] timeout ${command.commandId}`); + logCommandFailureOutput(position, command.commandId, result); + await appendMachineLogEntry(machineLogPath, { + event: "command-result", + index: index + 1, + total: commands.length, + position, + commandId: command.commandId, + status: "failed", + failureCategory: "COMMAND_BUG", + durationMs: Date.now() - commandStartedAt, + timedOut: true, + stdout: result.stdout, + stderr: result.stderr, + }); + continue; + } + + if (result.exitCode === null) { + failedCommands += 1; + infrastructureFailures.push( + `${command.commandId} failed to execute (source=${invocation.argumentSource}): ${result.error?.message ?? "unknown error"}`, + ); + logProgress( + `[${position}] spawn-error ${command.commandId}: ${result.error?.message ?? "unknown error"}`, + ); + logCommandFailureOutput(position, command.commandId, result); + await appendMachineLogEntry(machineLogPath, { + event: "command-result", + index: index + 1, + total: commands.length, + position, + commandId: command.commandId, + status: "spawn-error", + durationMs: Date.now() - commandStartedAt, + errorMessage: result.error?.message ?? "unknown error", + stdout: result.stdout, + stderr: result.stderr, + }); + continue; + } + + if (result.exitCode !== 0) { + failedCommands += 1; + const category = classifyFailure(result); + failuresByCategory[category].push(command.commandId); + logProgress( + `[${position}] classified ${command.commandId} as ${category}`, + ); + logCommandFailureOutput(position, command.commandId, result); + + await appendMachineLogEntry(machineLogPath, { + event: "command-result", + index: index + 1, + total: commands.length, + position, + commandId: command.commandId, + status: "failed", + failureCategory: category, + durationMs: Date.now() - commandStartedAt, + exitCode: result.exitCode, + stdout: result.stdout, + stderr: result.stderr, + }); + } else { + successfulCommands += 1; + + await appendMachineLogEntry(machineLogPath, { + event: "command-result", + index: index + 1, + total: commands.length, + position, + commandId: command.commandId, + status: "succeeded", + durationMs: Date.now() - commandStartedAt, + exitCode: result.exitCode, + }); + } + + logProgress( + `[${position}] finished ${command.commandId} (exitCode=${result.exitCode})`, + ); + } + + logProgress(`[run-all] execution complete: ${commands.length} run`); + logProgress( + `[run-all] statistics: successful=${successfulCommands}, failed=${failedCommands}, waived-skipped=${waivedSkippedCommands}, total=${commands.length}`, + ); + logFailureTaxonomySummary(failuresByCategory); + logWaiverSummary(waivedByCategory); + + await appendMachineLogEntry(machineLogPath, { + event: "run-summary", + statistics: { + successful: successfulCommands, + failed: failedCommands, + waivedSkipped: waivedSkippedCommands, + total: commands.length, + }, + failuresByCategory, + waivedByCategory, + infrastructureFailures, + }); + + if (!categoryFilter) { + const classificationCatalog = buildClassificationCatalogFromBuckets({ + failuresByCategory, + waivedByCategory, + statistics: { + successful: successfulCommands, + failed: failedCommands, + waivedSkipped: waivedSkippedCommands, + total: commands.length, + }, + }); + + await saveClassificationCatalog(classificationCatalog); + logProgress( + `[run-all] wrote classification catalog with ${classificationCatalog.entries.length} entries`, + ); + } else { + logProgress("[run-all] skipped classification catalog write (category filter active)"); + } + + expect(infrastructureFailures).toEqual([]); + }); +}); diff --git a/src/test/integration/tools/generate-command-endpoint-map.ts b/src/test/integration/tools/generate-command-endpoint-map.ts new file mode 100644 index 000000000..5f856081f --- /dev/null +++ b/src/test/integration/tools/generate-command-endpoint-map.ts @@ -0,0 +1,1135 @@ +import fs from "node:fs"; +import path from "node:path"; +import ts from "typescript"; + +type CommandReference = { + commandId: string; + sourceFile: string; +}; + +type FailureCategory = + | "ARG_MISUSE" + | "INTERACTIVE_REQUIRED" + | "RESOURCE_PRECONDITION" + | "CONTRACT_SHAPE" + | "COMMAND_BUG"; + +type CliOptions = { + machineLogPath?: string; + category?: FailureCategory; + openapiPath: string; + outputJsonPath: string; + outputMarkdownPath: string; +}; + +type NdjsonRecord = { + timestamp?: string; + event?: string; + commandId?: string; + sourceFile?: string; + status?: string; + failureCategory?: FailureCategory; +}; + +type CommandLogRecord = { + status?: string; + failureCategory?: FailureCategory; +}; + +type ApiCallUsage = { + group: string; + method: string; + groupMethod: string; + filePath: string; +}; + +type DescriptorMeta = { + descriptorName: string; + path: string | null; + httpMethod: string | null; + operationId: string | null; +}; + +type OpenApiOperation = { + operationId: string | null; + deprecated: boolean; +}; + +type ResolvedEndpoint = { + groupMethod: string; + descriptorName: string | null; + descriptorPath: string | null; + descriptorHttpMethod: string | null; + descriptorOperationId: string | null; + openapiOperationId: string | null; + openapiDeprecated: boolean | null; + openapiStatus: "FOUND" | "MISSING_PATH" | "MISSING_METHOD" | "MISSING_DESCRIPTOR"; +}; + +type CommandMappingEntry = { + commandId: string; + sourceFile: string; + transitiveFiles: string[]; + logStatus: string | null; + logCategory: FailureCategory | null; + apiCalls: ApiCallUsage[]; + resolvedEndpoints: ResolvedEndpoint[]; + unresolvedGroupMethods: string[]; +}; + +type MappingOutput = { + generatedAt: string; + inputs: { + machineLogPath: string | null; + category: FailureCategory | null; + openapiPath: string; + }; + statistics: { + commandCount: number; + commandWithApiCalls: number; + unresolvedGroupMethodCount: number; + deprecatedEndpointCount: number; + }; + entries: CommandMappingEntry[]; +}; + +type FileImportBinding = { + sourceFilePath: string; + importedName: string; +}; + +type FunctionInfo = { + localCalls: Set; + importedCalls: Map; + apiCalls: ApiCallUsage[]; +}; + +type FileAnalysis = { + imports: Map; + localFunctions: Map; + exports: Map; + functionInfos: Map; + rootInfo: FunctionInfo; +}; + +type TraversalState = { + visitedFiles: Set; + visitedFunctions: Set; + apiCalls: ApiCallUsage[]; +}; + +const DEFAULT_OPENAPI_PATH = "openapi.json"; +const DEFAULT_OUTPUT_JSON_PATH = "command-endpoint-map.json"; +const DEFAULT_OUTPUT_MARKDOWN_PATH = "command-endpoint-map.md"; +const DEFAULT_MACHINE_LOG_PATH = "run-all-commands.ndjson"; + +function parseCliOptions(argv: string[]): CliOptions { + const options: CliOptions = { + openapiPath: DEFAULT_OPENAPI_PATH, + outputJsonPath: DEFAULT_OUTPUT_JSON_PATH, + outputMarkdownPath: DEFAULT_OUTPUT_MARKDOWN_PATH, + }; + + for (let i = 2; i < argv.length; i += 1) { + const token = argv[i]; + + if (token === "--machine-log") { + options.machineLogPath = requireNextArg(argv, i, token); + i += 1; + continue; + } + + if (token === "--category") { + const raw = requireNextArg(argv, i, token); + options.category = parseFailureCategory(raw); + i += 1; + continue; + } + + if (token === "--openapi") { + options.openapiPath = requireNextArg(argv, i, token); + i += 1; + continue; + } + + if (token === "--output-json") { + options.outputJsonPath = requireNextArg(argv, i, token); + i += 1; + continue; + } + + if (token === "--output-md") { + options.outputMarkdownPath = requireNextArg(argv, i, token); + i += 1; + continue; + } + + if (token === "--help" || token === "-h") { + printHelp(); + process.exit(0); + } + + throw new Error(`Unknown argument: ${token}`); + } + + return options; +} + +function requireNextArg(argv: string[], index: number, token: string): string { + const value = argv[index + 1]; + if (!value) { + throw new Error(`Missing value for ${token}`); + } + return value; +} + +function parseFailureCategory(value: string): FailureCategory { + const categories: FailureCategory[] = [ + "ARG_MISUSE", + "INTERACTIVE_REQUIRED", + "RESOURCE_PRECONDITION", + "CONTRACT_SHAPE", + "COMMAND_BUG", + ]; + + if (!categories.includes(value as FailureCategory)) { + throw new Error(`Invalid category '${value}'. Expected one of ${categories.join(", ")}`); + } + + return value as FailureCategory; +} + +function printHelp(): void { + process.stdout.write(`Usage:\n` + + ` yarn tool:integration:generate-command-endpoint-map [options]\n\n` + + `Options:\n` + + ` --machine-log NDJSON log from run-all integration test (default: ${DEFAULT_MACHINE_LOG_PATH})\n` + + ` --category Optional failure category filter\n` + + ` --openapi OpenAPI JSON file (default: ${DEFAULT_OPENAPI_PATH})\n` + + ` --output-json Output JSON mapping (default: ${DEFAULT_OUTPUT_JSON_PATH})\n` + + ` --output-md Output markdown summary (default: ${DEFAULT_OUTPUT_MARKDOWN_PATH})\n` + + ` -h, --help Show this help\n`); +} + +async function main(): Promise { + const options = parseCliOptions(process.argv); + const openapiPath = path.resolve(options.openapiPath); + const outputJsonPath = path.resolve(options.outputJsonPath); + const outputMarkdownPath = path.resolve(options.outputMarkdownPath); + + const machineLogPath = path.resolve( + options.machineLogPath ?? DEFAULT_MACHINE_LOG_PATH, + ); + + if (!fs.existsSync(machineLogPath)) { + throw new Error( + `Machine log not found at ${machineLogPath}. Run the integration command runner first to produce command-start and command-result events.`, + ); + } + + const machineLogData = loadMachineLogData(machineLogPath); + + const filteredCommands = filterCommands( + machineLogData.commands, + machineLogData.commandLogById, + options.category, + ); + + const groupMethodToDescriptor = buildGroupMethodToDescriptorIndex(); + const descriptorMetaByName = buildDescriptorMetaIndex(); + const openapi = JSON.parse(fs.readFileSync(openapiPath, "utf8")) as { + paths?: Record>; + }; + + const entries = filteredCommands.map((command) => { + const sourceAbsPath = path.resolve(process.cwd(), "src/commands", command.sourceFile); + const analysis = analyzeCommandTransitive(sourceAbsPath); + const uniqueApiCalls = deduplicateApiCalls(analysis.apiCalls); + + const resolvedEndpoints = resolveEndpoints( + uniqueApiCalls, + groupMethodToDescriptor, + descriptorMetaByName, + openapi, + ); + + const unresolvedGroupMethods = resolvedEndpoints + .filter((endpoint) => endpoint.openapiStatus === "MISSING_DESCRIPTOR") + .map((endpoint) => endpoint.groupMethod); + + const logRecord = machineLogData.commandLogById.get(command.commandId); + + return { + commandId: command.commandId, + sourceFile: command.sourceFile, + transitiveFiles: Array.from(analysis.visitedFiles) + .map((filePath) => path.relative(process.cwd(), filePath)) + .sort((a, b) => a.localeCompare(b)), + logStatus: logRecord?.status ?? null, + logCategory: logRecord?.failureCategory ?? null, + apiCalls: uniqueApiCalls + .map((call) => ({ + ...call, + filePath: path.relative(process.cwd(), call.filePath), + })) + .sort((a, b) => { + const methodCmp = a.groupMethod.localeCompare(b.groupMethod); + return methodCmp !== 0 ? methodCmp : a.filePath.localeCompare(b.filePath); + }), + resolvedEndpoints, + unresolvedGroupMethods, + } satisfies CommandMappingEntry; + }); + + const output: MappingOutput = { + generatedAt: new Date().toISOString(), + inputs: { + machineLogPath: fs.existsSync(machineLogPath) + ? path.relative(process.cwd(), machineLogPath) + : null, + category: options.category ?? null, + openapiPath: path.relative(process.cwd(), openapiPath), + }, + statistics: { + commandCount: entries.length, + commandWithApiCalls: entries.filter((entry) => entry.apiCalls.length > 0).length, + unresolvedGroupMethodCount: entries.reduce( + (sum, entry) => sum + entry.unresolvedGroupMethods.length, + 0, + ), + deprecatedEndpointCount: entries.reduce( + (sum, entry) => + sum + + entry.resolvedEndpoints.filter((endpoint) => endpoint.openapiDeprecated === true) + .length, + 0, + ), + }, + entries, + }; + + fs.writeFileSync(outputJsonPath, `${JSON.stringify(output, null, 2)}\n`, "utf8"); + fs.writeFileSync(outputMarkdownPath, renderMarkdown(output), "utf8"); + + process.stdout.write( + `Wrote ${path.relative(process.cwd(), outputJsonPath)} and ${path.relative(process.cwd(), outputMarkdownPath)} for ${entries.length} commands.\n`, + ); +} + +function loadMachineLogData(machineLogPath: string): { + commands: CommandReference[]; + commandLogById: Map; +} { + const lines = fs + .readFileSync(machineLogPath, "utf8") + .split("\n") + .map((line) => line.trim()) + .filter((line) => line.length > 0); + + const commandLogById = new Map(); + const commandById = new Map(); + + for (let idx = 0; idx < lines.length; idx += 1) { + const line = lines[idx]; + let parsed: NdjsonRecord; + + try { + parsed = JSON.parse(line) as NdjsonRecord; + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + throw new Error(`Invalid NDJSON at ${machineLogPath}:${idx + 1}: ${message}`); + } + + if (parsed.event === "command-start") { + if (typeof parsed.commandId !== "string" || typeof parsed.sourceFile !== "string") { + continue; + } + + if (!commandById.has(parsed.commandId)) { + commandById.set(parsed.commandId, { + commandId: parsed.commandId, + sourceFile: parsed.sourceFile, + }); + } + continue; + } + + if (parsed.event === "command-result") { + if (typeof parsed.commandId !== "string") { + continue; + } + + commandLogById.set(parsed.commandId, { + status: parsed.status, + failureCategory: parsed.failureCategory, + }); + } + } + + const commands = Array.from(commandById.values()).sort((a, b) => + a.commandId.localeCompare(b.commandId), + ); + + if (commands.length === 0) { + throw new Error( + `No command-start entries with sourceFile found in ${machineLogPath}. Ensure run-all integration test writes discovery metadata to the machine log.`, + ); + } + + return { + commands, + commandLogById, + }; +} + +function filterCommands( + commands: CommandReference[], + commandLogById: Map, + category: FailureCategory | undefined, +): CommandReference[] { + if (!category) { + return commands; + } + + return commands.filter((command) => { + const record = commandLogById.get(command.commandId); + return record?.failureCategory === category; + }); +} + +function buildGroupMethodToDescriptorIndex(): Map { + const clientPath = path.resolve( + process.cwd(), + "node_modules/@mittwald/api-client/dist/esm/generated/v2/client.js", + ); + const sourceText = fs.readFileSync(clientPath, "utf8"); + const sourceFile = ts.createSourceFile( + clientPath, + sourceText, + ts.ScriptTarget.Latest, + true, + ts.ScriptKind.JS, + ); + + const index = new Map(); + + const visit = (node: ts.Node): void => { + if (ts.isPropertyAssignment(node) && ts.isIdentifier(node.name)) { + const methodName = node.name.text; + const initializer = node.initializer; + + if (ts.isCallExpression(initializer)) { + const maybeRequestFactory = initializer.expression; + if ( + ts.isPropertyAccessExpression(maybeRequestFactory) && + maybeRequestFactory.name.text === "requestFunctionFactory" && + initializer.arguments.length === 1 + ) { + const arg = initializer.arguments[0]; + if ( + ts.isPropertyAccessExpression(arg) && + ts.isIdentifier(arg.expression) && + arg.expression.text === "descriptors" + ) { + const descriptorName = arg.name.text; + const groupName = getEnclosingGroupName(node); + if (groupName) { + index.set(`${groupName}.${methodName}`, descriptorName); + } + } + } + } + } + + ts.forEachChild(node, visit); + }; + + visit(sourceFile); + return index; +} + +function getEnclosingGroupName(node: ts.Node): string | null { + const objectLiteral = node.parent; + if (!ts.isObjectLiteralExpression(objectLiteral)) { + return null; + } + + const parent = objectLiteral.parent; + + if (ts.isPropertyAssignment(parent) && ts.isIdentifier(parent.name)) { + return parent.name.text; + } + + if (ts.isPropertyDeclaration(parent) && ts.isIdentifier(parent.name)) { + return parent.name.text; + } + + return null; +} + +function buildDescriptorMetaIndex(): Map { + const descriptorsPath = path.resolve( + process.cwd(), + "node_modules/@mittwald/api-client/dist/esm/generated/v2/descriptors.js", + ); + + const sourceText = fs.readFileSync(descriptorsPath, "utf8"); + const sourceFile = ts.createSourceFile( + descriptorsPath, + sourceText, + ts.ScriptTarget.Latest, + true, + ts.ScriptKind.JS, + ); + + const map = new Map(); + + for (const statement of sourceFile.statements) { + if (!ts.isVariableStatement(statement)) { + continue; + } + + const hasExport = statement.modifiers?.some( + (modifier) => modifier.kind === ts.SyntaxKind.ExportKeyword, + ); + if (!hasExport) { + continue; + } + + for (const decl of statement.declarationList.declarations) { + if (!ts.isIdentifier(decl.name) || !decl.initializer) { + continue; + } + + const descriptorName = decl.name.text; + if (!ts.isObjectLiteralExpression(decl.initializer)) { + continue; + } + + let apiPath: string | null = null; + let httpMethod: string | null = null; + let operationId: string | null = null; + + for (const prop of decl.initializer.properties) { + if (!ts.isPropertyAssignment(prop) || !ts.isIdentifier(prop.name)) { + continue; + } + + const key = prop.name.text; + const value = prop.initializer; + + if (key === "path" && ts.isStringLiteralLike(value)) { + apiPath = value.text; + continue; + } + + if (key === "method" && ts.isStringLiteralLike(value)) { + httpMethod = value.text; + continue; + } + + if (key === "operationId" && ts.isStringLiteralLike(value)) { + operationId = value.text; + } + } + + map.set(descriptorName, { + descriptorName, + path: apiPath, + httpMethod, + operationId, + }); + } + } + + return map; +} + +function analyzeCommandTransitive(commandFilePath: string): { + visitedFiles: Set; + apiCalls: ApiCallUsage[]; +} { + const state: TraversalState = { + visitedFiles: new Set(), + visitedFunctions: new Set(), + apiCalls: [], + }; + + traverseFile(commandFilePath, null, state); + + return { + visitedFiles: state.visitedFiles, + apiCalls: state.apiCalls, + }; +} + +function traverseFile( + filePath: string, + exportToFollow: string | null, + state: TraversalState, +): void { + const normalizedPath = path.resolve(filePath); + const fileCacheKey = normalizedPath; + + const analysis = analyzeFile(normalizedPath); + state.visitedFiles.add(normalizedPath); + + if (exportToFollow === null) { + enqueueFunctionInfo(analysis.rootInfo, normalizedPath, state); + + for (const localName of analysis.rootInfo.localCalls) { + followLocalFunction(analysis, normalizedPath, localName, state); + } + + for (const binding of analysis.rootInfo.importedCalls.values()) { + followImportedBinding(binding, state); + } + + return; + } + + const localName = analysis.exports.get(exportToFollow); + if (!localName) { + return; + } + + followLocalFunction(analysis, fileCacheKey, localName, state); +} + +function followLocalFunction( + analysis: FileAnalysis, + filePath: string, + localName: string, + state: TraversalState, +): void { + const key = `${filePath}::${localName}`; + if (state.visitedFunctions.has(key)) { + return; + } + state.visitedFunctions.add(key); + + const info = analysis.functionInfos.get(localName); + if (!info) { + return; + } + + enqueueFunctionInfo(info, filePath, state); + + for (const nestedLocal of info.localCalls) { + followLocalFunction(analysis, filePath, nestedLocal, state); + } + + for (const binding of info.importedCalls.values()) { + followImportedBinding(binding, state); + } +} + +function followImportedBinding(binding: FileImportBinding, state: TraversalState): void { + if (binding.importedName === "*") { + return; + } + + traverseFile(binding.sourceFilePath, binding.importedName, state); +} + +function enqueueFunctionInfo( + info: FunctionInfo, + filePath: string, + state: TraversalState, +): void { + for (const call of info.apiCalls) { + state.apiCalls.push({ ...call, filePath }); + } +} + +const fileAnalysisCache = new Map(); + +function analyzeFile(filePath: string): FileAnalysis { + const normalized = path.resolve(filePath); + const cached = fileAnalysisCache.get(normalized); + if (cached) { + return cached; + } + + const sourceText = fs.readFileSync(normalized, "utf8"); + const scriptKind = normalized.endsWith(".tsx") ? ts.ScriptKind.TSX : ts.ScriptKind.TS; + const sourceFile = ts.createSourceFile( + normalized, + sourceText, + ts.ScriptTarget.Latest, + true, + scriptKind, + ); + + const imports = new Map(); + const localFunctions = new Map(); + const exports = new Map(); + + for (const stmt of sourceFile.statements) { + if (ts.isImportDeclaration(stmt) && stmt.importClause && ts.isStringLiteral(stmt.moduleSpecifier)) { + const moduleName = stmt.moduleSpecifier.text; + const resolvedImport = resolveRelativeImport(normalized, moduleName); + if (!resolvedImport) { + continue; + } + + if (stmt.importClause.name) { + imports.set(stmt.importClause.name.text, { + sourceFilePath: resolvedImport, + importedName: "default", + }); + } + + const bindings = stmt.importClause.namedBindings; + if (bindings && ts.isNamedImports(bindings)) { + for (const specifier of bindings.elements) { + const importedName = specifier.propertyName + ? specifier.propertyName.text + : specifier.name.text; + imports.set(specifier.name.text, { + sourceFilePath: resolvedImport, + importedName, + }); + } + } + + if (bindings && ts.isNamespaceImport(bindings)) { + imports.set(bindings.name.text, { + sourceFilePath: resolvedImport, + importedName: "*", + }); + } + } + + collectLocalAndExportedFunctions(stmt, localFunctions, exports); + } + + const functionInfos = new Map(); + for (const [name, node] of localFunctions.entries()) { + functionInfos.set(name, extractFunctionInfo(node, imports)); + } + + const rootInfo = extractRootInfo(sourceFile, imports, localFunctions); + + const result: FileAnalysis = { + imports, + localFunctions, + exports, + functionInfos, + rootInfo, + }; + + fileAnalysisCache.set(normalized, result); + return result; +} + +function collectLocalAndExportedFunctions( + stmt: ts.Statement, + localFunctions: Map, + exports: Map, +): void { + if (ts.isFunctionDeclaration(stmt) && stmt.name) { + localFunctions.set(stmt.name.text, stmt); + if (hasExportModifier(stmt)) { + exports.set(stmt.name.text, stmt.name.text); + } + return; + } + + if (ts.isVariableStatement(stmt)) { + const isExport = hasExportModifier(stmt); + + for (const decl of stmt.declarationList.declarations) { + if (!ts.isIdentifier(decl.name) || !decl.initializer) { + continue; + } + + if ( + ts.isArrowFunction(decl.initializer) || + ts.isFunctionExpression(decl.initializer) + ) { + localFunctions.set(decl.name.text, decl.initializer); + if (isExport) { + exports.set(decl.name.text, decl.name.text); + } + } + } + return; + } + + if (ts.isExportDeclaration(stmt) && stmt.exportClause && ts.isNamedExports(stmt.exportClause)) { + if (stmt.moduleSpecifier) { + return; + } + + for (const specifier of stmt.exportClause.elements) { + const exportName = specifier.name.text; + const localName = specifier.propertyName ? specifier.propertyName.text : exportName; + exports.set(exportName, localName); + } + return; + } + + if (ts.isExportAssignment(stmt) && ts.isIdentifier(stmt.expression)) { + exports.set("default", stmt.expression.text); + } +} + +function hasExportModifier(node: ts.Node): boolean { + const modifiers = ts.canHaveModifiers(node) ? ts.getModifiers(node) : undefined; + return !!modifiers?.some( + (modifier: ts.Modifier) => modifier.kind === ts.SyntaxKind.ExportKeyword, + ); +} + +function extractRootInfo( + sourceFile: ts.SourceFile, + imports: Map, + localFunctions: Map, +): FunctionInfo { + const localCalls = new Set(); + const importedCalls = new Map(); + const apiCalls: ApiCallUsage[] = []; + + const addCall = (group: string, method: string): void => { + apiCalls.push({ + group, + method, + groupMethod: `${group}.${method}`, + filePath: sourceFile.fileName, + }); + }; + + const visit = (node: ts.Node): void => { + if (ts.isCallExpression(node)) { + const callTarget = extractApiClientCall(node.expression); + if (callTarget) { + addCall(callTarget.group, callTarget.method); + } + + const callRefs = extractCallReferences(node.expression, imports, localFunctions); + for (const localName of callRefs.localCallNames) { + localCalls.add(localName); + } + for (const [name, binding] of callRefs.importedCalls.entries()) { + importedCalls.set(name, binding); + } + } + + ts.forEachChild(node, visit); + }; + + ts.forEachChild(sourceFile, visit); + + return { + localCalls, + importedCalls, + apiCalls, + }; +} + +function extractFunctionInfo( + node: ts.Node, + imports: Map, +): FunctionInfo { + const localCalls = new Set(); + const importedCalls = new Map(); + const apiCalls: ApiCallUsage[] = []; + + const enclosingFile = node.getSourceFile().fileName; + + const visit = (child: ts.Node): void => { + if (ts.isCallExpression(child)) { + const callTarget = extractApiClientCall(child.expression); + if (callTarget) { + apiCalls.push({ + group: callTarget.group, + method: callTarget.method, + groupMethod: `${callTarget.group}.${callTarget.method}`, + filePath: enclosingFile, + }); + } + + const callRefs = extractCallReferences(child.expression, imports, new Map()); + for (const localName of callRefs.localCallNames) { + localCalls.add(localName); + } + for (const [name, binding] of callRefs.importedCalls.entries()) { + importedCalls.set(name, binding); + } + } + + ts.forEachChild(child, visit); + }; + + ts.forEachChild(node, visit); + + return { + localCalls, + importedCalls, + apiCalls, + }; +} + +function extractCallReferences( + expression: ts.Expression, + imports: Map, + localFunctions: Map, +): { localCallNames: Set; importedCalls: Map } { + const localCallNames = new Set(); + const importedCalls = new Map(); + + if (ts.isIdentifier(expression)) { + const name = expression.text; + const binding = imports.get(name); + if (binding) { + importedCalls.set(name, binding); + } else if (localFunctions.has(name)) { + localCallNames.add(name); + } + return { localCallNames, importedCalls }; + } + + if (ts.isPropertyAccessExpression(expression) && ts.isIdentifier(expression.expression)) { + const namespaceBinding = imports.get(expression.expression.text); + if (namespaceBinding && namespaceBinding.importedName === "*") { + importedCalls.set( + `${expression.expression.text}.${expression.name.text}`, + { + sourceFilePath: namespaceBinding.sourceFilePath, + importedName: expression.name.text, + }, + ); + } + } + + return { localCallNames, importedCalls }; +} + +function extractApiClientCall( + expression: ts.Expression, +): { group: string; method: string } | null { + const parts = flattenPropertyAccess(expression); + if (!parts || parts.length < 3) { + return null; + } + + const apiClientIndex = parts.indexOf("apiClient"); + if (apiClientIndex >= 0 && parts.length >= apiClientIndex + 3) { + return { + group: parts[apiClientIndex + 1], + method: parts[apiClientIndex + 2], + }; + } + + const first = parts[0]; + if ((first === "apiClient" || first === "client") && parts.length >= 3) { + return { + group: parts[1], + method: parts[2], + }; + } + + return null; +} + +function flattenPropertyAccess(expression: ts.Expression): string[] | null { + if (expression.kind === ts.SyntaxKind.ThisKeyword) { + return ["this"]; + } + + if (expression.kind === ts.SyntaxKind.SuperKeyword) { + return ["super"]; + } + + if (ts.isIdentifier(expression)) { + return [expression.text]; + } + + if (ts.isPropertyAccessExpression(expression)) { + const left = flattenPropertyAccess(expression.expression); + if (!left) { + return null; + } + return [...left, expression.name.text]; + } + + if (ts.isElementAccessExpression(expression) && ts.isStringLiteral(expression.argumentExpression)) { + const left = flattenPropertyAccess(expression.expression); + if (!left) { + return null; + } + return [...left, expression.argumentExpression.text]; + } + + return null; +} + +function resolveRelativeImport(fromFilePath: string, specifier: string): string | null { + if (!specifier.startsWith(".")) { + return null; + } + + const fromDir = path.dirname(fromFilePath); + const base = path.resolve(fromDir, specifier); + + const candidates: string[] = []; + const ext = path.extname(base); + + if (ext.length > 0) { + candidates.push(base); + if (ext === ".js" || ext === ".mjs" || ext === ".cjs") { + candidates.push(base.slice(0, -ext.length) + ".ts"); + candidates.push(base.slice(0, -ext.length) + ".tsx"); + } + } else { + candidates.push(base + ".ts"); + candidates.push(base + ".tsx"); + candidates.push(path.join(base, "index.ts")); + candidates.push(path.join(base, "index.tsx")); + } + + for (const candidate of candidates) { + if (fs.existsSync(candidate) && fs.statSync(candidate).isFile()) { + return candidate; + } + } + + return null; +} + +function deduplicateApiCalls(calls: ApiCallUsage[]): ApiCallUsage[] { + const byKey = new Map(); + for (const call of calls) { + const key = `${call.groupMethod}::${call.filePath}`; + if (!byKey.has(key)) { + byKey.set(key, call); + } + } + return Array.from(byKey.values()); +} + +function resolveEndpoints( + apiCalls: ApiCallUsage[], + groupMethodToDescriptor: Map, + descriptorMetaByName: Map, + openapi: { + paths?: Record>; + }, +): ResolvedEndpoint[] { + return apiCalls.map((call) => { + const descriptorName = groupMethodToDescriptor.get(call.groupMethod); + + if (!descriptorName) { + return { + groupMethod: call.groupMethod, + descriptorName: null, + descriptorPath: null, + descriptorHttpMethod: null, + descriptorOperationId: null, + openapiOperationId: null, + openapiDeprecated: null, + openapiStatus: "MISSING_DESCRIPTOR", + }; + } + + const descriptor = descriptorMetaByName.get(descriptorName); + if (!descriptor || !descriptor.path || !descriptor.httpMethod) { + return { + groupMethod: call.groupMethod, + descriptorName, + descriptorPath: descriptor?.path ?? null, + descriptorHttpMethod: descriptor?.httpMethod ?? null, + descriptorOperationId: descriptor?.operationId ?? null, + openapiOperationId: null, + openapiDeprecated: null, + openapiStatus: "MISSING_DESCRIPTOR", + }; + } + + const operation = getOpenApiOperation(openapi, descriptor.path, descriptor.httpMethod); + + return { + groupMethod: call.groupMethod, + descriptorName, + descriptorPath: descriptor.path, + descriptorHttpMethod: descriptor.httpMethod, + descriptorOperationId: descriptor.operationId, + openapiOperationId: operation?.operationId ?? null, + openapiDeprecated: operation?.deprecated ?? null, + openapiStatus: operation + ? "FOUND" + : openapi.paths?.[descriptor.path] + ? "MISSING_METHOD" + : "MISSING_PATH", + }; + }); +} + +function getOpenApiOperation( + openapi: { + paths?: Record>; + }, + apiPath: string, + httpMethod: string, +): OpenApiOperation | null { + const pathItem = openapi.paths?.[apiPath]; + if (!pathItem) { + return null; + } + + const methodItem = pathItem[httpMethod.toLowerCase()]; + if (!methodItem) { + return null; + } + + return { + operationId: typeof methodItem.operationId === "string" ? methodItem.operationId : null, + deprecated: methodItem.deprecated === true, + }; +} + +function renderMarkdown(output: MappingOutput): string { + const lines: string[] = []; + + lines.push("# Command Endpoint Mapping"); + lines.push(""); + lines.push(`- Generated at: ${output.generatedAt}`); + lines.push(`- Machine log: ${output.inputs.machineLogPath ?? ""}`); + lines.push(`- Category filter: ${output.inputs.category ?? ""}`); + lines.push(`- OpenAPI: ${output.inputs.openapiPath}`); + lines.push(""); + + lines.push("## Statistics"); + lines.push(""); + lines.push(`- Commands: ${output.statistics.commandCount}`); + lines.push(`- Commands with API calls: ${output.statistics.commandWithApiCalls}`); + lines.push(`- Unresolved group methods: ${output.statistics.unresolvedGroupMethodCount}`); + lines.push(`- Deprecated endpoints: ${output.statistics.deprecatedEndpointCount}`); + lines.push(""); + + for (const entry of output.entries) { + lines.push(`## ${entry.commandId}`); + lines.push(""); + lines.push(`- Source file: ${entry.sourceFile}`); + lines.push(`- Log status: ${entry.logStatus ?? ""}`); + lines.push(`- Log category: ${entry.logCategory ?? ""}`); + + lines.push("- Resolved endpoints:"); + if (entry.resolvedEndpoints.length === 0) { + lines.push(" - "); + } else { + for (const endpoint of entry.resolvedEndpoints) { + lines.push( + ` - ${endpoint.groupMethod}: ${endpoint.descriptorHttpMethod ?? ""} ${endpoint.descriptorPath ?? ""} | descriptor=${endpoint.descriptorName ?? ""} | openapi=${endpoint.openapiStatus} | deprecated=${endpoint.openapiDeprecated ?? ""}`, + ); + } + } + + lines.push(""); + } + + return `${lines.join("\n")}\n`; +} + +await main();