From 019fb1a6f07d6af112809250b6c346b8ce547b4f Mon Sep 17 00:00:00 2001 From: Antony Rizzitelli Date: Thu, 19 Nov 2026 08:35:29 +0000 Subject: [PATCH 01/16] test: run the capture journeys as integration tests against the playground --- .gitignore | 5 + e2e/journeys/capture.mjs | 3 +- e2e/journeys/capture/captured.mjs | 78 ------ package.json | 7 +- pnpm-lock.yaml | 67 ++++-- tests/capture.journeys.test.ts | 140 +++++++++++ tests/helpers/backend.ts | 207 ++++++++++++++++ tests/helpers/dms.ts | 379 ++++++++++++++++++++++++++++++ tests/helpers/journeys-setup.ts | 34 +++ tests/helpers/json.ts | 38 +++ tests/helpers/seed.ts | 144 ++++++++++++ tsconfig.tools.json | 2 +- vitest.config.ts | 4 +- vitest.journeys.config.ts | 20 ++ 14 files changed, 1024 insertions(+), 104 deletions(-) delete mode 100644 e2e/journeys/capture/captured.mjs create mode 100644 tests/capture.journeys.test.ts create mode 100644 tests/helpers/backend.ts create mode 100644 tests/helpers/dms.ts create mode 100644 tests/helpers/journeys-setup.ts create mode 100644 tests/helpers/json.ts create mode 100644 tests/helpers/seed.ts create mode 100644 vitest.journeys.config.ts diff --git a/.gitignore b/.gitignore index 3631d97..edf6930 100644 --- a/.gitignore +++ b/.gitignore @@ -380,3 +380,8 @@ e2e/playwright-report e2e/test-results e2e/blob-report e2e/blob-reports + +# What a backend of the verification playground writes when it starts +.agents/skills/verify-dms-api/playground/antelope-lock +.agents/skills/verify-dms-api/playground/.codex +.agents/skills/verify-dms-api/playground/.mcp.json diff --git a/e2e/journeys/capture.mjs b/e2e/journeys/capture.mjs index 30d1f0e..5b9a94c 100644 --- a/e2e/journeys/capture.mjs +++ b/e2e/journeys/capture.mjs @@ -1,5 +1,4 @@ import { guarded } from "../guard.mjs"; -import { captured } from "./capture/captured.mjs"; import { redaction } from "./capture/redaction.mjs"; -export const journeys = guarded([...captured, ...redaction]); +export const journeys = guarded(redaction); diff --git a/e2e/journeys/capture/captured.mjs b/e2e/journeys/capture/captured.mjs deleted file mode 100644 index e1bafba..0000000 --- a/e2e/journeys/capture/captured.mjs +++ /dev/null @@ -1,78 +0,0 @@ -import { check, list, mon, num, pick, same, saveSettings, sendAndRead, settled, text, traffic } from "../../api.mjs"; -import { JSON_HEADERS } from "./shared.mjs"; - -/** @typedef {import("../../api.mjs").Ctx} Ctx */ - -/** The capture journeys about captured. */ -export const captured = [ - { - id: "a-request-is-captured-with-its-facts", - tag: "@smoke", - title: "a call to a project route is stored with its method, route pattern, raw path, path parameters, query, status, duration, client, request id, headers and bodies", - entryPoint: "GET /api/monitoring/logs/:id after a call to /fx/*", - async run(/** @type {Ctx} */ ctx) { - const put = await sendAndRead(ctx, "PUT an item", { method: "PUT", path: "/fx/items/42?color=red&size=l", headers: JSON_HEADERS, body: JSON.stringify({ name: "chair" }) }); - const log = put.log; - await ctx.verify("the stored request", [ - same("answered 200", put.status, 200), - same("method", text(log, "method"), "PUT"), - same("the route pattern", text(log, "uri"), "/fx/items/:id"), - same("the raw path", text(log, "rawPath"), "/fx/items/42"), - same("the path parameters", pick(log, "pathParams"), { id: "42" }), - same("the query", pick(log, "query"), { color: "red", size: "l" }), - same("the status", num(log, "statusCode"), 200), - check("a duration was measured", (num(log, "responseTimeMs") ?? -1) >= 0, num(log, "responseTimeMs")), - same("the client's user agent", text(log, "userAgent"), "verify-agent/1.0"), - check("the client address is the loopback", /127\.0\.0\.1/.test(text(log, "ip")), text(log, "ip")), - same("the request id comes from x-request-id", text(log, "requestId"), "req-capture-1"), - same("the request body", text(log, "requestBody"), '{"name":"chair"}'), - same("the response body", text(log, "responseBody"), '{"id":"42","body":{"name":"chair"}}'), - same("the route is registered", pick(log, "routeRegistered"), true), - same("the route reference", text(log, "route"), "PUT /fx/items/:id"), - ]); - const missing = await sendAndRead(ctx, "an unknown item", { path: "/fx/items/x9" }); - await ctx.verify("a client error", [same("status 404", num(missing.log, "statusCode"), 404), same("its message is the error shown", text(missing.log, "error", "message") || pick(missing.log, "error"), "no such item")]); - }, - }, - { - id: "a-thrown-error-keeps-its-message-and-its-stack", - title: "a handler that throws is stored as a 500 with the error's message and stack, a thrown HTTP result with its message, and a clean call with no error", - entryPoint: "GET /api/monitoring/logs/:id for /fx/boom, /fx/missing and /fx/ok", - async run(/** @type {Ctx} */ ctx) { - const boom = await sendAndRead(ctx, "boom", { path: "/fx/boom" }); - const missing = await sendAndRead(ctx, "missing", { path: "/fx/missing" }); - const ok = await sendAndRead(ctx, "ok", { path: "/fx/ok" }); - await ctx.verify("errors", [ - same("boom is a 500", num(boom.log, "statusCode"), 500), - same("boom's message", text(boom.log, "error", "message"), "fixture boom: the invoice has no customer"), - check("boom's stack names the handler", /fixture boom/.test(text(boom.log, "error", "stack")) && /boom/.test(text(boom.log, "error", "stack")), text(boom.log, "error", "stack").slice(0, 120)), - same("missing is a 404", num(missing.log, "statusCode"), 404), - same("missing's message is its body message", text(missing.log, "error", "message"), "fixture not found"), - same("ok has no error", pick(ok.log, "error"), null), - ]); - }, - }, - { - id: "the-consoles-own-and-the-dms-traffic-is-never-captured", - title: "even under All routes, calls to /api/monitoring, /dms, the realtime channel and page layouts are neither logged nor counted", - entryPoint: "GET /api/monitoring/logs under the All routes scope", - async run(/** @type {Ctx} */ ctx) { - await saveSettings(ctx, "watch all routes", { scope: "all" }); - await traffic(ctx, "one project call", [{ path: "/fx/ok", count: 1 }]); - for (const path of ["/overview/health", "/logs", "/routes/list", "/settings/summary"]) await mon(ctx, `the console reads ${path}`, path); - await ctx.api("a DMS call", { method: "GET", path: "/dms/pages/modules/api/overview/pagelayout" }); - await ctx.api("the onboarding probe", { method: "GET", path: "/api/onboarding/informations", as: "anon" }); - await new Promise((resolve) => setTimeout(resolve, 1500)); - const logs = await settled(ctx, "the logs", "/logs?limit=200", (body) => list(body, "results").some((row) => text(row, "path") === "/api/onboarding/informations")); - const paths = list(logs.body, "results").map((row) => text(row, "path")); - await ctx.verify("what is stored", [ - check("the project call is stored", paths.includes("/fx/ok"), paths), - check("a module route (the onboarding probe) is stored under All routes", paths.includes("/api/onboarding/informations"), paths), - same("no /api/monitoring call is stored", paths.filter((path) => path.startsWith("/api/monitoring")), []), - same("no /dms call is stored", paths.filter((path) => path.startsWith("/dms")), []), - same("no page layout is stored", paths.filter((path) => path.endsWith("/pagelayout")), []), - same("no realtime call is stored", paths.filter((path) => path.startsWith("/api/realtime")), []), - ]); - }, - }, -]; diff --git a/package.json b/package.json index 5da39ef..d886f18 100644 --- a/package.json +++ b/package.json @@ -34,10 +34,12 @@ "build": "rimraf dist && tsc", "dev": "ajs project dev -w -p playground", "e2e": "playwright test -c e2e", + "e2e:build": "pnpm run build && pnpm --dir playground install --ignore-workspace --frozen-lockfile --prefer-offline --ignore-scripts && ln -sfn ../../../../playground/node_modules .agents/skills/verify-dms-api/playground/node_modules && tsc -p .agents/skills/verify-dms-api/playground/tsconfig.build.json", "lint:fix": "oxlint --fix --max-warnings=1000000 && oxfmt . && pnpm --dir frontend-vue lint:fix", "lint": "oxlint && pnpm --dir frontend-vue lint", "typecheck": "tsc -p tests", "test": "vitest run && pnpm --dir frontend-vue test", + "test:journeys": "vitest run -c vitest.journeys.config.ts", "test:frontend": "ajs dms verify-source --layer node_modules/@antelopejs/dms/frontend-vue --module frontend-vue", "format": "oxfmt . && pnpm --dir frontend-vue format", "frontend:dev": "cd playground && pnpm frontend:dev", @@ -64,8 +66,8 @@ "zod": "^3.24.2" }, "devDependencies": { - "@antelopejs/core": "~1.24.0", - "@antelopejs/dms": ">=0.20.8 <0.21.0", + "@antelopejs/core": "~1.24.5", + "@antelopejs/dms": "0.21.4-next.939.g641c7fa", "@antelopejs/dms-frontend": ">=0.11.4 <0.12.0", "@antelopejs/interface-data-api": ">=0.8.0 <0.9.0", "@antelopejs/tooling-configs": ">=0.8.9 <0.9.0", @@ -74,6 +76,7 @@ "@types/semver": "^7.7.0", "knip": "^6.34.0", "lefthook": "^2.2.2", + "mongodb": "~7.5.0", "mongodb-memory-server-core": "^11.0.1", "oxfmt": "^0.66.0", "oxlint": "1.81.0", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 090d4f8..6963dfd 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -34,14 +34,14 @@ importers: version: 3.25.76 devDependencies: '@antelopejs/core': - specifier: ~1.24.0 - version: 1.24.0 + specifier: ~1.24.5 + version: 1.24.5 '@antelopejs/dms': - specifier: '>=0.20.8 <0.21.0' - version: 0.20.8(@antelopejs/core@1.24.0)(@playwright/test@1.65.1)(typescript@6.0.3) + specifier: 0.21.4-next.939.g641c7fa + version: 0.21.4-next.939.g641c7fa(@antelopejs/core@1.24.5)(@playwright/test@1.65.1)(typescript@6.0.3) '@antelopejs/dms-frontend': specifier: '>=0.11.4 <0.12.0' - version: 0.11.6(@antelopejs/core@1.24.0) + version: 0.11.6(@antelopejs/core@1.24.5) '@antelopejs/interface-data-api': specifier: '>=0.8.0 <0.9.0' version: 0.8.0(@antelopejs/interface-api-util@0.3.0(@antelopejs/interface-api@0.3.3(@antelopejs/interface-core@0.5.1))(@antelopejs/interface-core@0.5.1))(@antelopejs/interface-api@0.3.3(@antelopejs/interface-core@0.5.1))(@antelopejs/interface-core@0.5.1)(@antelopejs/interface-database-decorators@0.5.2(@antelopejs/interface-api@0.3.3(@antelopejs/interface-core@0.5.1))(@antelopejs/interface-core@0.5.1)(@antelopejs/interface-database@0.4.1(@antelopejs/interface-core@0.5.1)))(@antelopejs/interface-database@0.4.1(@antelopejs/interface-core@0.5.1)) @@ -63,6 +63,9 @@ importers: lefthook: specifier: ^2.2.2 version: 2.2.2 + mongodb: + specifier: ~7.5.0 + version: 7.5.0(socks@2.8.10) mongodb-memory-server-core: specifier: ^11.0.1 version: 11.3.0(socks@2.8.10) @@ -103,8 +106,8 @@ packages: 2fa@0.1.2: resolution: {integrity: sha512-PbQttc6USc2TmZwOPE/zlNWTJTySLt50rWhUbZgPkZ5LAOLpgShT3yclbxwHPfHGUXEjfLrTo3Lunyx8onGnXw==} - '@antelopejs/core@1.24.0': - resolution: {integrity: sha512-lYw0iwu8pMohkWvU+aRTHRsZvRi2YXph+ALF3lwCyjilxeXlMxoCDwHroGP3iWXrAhqK8G/8iTloOz/b0WGUKw==} + '@antelopejs/core@1.24.5': + resolution: {integrity: sha512-05vcPpagrOi5VXhecjsR5TFQrDeEJrqZ9Vfrmjp/iKlydYIvvbuyxCz9E51eGgWCa3UnZLgvWqYNxlxvYt5MlA==} engines: {node: ^20.19.0 || >=22.12.0} hasBin: true @@ -115,12 +118,12 @@ packages: peerDependencies: '@antelopejs/core': ~1.24.0 - '@antelopejs/dms@0.20.8': - resolution: {integrity: sha512-3+GBORC23xOkPpDbl5eEVugj2tduJ85ftNI1EJxEsviE7NVdIRq4Hwd4cNQ1M43HmhpC5ogQHjQ80xDE4Yvh5Q==} + '@antelopejs/dms@0.21.4-next.939.g641c7fa': + resolution: {integrity: sha512-wAF+SPGjEl4X8JM23HFXBLtTRjBQqx5OoWg3Cxm2Pm260UDdeLukgbZw0dw5ifNZoMv7wjV/uyTiqZ+WSeK7Rw==} engines: {node: ^20.19.0 || >=22.12.0} hasBin: true peerDependencies: - '@antelopejs/core': ~1.24.0 + '@antelopejs/core': ~1.24.5 '@playwright/test': ^1.65.0 typescript: '>=5.8.0 <7.0.0' peerDependenciesMeta: @@ -176,8 +179,8 @@ packages: peerDependencies: '@antelopejs/interface-core': '>=0.5.1 <0.6.0' - '@antelopejs/interface-dms-builder@0.5.2': - resolution: {integrity: sha512-xtP5ZdkjTlvYxiRcLfDnHqtep9K369Nqgdf7ym0Yf8j93LF+XcRP35Z1Egs5abXrQ6F7zchey+1kOEIo2g8JNg==} + '@antelopejs/interface-dms-builder@0.6.2-next.936.g9ee0eb3': + resolution: {integrity: sha512-SfKa5UizRJzhQ3ZNgl2ur+El+F8aWbk4W4JdrMm2qOOQWsHhAvo+DdHYie1K56u9LrD2JfaOmBH+vTabJ7Yf4Q==} peerDependencies: '@antelopejs/interface-core': '>=0.5.1 <0.6.0' @@ -192,6 +195,17 @@ packages: '@antelopejs/interface-database-decorators': '>=0.5.2 <0.6.0' '@antelopejs/interface-file-storage': '>=0.1.4 <0.2.0' + '@antelopejs/interface-dms@0.14.7-next.936.g9ee0eb3': + resolution: {integrity: sha512-pJvCfrXtMnFFlIbj4rnfsOH53RgHeUD9pSv1W/aWZ89N6gQaEsH4VB0/+sVjEnbGI3SEqNhu3OXi2M65gOA3jg==} + peerDependencies: + '@antelopejs/interface-api': '>=0.3.0 <0.4.0' + '@antelopejs/interface-api-util': '>=0.3.0 <0.4.0' + '@antelopejs/interface-core': '>=0.5.1 <0.6.0' + '@antelopejs/interface-data-api': '>=0.8.0 <0.9.0' + '@antelopejs/interface-database': '>=0.4.0 <0.5.0' + '@antelopejs/interface-database-decorators': '>=0.5.2 <0.6.0' + '@antelopejs/interface-file-storage': '>=0.1.4 <0.2.0' + '@antelopejs/interface-email@0.2.1': resolution: {integrity: sha512-qVCJE7dBX90HYSvXXnD4VIEgUn7ZL3FMt8q/BDuNscUcl5pczMb6o480Hyecdsd8c3T1/28n9+1JhM026d8XEQ==} peerDependencies: @@ -3544,7 +3558,7 @@ snapshots: qr-image: 3.2.0 thirty-two: 0.0.2 - '@antelopejs/core@1.24.0': + '@antelopejs/core@1.24.5': dependencies: '@antelopejs/interface-core': 0.5.1 '@clack/prompts': 1.8.1 @@ -3559,9 +3573,9 @@ snapshots: semver: 7.8.5 typescript: 5.9.3 - '@antelopejs/dms-frontend@0.11.6(@antelopejs/core@1.24.0)': + '@antelopejs/dms-frontend@0.11.6(@antelopejs/core@1.24.5)': dependencies: - '@antelopejs/core': 1.24.0 + '@antelopejs/core': 1.24.5 '@intlify/message-compiler': 11.4.20 chokidar: 4.0.3 commander: 13.1.0 @@ -3575,7 +3589,7 @@ snapshots: '@oxc-parser/binding-linux-x64-gnu': 0.95.0 '@oxc-parser/binding-win32-x64-msvc': 0.95.0 - '@antelopejs/dms@0.20.8(@antelopejs/core@1.24.0)(@playwright/test@1.65.1)(typescript@6.0.3)': + '@antelopejs/dms@0.21.4-next.939.g641c7fa(@antelopejs/core@1.24.5)(@playwright/test@1.65.1)(typescript@6.0.3)': dependencies: 2fa: 0.1.2 '@antelopejs/interface-api': 0.3.3(@antelopejs/interface-core@0.5.1) @@ -3584,8 +3598,8 @@ snapshots: '@antelopejs/interface-data-api': 0.8.0(@antelopejs/interface-api-util@0.3.0(@antelopejs/interface-api@0.3.3(@antelopejs/interface-core@0.5.1))(@antelopejs/interface-core@0.5.1))(@antelopejs/interface-api@0.3.3(@antelopejs/interface-core@0.5.1))(@antelopejs/interface-core@0.5.1)(@antelopejs/interface-database-decorators@0.5.2(@antelopejs/interface-api@0.3.3(@antelopejs/interface-core@0.5.1))(@antelopejs/interface-core@0.5.1)(@antelopejs/interface-database@0.4.1(@antelopejs/interface-core@0.5.1)))(@antelopejs/interface-database@0.4.1(@antelopejs/interface-core@0.5.1)) '@antelopejs/interface-database': 0.4.1(@antelopejs/interface-core@0.5.1) '@antelopejs/interface-database-decorators': 0.5.2(@antelopejs/interface-api@0.3.3(@antelopejs/interface-core@0.5.1))(@antelopejs/interface-core@0.5.1)(@antelopejs/interface-database@0.4.1(@antelopejs/interface-core@0.5.1)) - '@antelopejs/interface-dms': 0.14.3(9d972b62362fb46a077a8d98b79bc383) - '@antelopejs/interface-dms-builder': 0.5.2(@antelopejs/interface-core@0.5.1) + '@antelopejs/interface-dms': 0.14.7-next.936.g9ee0eb3(9d972b62362fb46a077a8d98b79bc383) + '@antelopejs/interface-dms-builder': 0.6.2-next.936.g9ee0eb3(@antelopejs/interface-core@0.5.1) '@antelopejs/interface-email': 0.2.1(@antelopejs/interface-core@0.5.1) '@antelopejs/interface-file-storage': 0.1.4(@antelopejs/interface-core@0.5.1) '@csstools/css-tokenizer': 4.0.0 @@ -3609,7 +3623,7 @@ snapshots: zod: 3.25.76 zod-to-json-schema: 3.24.6(zod@3.25.76) optionalDependencies: - '@antelopejs/core': 1.24.0 + '@antelopejs/core': 1.24.5 '@antelopejs/interface-database-catalog': 0.1.0(@antelopejs/interface-core@0.5.1) '@antelopejs/interface-dms-automation': 0.7.0(@antelopejs/interface-core@0.5.1) '@antelopejs/interface-redis': 0.0.9(@antelopejs/interface-core@0.5.1) @@ -3667,7 +3681,7 @@ snapshots: '@antelopejs/interface-core': 0.5.1 optional: true - '@antelopejs/interface-dms-builder@0.5.2(@antelopejs/interface-core@0.5.1)': + '@antelopejs/interface-dms-builder@0.6.2-next.936.g9ee0eb3(@antelopejs/interface-core@0.5.1)': dependencies: '@antelopejs/interface-core': 0.5.1 @@ -3684,6 +3698,19 @@ snapshots: zod: 3.25.76 zod-to-json-schema: 3.24.6(zod@3.25.76) + '@antelopejs/interface-dms@0.14.7-next.936.g9ee0eb3(9d972b62362fb46a077a8d98b79bc383)': + dependencies: + '@antelopejs/interface-api': 0.3.3(@antelopejs/interface-core@0.5.1) + '@antelopejs/interface-api-util': 0.3.0(@antelopejs/interface-api@0.3.3(@antelopejs/interface-core@0.5.1))(@antelopejs/interface-core@0.5.1) + '@antelopejs/interface-core': 0.5.1 + '@antelopejs/interface-data-api': 0.8.0(@antelopejs/interface-api-util@0.3.0(@antelopejs/interface-api@0.3.3(@antelopejs/interface-core@0.5.1))(@antelopejs/interface-core@0.5.1))(@antelopejs/interface-api@0.3.3(@antelopejs/interface-core@0.5.1))(@antelopejs/interface-core@0.5.1)(@antelopejs/interface-database-decorators@0.5.2(@antelopejs/interface-api@0.3.3(@antelopejs/interface-core@0.5.1))(@antelopejs/interface-core@0.5.1)(@antelopejs/interface-database@0.4.1(@antelopejs/interface-core@0.5.1)))(@antelopejs/interface-database@0.4.1(@antelopejs/interface-core@0.5.1)) + '@antelopejs/interface-database': 0.4.1(@antelopejs/interface-core@0.5.1) + '@antelopejs/interface-database-decorators': 0.5.2(@antelopejs/interface-api@0.3.3(@antelopejs/interface-core@0.5.1))(@antelopejs/interface-core@0.5.1)(@antelopejs/interface-database@0.4.1(@antelopejs/interface-core@0.5.1)) + '@antelopejs/interface-file-storage': 0.1.4(@antelopejs/interface-core@0.5.1) + randomstring: 1.3.2 + zod: 3.25.76 + zod-to-json-schema: 3.24.6(zod@3.25.76) + '@antelopejs/interface-email@0.2.1(@antelopejs/interface-core@0.5.1)': dependencies: '@antelopejs/interface-core': 0.5.1 diff --git a/tests/capture.journeys.test.ts b/tests/capture.journeys.test.ts new file mode 100644 index 0000000..4ea66bd --- /dev/null +++ b/tests/capture.journeys.test.ts @@ -0,0 +1,140 @@ +import { describe, expect } from "vitest"; +import { z } from "zod"; +import { test } from "./helpers/dms"; +import { resultsOf } from "./helpers/json"; + +const JSON_HEADERS = { + "content-type": "application/json", + "user-agent": "verify-agent/1.0", + "x-request-id": "req-capture-1", + "x-webhook-signature": "sig-1", + authorization: "Bearer top-secret", + cookie: "sid=1", + "proxy-authorization": "Basic cHJveHk6cHc=", + "set-cookie": "sid=2", + "x-api-key": "k-1", + origin: "https://client.example.org", + referer: "https://client.example.org/page", +}; + +const pathSchema = z.object({ path: z.string() }); + +describe("capture", () => { + test("a-request-is-captured-with-its-facts", async ({ dms }) => { + const put = await dms.sendAndRead({ + method: "PUT", + path: "/fx/items/42?color=red&size=l", + headers: JSON_HEADERS, + body: JSON.stringify({ name: "chair" }), + }); + expect.soft(put.status, "answered 200").toBe(200); + expect.soft(put.log, "the stored request").toMatchObject({ + method: "PUT", + uri: "/fx/items/:id", + rawPath: "/fx/items/42", + pathParams: { id: "42" }, + query: { color: "red", size: "l" }, + statusCode: 200, + userAgent: "verify-agent/1.0", + requestId: "req-capture-1", + requestBody: '{"name":"chair"}', + responseBody: '{"id":"42","body":{"name":"chair"}}', + routeRegistered: true, + route: "PUT /fx/items/:id", + }); + const measured = z + .object({ responseTimeMs: z.number(), ip: z.string() }) + .parse(put.log); + expect + .soft(measured.responseTimeMs, "a duration was measured") + .toBeGreaterThanOrEqual(0); + expect + .soft(measured.ip, "the client address is the loopback") + .toMatch(/127\.0\.0\.1/); + const missing = await dms.sendAndRead({ path: "/fx/items/x9" }); + expect + .soft(missing.log, "a client error keeps its message") + .toMatchObject({ statusCode: 404, error: { message: "no such item" } }); + }); + + test("a-thrown-error-keeps-its-message-and-its-stack", async ({ dms }) => { + const boom = await dms.sendAndRead({ path: "/fx/boom" }); + const missing = await dms.sendAndRead({ path: "/fx/missing" }); + const ok = await dms.sendAndRead({ path: "/fx/ok" }); + expect + .soft(boom.log, "a thrown error is a 500 with its message") + .toMatchObject({ + statusCode: 500, + error: { message: "fixture boom: the invoice has no customer" }, + }); + const stack = z + .object({ error: z.object({ stack: z.string() }) }) + .parse(boom.log).error.stack; + expect + .soft(stack, "its stack names the handler") + .toMatch(/fixture boom[\s\S]*boom/); + expect + .soft(missing.log, "a thrown HTTP result keeps its body message") + .toMatchObject({ + statusCode: 404, + error: { message: "fixture not found" }, + }); + expect + .soft(ok.log, "a clean call has no error") + .toMatchObject({ error: null }); + }); + + test("the-consoles-own-and-the-dms-traffic-is-never-captured", async ({ + dms, + }) => { + await dms.saveSettings({ scope: "all" }); + await dms.traffic([{ path: "/fx/ok", count: 1 }]); + for (const path of [ + "/overview/health", + "/logs", + "/routes/list", + "/settings/summary", + ]) + await dms.mon(path); + await dms.api("GET", "/dms/pages/modules/api/overview/pagelayout"); + await dms.api("GET", "/api/onboarding/informations", { as: "anon" }); + await new Promise((done) => setTimeout(done, 1500)); + const logs = await dms.settled("/logs?limit=200", (body) => + resultsOf(body, pathSchema).some( + (row) => row.path === "/api/onboarding/informations", + ), + ); + const paths = resultsOf(logs.body, pathSchema).map((row) => row.path); + expect.soft(paths, "the project call is stored").toContain("/fx/ok"); + expect + .soft( + paths, + "a module route (the onboarding probe) is stored under All routes", + ) + .toContain("/api/onboarding/informations"); + expect + .soft( + paths.filter((path) => path.startsWith("/api/monitoring")), + "no /api/monitoring call is stored", + ) + .toEqual([]); + expect + .soft( + paths.filter((path) => path.startsWith("/dms")), + "no /dms call is stored", + ) + .toEqual([]); + expect + .soft( + paths.filter((path) => path.endsWith("/pagelayout")), + "no page layout is stored", + ) + .toEqual([]); + expect + .soft( + paths.filter((path) => path.startsWith("/api/realtime")), + "no realtime call is stored", + ) + .toEqual([]); + }); +}); diff --git a/tests/helpers/backend.ts b/tests/helpers/backend.ts new file mode 100644 index 0000000..d004b09 --- /dev/null +++ b/tests/helpers/backend.ts @@ -0,0 +1,207 @@ +import { type ChildProcess, spawn } from "node:child_process"; +import { randomBytes } from "node:crypto"; +import { + closeSync, + mkdirSync, + openSync, + readFileSync, + statSync, +} from "node:fs"; +import { createRequire } from "node:module"; +import { createServer } from "node:net"; +import { tmpdir } from "node:os"; +import { join, resolve } from "node:path"; +import { z } from "zod"; + +/** The verification playground: the fixture app, dms-api from this checkout, the DMS, MongoDB and auth modules. */ +export const PLAYGROUND = resolve( + __dirname, + "../../.agents/skills/verify-dms-api/playground", +); + +const READY_TIMEOUT_MS = 90_000; +const POLL_MS = 250; +const STOP_TIMEOUT_MS = 10_000; +const DROPPED_PREFIXES = [ + "ANTELOPEJS_", + "npm_", + "PNPM_", + "DMS_", + "MONGO_", + "VERIFY_API_", + "NODE_OPTIONS", +]; + +/** Fresh secrets of a backend: the DMS and auth signing keys and the setup token the first registration presents. */ +function newSecrets() { + const secret = () => randomBytes(18).toString("base64url"); + return { + DMS_JWT_SECRET: secret(), + AUTH_JWT_SECRET: secret(), + DMS_BOOTSTRAP_SECRET: secret(), + DMS_HTML_RENDER_SECRET: secret(), + DMS_SESSION_SECRET: secret(), + DMS_SETUP_TOKEN: secret(), + }; +} + +/** The `ajs` command line the playground installs with `@antelopejs/core`. */ +function ajsCli(): string { + const require = createRequire(join(PLAYGROUND, "package.json")); + return join( + resolve(require.resolve("@antelopejs/core/package.json"), ".."), + "dist/core/cli/index.js", + ); +} + +/** A loopback port no process listens on. */ +function freePort(): Promise { + return new Promise((done, fail) => { + const server = createServer(); + server.once("error", fail); + server.listen(0, "127.0.0.1", () => { + const { port } = z.object({ port: z.number() }).parse(server.address()); + server.close(() => done(port)); + }); + }); +} + +function inheritedEnv(): NodeJS.ProcessEnv { + return Object.fromEntries( + Object.entries(process.env).filter( + ([key]) => !DROPPED_PREFIXES.some((prefix) => key.startsWith(prefix)), + ), + ); +} + +/** Thrown when a backend exits or stays unready: its log since that start says why. */ +export class BackendStartError extends Error { + constructor( + reason: string, + readonly log: string, + ) { + super(`${reason}\n${log.slice(-4000)}`); + } +} + +/** + * One backend of the playground on its own port and its own database of a + * shared MongoDB, run with `ajs project dev`. `extraEnv` is added to the + * environment of every later start, so a test restarts it under other + * module options (`VERIFY_API_CONFIG`) or with the automation stand-in + * (`VERIFY_API_AUTOMATION=1`). + */ +export class Backend { + readonly secrets = newSecrets(); + readonly logFile: string; + extraEnv: Record = {}; + private child: ChildProcess | null = null; + private port = 0; + + constructor( + readonly mongoUrl: string, + readonly database: string, + ) { + const logs = join(tmpdir(), "dms-api-journeys"); + mkdirSync(logs, { recursive: true }); + this.logFile = join(logs, `${database}.log`); + } + + get apiUrl(): string { + return `http://127.0.0.1:${this.port}`; + } + + /** + * Starts the backend and waits until the DMS answers on its port. + * + * @throws {BackendStartError} When it exits first or stays unready for 90 seconds + */ + async start(): Promise { + this.port = await freePort(); + const from = this.logSize(); + const log = openSync(this.logFile, "a"); + const child = spawn( + process.execPath, + [ajsCli(), "project", "dev", "-e", "docs"], + { + cwd: PLAYGROUND, + stdio: ["ignore", log, log], + env: { + ...inheritedEnv(), + CI: "1", + NO_COLOR: "1", + AJS_NO_UPDATE_CHECK: "1", + NO_UPDATE_NOTIFIER: "1", + PORT: String(this.port), + MONGO_URL: this.mongoUrl, + MONGO_DATABASE: this.database, + ...this.secrets, + ...this.extraEnv, + }, + }, + ); + closeSync(log); + this.child = child; + const exited = new Promise((done) => + child.once("exit", () => done()), + ); + let gone = false; + void exited.then(() => (gone = true)); + const deadline = Date.now() + READY_TIMEOUT_MS; + while (Date.now() < deadline && !gone) { + const status = await fetch(`${this.apiUrl}/api/onboarding/informations`, { + signal: AbortSignal.timeout(2000), + }).then( + (response) => response.status, + () => 0, + ); + if (status === 200) return; + await new Promise((done) => setTimeout(done, POLL_MS)); + } + await this.stop(); + throw new BackendStartError( + gone + ? "the backend exited before it answered" + : "the backend did not answer in 90 s", + this.logSince(from), + ); + } + + /** Stops the backend, gracefully then by force. */ + async stop(): Promise { + const child = this.child; + this.child = null; + if (!child || child.exitCode !== null || child.signalCode !== null) return; + const exited = new Promise((done) => + child.once("exit", () => done()), + ); + child.kill("SIGTERM"); + const timer = setTimeout(() => child.kill("SIGKILL"), STOP_TIMEOUT_MS); + await exited; + clearTimeout(timer); + } + + /** Stops the backend and starts it again with the current `extraEnv`. */ + async restart(): Promise { + await this.stop(); + await this.start(); + } + + /** The size of the log so far, to read what a later start writes with {@link Backend.logSince}. */ + logSize(): number { + try { + return statSync(this.logFile).size; + } catch { + return 0; + } + } + + /** What the backend logged since the log held `from` bytes. */ + logSince(from = 0): string { + try { + return readFileSync(this.logFile).subarray(from).toString("utf8"); + } catch { + return ""; + } + } +} diff --git a/tests/helpers/dms.ts b/tests/helpers/dms.ts new file mode 100644 index 0000000..04eac7e --- /dev/null +++ b/tests/helpers/dms.ts @@ -0,0 +1,379 @@ +import { randomBytes, randomUUID } from "node:crypto"; +import { inject, test as base } from "vitest"; +import { z } from "zod"; +import { Backend } from "./backend"; +import { type Json, parseBody, resultsOf } from "./json"; + +declare module "vitest" { + export interface ProvidedContext { + mongoUrl: string; + } +} + +/** The console's HTTP API, which the dashboard calls. */ +export const MONITOR = "/api/monitoring"; + +/** Routes the verification playground registers in the project's scope: 17 under /fx, the member route and the five node routes. */ +export const FIXTURE_ROUTES = 23; + +const SETTLE_TIMEOUT_MS = 15_000; +const SETTLE_POLL_MS = 300; +const REQUEST_TIMEOUT_MS = 30_000; + +/** An answer of the API: its status and its JSON body, or its text when it is not JSON. */ +export interface Answer { + status: number; + body: Json; +} + +/** An answer read as sent, redirects not followed. */ +export interface Raw { + status: number; + headers: Headers; + bytes: Buffer; + body: Json; +} + +/** Settings, module options or an action's input: JSON fields by name. */ +export type Fields = { [key: string]: Json }; + +interface Account { + email: string; + password: string; +} + +/** A call to the API: `as` names an account, `admin` by default, and `anon` sends no session. */ +export interface Call { + method?: string; + body?: Json; + as?: string; +} + +const tokenSchema = z.object({ access_token: z.string() }); +const invitationSchema = z.object({ email: z.string(), token: z.string() }); +const loggedSchema = z.object({ + _id: z.string(), + method: z.string(), + path: z.string(), + requestId: z.string().nullish(), +}); + +/** A password the DMS policy accepts: an uppercase letter, a digit and one of `@$!%*?&`. */ +function newPassword(): string { + return `Vr1!${randomBytes(12).toString("base64url").replace(/[-_]/g, "x")}`; +} + +/** + * A DMS of its own for one test: a backend of the playground on a fresh + * database, its administrator `admin` onboarded, and the accounts the test + * adds. Every call signs its account in again once when its session was + * refused, so a restart keeps the accounts usable. + */ +export class Dms { + private readonly accounts = new Map(); + private readonly tokens = new Map(); + + private constructor(readonly backend: Backend) {} + + /** Starts a backend on a fresh database and onboards its administrator. */ + static async start(mongoUrl: string): Promise { + const dms = new Dms( + new Backend( + mongoUrl, + `journey_${randomUUID().replaceAll("-", "").slice(0, 12)}`, + ), + ); + await dms.backend.start(); + const admin = { + email: `admin-${randomUUID()}@verify.local`, + password: newPassword(), + }; + const register = await dms.api("POST", "/api/onboarding/register", { + body: { + ...admin, + firstName: "Verify", + lastName: "Admin", + setupToken: dms.backend.secrets.DMS_SETUP_TOKEN, + platformName: "Verify", + }, + as: "anon", + }); + if (register.status !== 200) + throw new Error( + `onboarding answered ${register.status}: ${JSON.stringify(register.body)}`, + ); + dms.accounts.set("admin", admin); + return dms; + } + + get apiUrl(): string { + return this.backend.apiUrl; + } + + /** Stops the backend. The database stays in the shared MongoDB until the suite ends. */ + stop(): Promise { + return this.backend.stop(); + } + + /** A call to the backend as an account, its body sent as JSON. */ + async api( + method: string, + path: string, + call: Omit = {}, + ): Promise { + const as = call.as ?? "admin"; + const send = async (): Promise => { + const headers = new Headers({ accept: "application/json" }); + if (call.body !== undefined) + headers.set("content-type", "application/json"); + const token = as === "anon" ? undefined : this.tokens.get(as); + if (token) headers.set("authorization", `Bearer ${token}`); + const response = await fetch(`${this.apiUrl}${path}`, { + method, + headers, + body: call.body === undefined ? undefined : JSON.stringify(call.body), + signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS), + }); + return { + status: response.status, + body: parseBody(await response.text()), + }; + }; + if (as !== "anon" && !this.tokens.has(as)) await this.signIn(as); + const first = await send(); + if (first.status !== 401 || as === "anon") return first; + await this.signIn(as); + return send(); + } + + /** A call to the console's API under {@link MONITOR}, as the administrator unless `as` says otherwise. */ + mon(path: string, call: Call = {}): Promise { + return this.api(call.method ?? "GET", `${MONITOR}${path}`, { + body: call.body, + as: call.as ?? "admin", + }); + } + + /** Signs an account in and keeps its session. */ + async signIn(account: string): Promise { + const credentials = this.accounts.get(account); + if (!credentials) throw new Error(`no account ${account}`); + const answer = await this.api("POST", "/api/auth/login", { + body: { ...credentials }, + as: "anon", + }); + const signedIn = tokenSchema.safeParse(answer.body); + if (answer.status !== 200 || !signedIn.success) + throw new Error(`${account} could not sign in: ${answer.status}`); + this.tokens.set(account, signedIn.data.access_token); + } + + /** The session token of an account, signing it in first. */ + async tokenOf(account: string): Promise { + if (!this.tokens.has(account)) await this.signIn(account); + return this.tokens.get(account) ?? ""; + } + + /** + * Sends the project's own routes (`/fx/*`) batches of calls, one after + * the other. + * + * @returns How many calls answered each status + */ + async traffic( + batches: { + path: string; + count: number; + method?: string; + headers?: HeadersInit; + body?: string; + }[], + ): Promise> { + const statuses = new Map(); + for (const batch of batches) { + for (let index = 0; index < batch.count; index += 1) { + const response = await fetch(`${this.apiUrl}${batch.path}`, { + method: batch.method ?? "GET", + headers: batch.headers, + body: batch.body, + }); + await response.arrayBuffer(); + statuses.set(response.status, (statuses.get(response.status) ?? 0) + 1); + } + } + return statuses; + } + + /** Reads the console until `accept` takes the answer, for 15 seconds at most: the capture writes behind the response. */ + async settled( + path: string, + accept: (body: Json) => boolean, + ): Promise { + const deadline = Date.now() + SETTLE_TIMEOUT_MS; + let answer = await this.mon(path); + while (!accept(answer.body) && Date.now() < deadline) { + await new Promise((done) => setTimeout(done, SETTLE_POLL_MS)); + answer = await this.mon(path); + } + return answer; + } + + /** A request sent as given, any method, its answer kept byte for byte; `as` adds the account's session. */ + async raw(request: { + url?: string; + path?: string; + method?: string; + headers?: HeadersInit; + as?: string; + body?: string; + }): Promise { + const headers = new Headers(request.headers); + if (request.as) + headers.set("authorization", `Bearer ${await this.tokenOf(request.as)}`); + const response = await fetch( + request.url ?? `${this.apiUrl}${request.path ?? "/"}`, + { + method: request.method ?? "GET", + headers, + redirect: "manual", + body: request.body, + }, + ); + const bytes = Buffer.from(await response.arrayBuffer()); + return { + status: response.status, + headers: response.headers, + bytes, + body: parseBody(bytes.toString("utf8")), + }; + } + + /** Saves settings through the settings form's route; `null` returns a field to its module default. */ + saveSettings(values: Fields, as?: string): Promise { + return this.mon("/settings", { method: "POST", body: values, as }); + } + + /** + * Sends one call to the project's routes and reads back the request log + * it made, found by its `x-request-id` or by its method and path. + */ + async sendAndRead(request: { + method?: string; + path: string; + headers?: HeadersInit; + body?: string; + }): Promise<{ status: number; log: Json }> { + const sent = await this.raw(request); + const path = request.path.split("?")[0]; + const method = request.method ?? "GET"; + const requestId = new Headers(request.headers).get("x-request-id"); + const deadline = Date.now() + SETTLE_TIMEOUT_MS; + while (Date.now() < deadline) { + const listed = await this.mon("/logs?limit=50"); + const row = resultsOf(listed.body, loggedSchema).find( + (entry) => + (requestId !== null && entry.requestId === requestId) || + (entry.path === path && entry.method === method), + ); + if (row) + return { + status: sent.status, + log: (await this.mon(`/logs/${row._id}`)).body, + }; + await new Promise((done) => setTimeout(done, SETTLE_POLL_MS)); + } + return { status: sent.status, log: null }; + } + + /** Adds a member whose role holds exactly `permissions`, signed up through the playground's invitation route. */ + async member(name: string, permissions: string[]): Promise { + const invite = invitationSchema.parse( + ( + await this.api("POST", "/verify-api/member", { + body: { name, permissions }, + as: "anon", + }) + ).body, + ); + const email = invite.email; + const password = newPassword(); + const signup = await this.api("POST", "/api/auth/signup", { + body: { name, email, password, lang: "en", token: invite.token }, + as: "anon", + }); + if (signup.status !== 200) + throw new Error( + `${name} could not sign up: ${signup.status} ${JSON.stringify(signup.body)}`, + ); + this.accounts.set(name, { email, password }); + } + + /** Restarts the backend with the current extra environment. */ + restart(): Promise { + return this.backend.restart(); + } + + /** Restarts the backend with extra environment variables, runs `body`, then restarts it with the previous ones. */ + async withEnv( + env: Record, + body: () => Promise, + ): Promise { + const previous = { ...this.backend.extraEnv }; + this.backend.extraEnv = { ...previous, ...env }; + try { + await this.restart(); + await body(); + } finally { + this.backend.extraEnv = previous; + await this.restart(); + } + } + + /** {@link Dms.withEnv} with the module's options (`VERIFY_API_CONFIG`). */ + withConfig(config: Fields, body: () => Promise): Promise { + return this.withEnv({ VERIFY_API_CONFIG: JSON.stringify(config) }, body); + } + + /** + * Restarts the backend with the module's options and tells whether it + * came back, with what it logged. The extra environment is put back, not + * the backend: a test restarts it before its next call. + */ + async startWith(config: Fields): Promise<{ started: boolean; log: string }> { + const previous = { ...this.backend.extraEnv }; + const from = this.backend.logSize(); + this.backend.extraEnv = { + ...previous, + VERIFY_API_CONFIG: JSON.stringify(config), + }; + let started = true; + try { + await this.restart(); + } catch { + started = false; + } + this.backend.extraEnv = previous; + return { started, log: this.backend.logSince(from) }; + } + + /** Runs an automation action the module registered with the playground's stand-in provider. */ + runAction(id: string, input: Json): Promise { + return this.api("POST", `/dms/verify-api/nodes/action/${id}`, { + body: input, + as: "anon", + }); + } +} + +/** `test` with `dms`: a DMS of the test's own, stopped once the test ends. */ +export const test = base.extend<{ dms: Dms }>({ + // oxlint-disable-next-line no-empty-pattern -- Vitest reads a fixture's dependencies from its first parameter's pattern. + dms: async ({}, use) => { + const dms = await Dms.start(inject("mongoUrl")); + try { + await use(dms); + } finally { + await dms.stop(); + } + }, +}); diff --git a/tests/helpers/journeys-setup.ts b/tests/helpers/journeys-setup.ts new file mode 100644 index 0000000..b69f1c7 --- /dev/null +++ b/tests/helpers/journeys-setup.ts @@ -0,0 +1,34 @@ +import { existsSync } from "node:fs"; +import { join } from "node:path"; +import { MongoMemoryServer } from "mongodb-memory-server-core"; +import { Backend, PLAYGROUND } from "./backend"; + +/** + * Starts the MongoDB every test's backend shares, each on a database of its + * own, and starts one backend first: the first start of the playground + * installs its package modules, which parallel starts would race on. + * + * @throws When `pnpm e2e:build` has not built the module and the playground + */ +export default async function setup(project: { + provide: (key: "mongoUrl", value: string) => void; +}): Promise<() => Promise> { + for (const built of [ + join(PLAYGROUND, "../../../../dist/index.js"), + join(PLAYGROUND, "dist/index.js"), + ]) { + if (!existsSync(built)) + throw new Error(`${built} is missing: run \`pnpm e2e:build\` first`); + } + const mongo = await MongoMemoryServer.create(); + const warm = new Backend(mongo.getUri(), "journey_warm_up"); + try { + await warm.start(); + } finally { + await warm.stop(); + } + project.provide("mongoUrl", mongo.getUri()); + return async () => { + await mongo.stop(); + }; +} diff --git a/tests/helpers/json.ts b/tests/helpers/json.ts new file mode 100644 index 0000000..3e541f0 --- /dev/null +++ b/tests/helpers/json.ts @@ -0,0 +1,38 @@ +import { z } from "zod"; + +/** A JSON value, as an answer of the API holds it. */ +export type Json = + | string + | number + | boolean + | null + | Json[] + | { [key: string]: Json }; + +const jsonSchema: z.ZodType = z.lazy(() => + z.union([ + z.string(), + z.number(), + z.boolean(), + z.null(), + z.array(jsonSchema), + z.record(jsonSchema), + ]), +); + +/** A response body as JSON, or as its text when it is not JSON. */ +export function parseBody(body: string): Json { + try { + return jsonSchema.parse(JSON.parse(body)); + } catch { + return body; + } +} + +/** The rows of a list answer of the console: `{ results: [...] }`. */ +export function resultsOf( + body: Json, + row: T, +): z.infer[] { + return z.object({ results: z.array(row) }).parse(body).results; +} diff --git a/tests/helpers/seed.ts b/tests/helpers/seed.ts new file mode 100644 index 0000000..4ae5900 --- /dev/null +++ b/tests/helpers/seed.ts @@ -0,0 +1,144 @@ +import { randomUUID } from "node:crypto"; +import { type Db, MongoClient } from "mongodb"; +import { z } from "zod"; +import type { Dms } from "./dms"; +import type { Json } from "./json"; + +const MS_PER_DAY = 86_400_000; + +async function withDatabase( + dms: Dms, + body: (db: Db) => Promise, +): Promise { + const client = new MongoClient(dms.backend.mongoUrl); + await client.connect(); + try { + return await body(client.db(dms.backend.database)); + } finally { + await client.close(); + } +} + +/** The first millisecond of the local day `daysAgo` days back, how the module keys its per-day statistics. */ +export function dayStart(daysAgo: number): number { + const now = new Date(); + return new Date( + now.getFullYear(), + now.getMonth(), + now.getDate() - daysAgo, + ).getTime(); +} + +/** Request logs `minutesAgo` or `daysAgo` old, one second apart, stored the way the capture stores them. */ +export interface LogBatch { + daysAgo?: number; + minutesAgo?: number; + count: number; + method?: string; + uri?: string; + statusCode?: number; + message?: string; + requestBody?: string; + responseBody?: string; + responseTimeMs?: number; +} + +/** Inserts request logs, as the capture stores them, with the user agent `seed`. */ +export async function seedLogs(dms: Dms, batch: LogBatch): Promise { + const age = + batch.minutesAgo === undefined + ? (batch.daysAgo ?? 0) * MS_PER_DAY + : batch.minutesAgo * 60_000; + const rows = Array.from({ length: batch.count }, (_, index) => ({ + _id: randomUUID(), + timestamp: new Date(Date.now() - age + index * 1000), + method: batch.method ?? "GET", + uri: batch.uri ?? "/fx/ok", + rawPath: batch.uri ?? "/fx/ok", + statusCode: batch.statusCode ?? 200, + responseTimeMs: batch.responseTimeMs ?? 5, + error: batch.message ? { message: batch.message } : null, + routeId: null, + routeMethod: null, + source: null, + requestId: null, + ip: "::1", + userAgent: "seed", + pathParams: {}, + query: {}, + requestHeaders: {}, + requestBody: batch.requestBody ?? null, + requestBodyTruncated: false, + requestBodyOmitted: batch.requestBody === undefined ? "empty" : null, + requestBodySize: null, + responseHeaders: {}, + responseBody: batch.responseBody ?? null, + responseBodyTruncated: false, + responseBodyOmitted: batch.responseBody === undefined ? "empty" : null, + responseBodySize: null, + redactions: { query: 0, requestBody: 0, responseBody: 0, headers: [] }, + _instance: null, + })); + await withDatabase(dms, (db) => + db.collection<{ _id: string }>("dms-api__request_log").insertMany(rows), + ); +} + +/** A day of statistics of a route, `daysAgo` days back. */ +export interface StatisticsDay { + method: string; + uri: string; + daysAgo: number; + requests: number; + serverErrors?: number; + clientErrors?: number; + averageMs?: number; +} + +/** Adds a day of statistics to a route. */ +export async function seedStatistics( + dms: Dms, + day: StatisticsDay, +): Promise { + const average = day.averageMs ?? 10; + const entry = { + day: dayStart(day.daysAgo), + requestsCount: day.requests, + averageResponseTime: average, + minResponseTime: average, + maxResponseTime: average, + totalResponseTime: average * day.requests, + clientErrorsCount: day.clientErrors ?? 0, + serverErrorsCount: day.serverErrors ?? 0, + timedRequestsCount: day.requests, + latencyHistogram: [day.requests, ...Array.from({ length: 17 }, () => 0)], + }; + await withDatabase(dms, (db) => + db.collection<{ _id: string }>("dms-api__routes_statistics").updateOne( + { _id: `${day.method}:${day.uri}` }, + { + $push: { statistics: entry }, + $setOnInsert: { + method: day.method, + uri: day.uri, + revision: randomUUID(), + _instance: null, + }, + }, + { upsert: true }, + ), + ); +} + +/** The documents of a collection of the test's database that match `filter`, each parsed with `schema`. */ +export async function readDocuments( + dms: Dms, + collection: string, + schema: T, + filter: { [key: string]: Json } = {}, +): Promise[]> { + const documents = await withDatabase(dms, (db) => + db.collection(collection).find(filter).toArray(), + ); + return z.array(schema).parse(documents); +} diff --git a/tsconfig.tools.json b/tsconfig.tools.json index 6cb34bc..6360dee 100644 --- a/tsconfig.tools.json +++ b/tsconfig.tools.json @@ -10,5 +10,5 @@ "skipLibCheck": true, "types": ["node"] }, - "include": ["vitest.config.ts"] + "include": ["vitest.config.ts", "vitest.journeys.config.ts"] } diff --git a/vitest.config.ts b/vitest.config.ts index 6defbf9..9ea5998 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -2,7 +2,8 @@ import { fileURLToPath } from "node:url"; import { defineConfig } from "vitest/config"; /** - * Back-end unit tests. The front-end keeps its own config under frontend-vue/. + * Back-end unit tests. The front-end keeps its own config under frontend-vue/, + * and the journeys theirs in vitest.journeys.config.ts. */ export default defineConfig({ resolve: { @@ -11,5 +12,6 @@ export default defineConfig({ test: { environment: "node", include: ["tests/**/*.test.ts"], + exclude: ["tests/**/*.journeys.test.ts"], }, }); diff --git a/vitest.journeys.config.ts b/vitest.journeys.config.ts new file mode 100644 index 0000000..d0628bd --- /dev/null +++ b/vitest.journeys.config.ts @@ -0,0 +1,20 @@ +import { fileURLToPath } from "node:url"; +import { defineConfig } from "vitest/config"; + +/** + * The journeys that call the API or run commands, against backends of the + * verification playground on a shared MongoDB. `pnpm e2e:build` builds what + * they start. + */ +export default defineConfig({ + resolve: { + alias: { "@": fileURLToPath(new URL("src", import.meta.url)) }, + }, + test: { + environment: "node", + include: ["tests/**/*.journeys.test.ts"], + globalSetup: ["tests/helpers/journeys-setup.ts"], + testTimeout: 180_000, + hookTimeout: 180_000, + }, +}); From aabf814bece0f04a3de5b1a2d761c3e73253cad3 Mon Sep 17 00:00:00 2001 From: Antony Rizzitelli Date: Thu, 19 Nov 2026 08:35:58 +0000 Subject: [PATCH 02/16] ci: run the journey integration tests and build the playground with e2e:build --- .github/workflows/ci.yml | 34 +++++++++++++++++++++++++++++++--- 1 file changed, 31 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e24b2f1..fd2a2a8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -110,10 +110,36 @@ jobs: - run: pnpm install --frozen-lockfile --ignore-scripts - run: pnpm build + # The journeys that call the API or run commands, as Vitest integration + # tests against backends of the verification playground on one MongoDB. + journeys: + needs: select + if: needs.select.outputs.run == 'true' + runs-on: ubuntu-latest + timeout-minutes: 20 + env: + TZ: UTC + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + - uses: actions/setup-node@v4 + with: + node-version-file: .nvmrc + - run: corepack enable + - run: pnpm install --frozen-lockfile --ignore-scripts + - uses: actions/cache@v4 + with: + path: ~/.cache/mongodb-binaries + key: journeys-${{ runner.os }}-${{ hashFiles('pnpm-lock.yaml') }} + - run: pnpm e2e:build + - run: pnpm test:journeys + # The journeys of e2e/ as a Playwright suite: the @smoke ones on a pull # request or a push, every journey in two shards in the merge queue, the # nightly run, a manual run and a pull request labelled ci:full, nothing on - # a draft. Its global setup prepares and starts one dms-control run. + # a draft. `pnpm e2e:build` builds the module and the playground, then the + # kit's global setup starts one run of them. e2e: needs: select if: needs.select.outputs.e2e != 'none' @@ -125,6 +151,7 @@ jobs: shard: ${{ fromJSON(needs.select.outputs.e2e == 'all' && '[1, 2]' || '[1]') }} env: E2E: ${{ needs.select.outputs.e2e }} + DMS_E2E_PREBUILT: "1" TZ: UTC steps: - uses: actions/checkout@v4 @@ -154,6 +181,7 @@ jobs: fi echo "::warning::Chromium's system packages did not install: installing the browser alone" pnpm exec playwright install --only-shell chromium + - run: pnpm e2e:build - if: env.E2E == 'smoke' run: pnpm e2e --grep @smoke - if: env.E2E == 'all' @@ -214,12 +242,12 @@ jobs: # it, so merge-when-green cannot merge on the draft run. ubuntu: if: always() - needs: [select, ci, build, formal-route-statistics, e2e] + needs: [select, ci, build, formal-route-statistics, journeys, e2e] runs-on: ubuntu-latest timeout-minutes: 5 steps: - if: >- - needs.select.result != 'success' || contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') || needs.ci.result == 'skipped' || (needs.select.outputs.run == 'true' && (needs.build.result == 'skipped' || needs.formal-route-statistics.result == 'skipped')) || (needs.select.outputs.e2e != 'none' && needs.e2e.result == 'skipped') + needs.select.result != 'success' || contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') || needs.ci.result == 'skipped' || (needs.select.outputs.run == 'true' && (needs.build.result == 'skipped' || needs.formal-route-statistics.result == 'skipped' || needs.journeys.result == 'skipped')) || (needs.select.outputs.e2e != 'none' && needs.e2e.result == 'skipped') run: exit 1 - if: needs.select.outputs.tier == 'draft' run: | From f67d94d8e6ae7b25fc8358c5f46894435c04f1d7 Mon Sep 17 00:00:00 2001 From: Antony Rizzitelli Date: Thu, 19 Nov 2026 08:38:25 +0000 Subject: [PATCH 03/16] test(package): drop the package journeys that tests/package.test.ts already runs --- e2e/journeys/logs/histogram-and-drawer.mjs | 1 + e2e/journeys/package.mjs | 149 +-------------------- 2 files changed, 2 insertions(+), 148 deletions(-) diff --git a/e2e/journeys/logs/histogram-and-drawer.mjs b/e2e/journeys/logs/histogram-and-drawer.mjs index 0ee2e1b..05dfde0 100644 --- a/e2e/journeys/logs/histogram-and-drawer.mjs +++ b/e2e/journeys/logs/histogram-and-drawer.mjs @@ -62,6 +62,7 @@ export const histogramAndDrawer = [ }, { id: "a-request-opens-in-a-drawer-that-steps-through-the-list", + tag: "@smoke", title: "a row opens the request in a drawer titled with its method and path and its position over the whole result; Next, Previous, J and K step; ?record names it and a reload opens it again; Summary, Request and Response show what was captured; Replay in tester and Same error follow its links", entryPoint: "/modules/api/logs?record=", async run(/** @type {Ctx} */ ctx) { diff --git a/e2e/journeys/package.mjs b/e2e/journeys/package.mjs index 8e504d1..e5cb694 100644 --- a/e2e/journeys/package.mjs +++ b/e2e/journeys/package.mjs @@ -1,150 +1,3 @@ -import { execFileSync } from "node:child_process"; -import { existsSync, readFileSync } from "node:fs"; -import { createRequire } from "node:module"; -import { join } from "node:path"; import { guarded } from "../guard.mjs"; -import { check, isFunction, isRow, isText, list, parseJson, pick, same, text } from "../api.mjs"; -/** @typedef {import("../api.mjs").Ctx} Ctx */ - -const NO_DATABASE = { database: "shared", onboard: false }; - -/** - * @param {Ctx} ctx The journey - * @param {string} path A path in the checkout - * @returns {string} The file's text - */ -const read = (ctx, path) => readFileSync(join(ctx.state.repo, path), "utf8"); - -/** - * @param {Ctx} ctx The journey - * @param {string} path A JSON file in the checkout - * @returns {unknown} Its value - */ -const readJson = (ctx, path) => parseJson(read(ctx, path)); - -/** - * @param {unknown} value A JSON value - * @returns {string[]} The strings of an array value - */ -const strings = (value) => (Array.isArray(value) ? value.filter(isText) : []); - -/** - * @param {unknown} value A loaded module - * @returns {value is { satisfies: (version: unknown, range: unknown) => unknown }} Whether it is the semver module - */ -function isSemver(value) { - return isRow(value) && isFunction(value.satisfies); -} - -/** - * @param {unknown} value A value - * @param {unknown} range A semver range - * @param {Ctx} ctx The journey - * @returns {boolean} Whether the version satisfies the range - */ -function satisfies(value, range, ctx) { - /** @type {unknown} */ - const semver = createRequire(join(ctx.state.repo, "package.json"))("semver"); - return isSemver(semver) ? Boolean(semver.satisfies(value, range)) : false; -} - -export const journeys = guarded([ - { - id: "the-tarball-holds-what-the-manifest-names", - title: "the package ships dist, the Vue layer and the skill, declares the skill and the frontend module, and leaves out sources, tests, the playground and the verification skill", - entryPoint: "npm pack", - ...NO_DATABASE, - async run(/** @type {Ctx} */ ctx) { - const manifest = readJson(ctx, "package.json"); - const packed = parseJson(execFileSync("npm", ["pack", "--dry-run", "--json", "--ignore-scripts"], { cwd: ctx.state.repo, encoding: "utf8" })); - const files = Array.isArray(packed) ? list(packed[0], "files").map((entry) => text(entry, "path")) : []; - const folder = ctx.step("npm pack"); - ctx.record(folder, [ - same("main", pick(manifest, "main"), "dist/index.js"), - same("the shipped folders", pick(manifest, "files"), ["dist", "frontend-vue", "skills"]), - same("the skills folder is declared", pick(manifest, "antelopeJs", "skills"), ["./skills"]), - same("the frontend module is declared", pick(manifest, "dmsFrontend"), "dist/frontend-module.js"), - check("the built entry is packed", files.includes("dist/index.js"), files.length), - check("the frontend module is packed", files.includes("dist/frontend-module.js"), files.length), - check("the Vue layer is packed", files.includes("frontend-vue/dms.frontend.ts") && files.includes("frontend-vue/package.json"), files.length), - check("the skill is packed", files.includes("skills/dms-api/SKILL.md") && files.includes("skills/dms-api/cases.json"), files.length), - same("nothing else but the manifest, licence and readme", files.filter((path) => !/^(dist|frontend-vue|skills)\//.test(path)).toSorted((left, right) => left.localeCompare(right)), ["LICENSE", "README.md", "package.json"].toSorted((left, right) => left.localeCompare(right))), - same("no source, test or playground file is packed", files.filter((path) => /^(src|tests|playground|\.agents|docs)\//.test(path)), []), - same("no installed dependency is packed", files.filter((path) => path.includes("node_modules/")), []), - ]); - }, - }, - { - id: "the-ranges-and-the-skill-match-what-the-docs-name", - title: "the renderer range of the Vue layer admits the installed renderer, the installation docs' version range admits this version, the README table lists every interface the module depends on, and the skill's frontmatter and cases are valid", - entryPoint: "README, docs/2.installation.md, frontend-vue/package.json, skills/dms-api", - ...NO_DATABASE, - async run(/** @type {Ctx} */ ctx) { - const manifest = readJson(ctx, "package.json"); - const frontend = readJson(ctx, "frontend-vue/package.json"); - const renderer = readJson(ctx, "playground/node_modules/@antelopejs/dms-frontend/package.json"); - const readme = read(ctx, "README.md"); - const table = readme.slice(readme.indexOf("antelopejs-docs-compat:start"), readme.indexOf("antelopejs-docs-compat:end")); - const keysOfRow = (/** @type {unknown} */ value) => (isRow(value) ? Object.keys(value) : []); - const names = [...keysOfRow(pick(manifest, "dependencies")), ...keysOfRow(pick(manifest, "optionalDependencies"))]; - const interfaces = names.filter((name) => name.startsWith("@antelopejs/interface-")); - const installation = read(ctx, "docs/2.installation.md"); - const range = /version: "([^"]+)"/.exec(installation)?.[1] ?? ""; - const skill = read(ctx, "skills/dms-api/SKILL.md"); - const cases = list(readJson(ctx, "skills/dms-api/cases.json"), "cases"); - const engines = text(frontend, "engines", "@antelopejs/dms-frontend"); - await ctx.verify("ranges", [ - check("the renderer installed is inside the Vue layer's engines range", satisfies(text(renderer, "version"), engines, ctx), `${text(renderer, "version")} in ${engines}`), - check("this version is inside the range the installation docs show", satisfies(text(manifest, "version"), range, ctx), `${text(manifest, "version")} in ${range}`), - same("the README table lists each interface dependency", interfaces.filter((name) => !table.includes(`\`${name}\``)), []), - check("the docs name the module's package", installation.includes('package: "@antelopejs/dms-api"'), range), - ]); - await ctx.verify("the skill", [ - check("the frontmatter names dms-api", skill.startsWith("---\nname: dms-api\n"), skill.slice(0, 40)), - check("it names the three automation nodes", ["api.route-unhealthy", "api.get-api-health", "api.query-request-logs"].every((node) => skill.includes(node)), "nodes"), - check("the cases file is version 1, its ids are unique and the skill is expected by the cases that name it", pick(readJson(ctx, "skills/dms-api/cases.json"), "version") === 1 && cases.length > 0 && new Set(cases.map((entry) => text(entry, "id"))).size === cases.length && cases.some((entry) => strings(pick(entry, "expect", "skills")).includes("dms-api")), cases.length), - ]); - }, - }, - { - id: "the-docs-links-and-the-two-locales-are-complete", - title: "every console link of the docs points at a page that exists, the docs navigation lists the pages, and the English and French locale files hold the same keys", - entryPoint: "docs/, frontend-vue/i18n/locales", - ...NO_DATABASE, - async run(/** @type {Ctx} */ ctx) { - const pages = ["introduction", "installation", "configuration", "extending", "faq", "usage/overview", "usage/request-logs", "usage/routes", "usage/settings"]; - const files = ["1.introduction.md", "2.installation.md", "3.configuration.md", "5.extending.md", "6.faq.md", "4.usage/1.overview.md", "4.usage/2.request-logs.md", "4.usage/3.routes.md", "4.usage/4.settings.md"]; - const docs = files.map((file) => read(ctx, `docs/${file}`)).join("\n"); - const unknown = [...docs.matchAll(/\/docs\/modules\/api\/([a-z/-]+)/g)].map((match) => match[1] ?? "").filter((target) => !pages.includes(target)); - /** - * @param {unknown} value A locale value - * @param {string} [prefix] The path so far - * @returns {string[]} The dotted keys of its leaves - */ - const keysOf = (value, prefix = "") => (isRow(value) ? Object.entries(value).flatMap(([key, inner]) => keysOf(inner, `${prefix}${key}.`)) : [prefix]); - const byName = (/** @type {string} */ left, /** @type {string} */ right) => left.localeCompare(right); - const english = keysOf(readJson(ctx, "frontend-vue/i18n/locales/api-en-GB.json")).toSorted(byName); - const french = keysOf(readJson(ctx, "frontend-vue/i18n/locales/api-fr-FR.json")).toSorted(byName); - await ctx.verify("links", [ - same("every page exists", files.filter((file) => !existsSync(join(ctx.state.repo, "docs", file))), []), - same("no console link points at a missing page", unknown, []), - same("the README links the module's documentation", readme(ctx).includes("https://dms.antelopejs.com/docs/modules/api"), true), - same("the navigation names the section", read(ctx, "docs/.navigation.yml").includes("title: API"), true), - ]); - await ctx.verify("locales", [ - same("French lacks no English key", english.filter((key) => !french.includes(key)), []), - same("English lacks no French key", french.filter((key) => !english.includes(key)), []), - check("the locale files are not empty", english.length > 50, english.length), - ]); - }, - }, -]); - -/** - * @param {Ctx} ctx The journey - * @returns {string} The README - */ -function readme(ctx) { - return read(ctx, "README.md"); -} +export const journeys = guarded([]); From abe6c6f67cf6960d15bed935b842faffb8161109 Mon Sep 17 00:00:00 2001 From: Antony Rizzitelli Date: Thu, 19 Nov 2026 08:43:12 +0000 Subject: [PATCH 04/16] test(logs): run the logs journeys as integration tests --- e2e/journeys/logs/filters-and-labels.mjs | 33 +-- e2e/journeys/logs/histogram-and-drawer.mjs | 43 --- e2e/journeys/logs/list-and-pause.mjs | 27 +- tests/logs.journeys.test.ts | 300 +++++++++++++++++++++ 4 files changed, 302 insertions(+), 101 deletions(-) create mode 100644 tests/logs.journeys.test.ts diff --git a/e2e/journeys/logs/filters-and-labels.mjs b/e2e/journeys/logs/filters-and-labels.mjs index bdea621..2a15d7f 100644 --- a/e2e/journeys/logs/filters-and-labels.mjs +++ b/e2e/journeys/logs/filters-and-labels.mjs @@ -1,41 +1,10 @@ -import { isOn, list, mon, num, pick, same, saveSettings, settled, text, traffic, openPage } from "../../api.mjs"; +import { isOn, list, mon, pick, same, saveSettings, settled, text, traffic, openPage } from "../../api.mjs"; import { seedLogs } from "../../seed.mjs"; -import { listed } from "./shared.mjs"; /** @typedef {import("../../api.mjs").Ctx} Ctx */ /** The logs journeys about filters and labels. */ export const filtersAndLabels = [ - { - id: "filters-combine-with-the-status-tab", - title: "a status tab, the Method filter, Slow and a route narrow the list together, each by its own rule", - entryPoint: "GET /api/monitoring/logs?filter_statusClass=&filter_method=&filter_slow=&filter_route=", - async run(/** @type {Ctx} */ ctx) { - await saveSettings(ctx, "slow from 50 ms", { requestSlownessThresholdMs: 50 }); - await traffic(ctx, "eight calls", [ - { path: "/fx/ok", count: 3 }, { path: "/fx/slow?ms=80", count: 1 }, { path: "/fx/echo", count: 2, method: "POST", headers: { "content-type": "application/json" }, body: "{}" }, - { path: "/fx/boom", count: 1 }, { path: "/fx/missing", count: 1 }, - ]); - await settled(ctx, "the logs hold 8", "/logs?preset=last-24h", (body) => num(body, "total") === 8); - const cases = [ - ["2xx and POST", "filter_statusClass=is:2xx&filter_method=is:POST", 2, ["/fx/echo"]], - ["2xx and GET", "filter_statusClass=is:2xx&filter_method=is:GET", 4, ["/fx/ok", "/fx/slow"]], - ["4xx and GET", "filter_statusClass=is:4xx&filter_method=is:GET", 1, ["/fx/missing"]], - ["4xx and POST", "filter_statusClass=is:4xx&filter_method=is:POST", 0, []], - ["5xx and slow", "filter_statusClass=is:5xx&filter_slow=is:true", 0, []], - ["2xx and slow", "filter_statusClass=is:2xx&filter_slow=is:true", 1, ["/fx/slow"]], - ["slow and POST", "filter_slow=is:true&filter_method=is:POST", 0, []], - ["2xx and a route", `filter_statusClass=is:2xx&filter_route=is:${encodeURIComponent("POST /fx/echo")}`, 2, ["/fx/echo"]], - ["a route under another tab", `filter_statusClass=is:5xx&filter_route=is:${encodeURIComponent("POST /fx/echo")}`, 0, []], - ]; - const results = []; - for (const [label, query, total, paths] of cases) { - const got = await listed(ctx, String(label), String(query)); - results.push(same(String(label), [got.total, [...new Set(got.paths)].toSorted((left, right) => left.localeCompare(right))], [total, paths])); - } - await ctx.verify("combined filters", results); - }, - }, { id: "the-list-sorts-by-duration-both-ways", title: "sorting by duration puts the shortest request first ascending and the longest first descending, and the table writes durations as less than 1 ms, whole milliseconds and grouped thousands", diff --git a/e2e/journeys/logs/histogram-and-drawer.mjs b/e2e/journeys/logs/histogram-and-drawer.mjs index 0ee2e1b..e065739 100644 --- a/e2e/journeys/logs/histogram-and-drawer.mjs +++ b/e2e/journeys/logs/histogram-and-drawer.mjs @@ -140,47 +140,4 @@ export const histogramAndDrawer = [ await ctx.verify("without the range", [tab.visible({ role: "button", name: "All 7" }), tab.visible({ role: "button", name: "5xx 2" })]); }, }, - { - id: "the-histogram-counts-past-days-by-class", - title: "the chart over 7D and 30D counts the requests of past days by status class, each in the bar of its time, as the database counts them", - entryPoint: "GET /api/monitoring/logs/live", - async run(/** @type {Ctx} */ ctx) { - await seedLogs(ctx, "4 good calls 3 days ago", { daysAgo: 3, count: 4, uri: "/fx/ok" }); - await seedLogs(ctx, "2 client errors 3 days ago", { daysAgo: 3, count: 2, uri: "/fx/missing", statusCode: 404 }); - await seedLogs(ctx, "1 server error 3 days ago", { daysAgo: 3, count: 1, uri: "/fx/boom", statusCode: 500 }); - await seedLogs(ctx, "2 good calls 10 days ago", { daysAgo: 10, count: 2, uri: "/fx/ok" }); - const at = Date.now() - 3 * 86_400_000; - const week = await mon(ctx, "histogram last-7-days", "/logs/live?preset=last-7-days"); - const month = await mon(ctx, "histogram last-30-days", "/logs/live?preset=last-30-days"); - /** @param {unknown} body */ - const byClass = (body) => list(body, "buckets").reduce((all, bucket) => [all[0] + (num(bucket, "success") ?? 0), all[1] + (num(bucket, "clientErrors") ?? 0), all[2] + (num(bucket, "serverErrors") ?? 0)], [0, 0, 0]); - const near = list(week.body, "buckets").filter((bucket) => Math.abs((num(bucket, "start") ?? 0) - at) <= 2 * 7_200_000); - await ctx.verify("the past days", [ - same("7D: the totals", [num(week.body, "total"), num(week.body, "clientErrors"), num(week.body, "serverErrors")], [7, 2, 1]), - same("7D: the bars by class", byClass(week.body), [4, 2, 1]), - same("7D: the bars around 3 days ago hold them", byClass({ buckets: near }), [4, 2, 1]), - same("30D: the totals, the calls of 10 days ago included", [num(month.body, "total"), num(month.body, "clientErrors"), num(month.body, "serverErrors")], [9, 2, 1]), - same("30D: the bars by class", byClass(month.body), [6, 2, 1]), - ]); - }, - }, - { - id: "a-past-range-draws-at-most-120-bars", - title: "the chart keeps between 50 and 120 bars for a past range, one that starts off a bar boundary included", - entryPoint: "GET /api/monitoring/logs/live?from=&to=", - async run(/** @type {Ctx} */ ctx) { - /** @type {[number, number][]} */ - const ranges = [[3, 2], [6, 1], [20, 10], [50, 2], [0.1, 0.05]]; - const answers = []; - for (const [startDays, endDays] of ranges) { - const from = new Date(Date.now() - startDays * 86_400_000).toISOString(); - const to = new Date(Date.now() - endDays * 86_400_000).toISOString(); - answers.push(await mon(ctx, `a past range from ${startDays} to ${endDays} days ago`, `/logs/live?from=${encodeURIComponent(from)}&to=${encodeURIComponent(to)}`)); - } - await ctx.verify("the bar count the docs promise", answers.map((answer, index) => { - const bars = list(answer.body, "buckets").length; - return check(`${ranges[index]?.join(" to ")} days ago: 50 to 120 bars`, bars >= 50 && bars <= 120, bars); - })); - }, - }, ]; diff --git a/e2e/journeys/logs/list-and-pause.mjs b/e2e/journeys/logs/list-and-pause.mjs index 86d3de6..0d7cf64 100644 --- a/e2e/journeys/logs/list-and-pause.mjs +++ b/e2e/journeys/logs/list-and-pause.mjs @@ -1,5 +1,4 @@ -import { check, isOn, list, mon, num, pick, same, saveSettings, settled, text, traffic, openPage } from "../../api.mjs"; -import { seedLogs } from "../../seed.mjs"; +import { isOn, num, same, saveSettings, settled, traffic, openPage } from "../../api.mjs"; import { listed } from "./shared.mjs"; /** @typedef {import("../../api.mjs").Ctx} Ctx */ @@ -63,30 +62,6 @@ export const listAndPause = [ await ctx.verify("Slow", [tab.shows(/\/fx\/slow/), tab.shows(/\/fx\/boom/, { not: true, timeoutMs: 3000 })]); }, }, - { - id: "the-list-is-newest-first-sorts-and-pages", - title: "the list is newest first, sorts by status and duration, pages by 50 and keeps the total over the pages", - entryPoint: "GET /api/monitoring/logs?sortKey=...&offset=...", - async run(/** @type {Ctx} */ ctx) { - await seedLogs(ctx, "58 ok", { daysAgo: 0.01, count: 58 }); - await seedLogs(ctx, "two not found", { daysAgo: 0.02, count: 2, statusCode: 404, uri: "/fx/missing", message: "seeded" }); - const first = await listed(ctx, "first page", ""); - const second = await listed(ctx, "second page", "offset=50"); - const byStatus = await mon(ctx, "by status", "/logs?preset=last-24h&sortKey=status&sortDirection=asc&limit=3"); - const byStatusDesc = await mon(ctx, "by status, descending", "/logs?preset=last-24h&sortKey=status&sortDirection=desc&limit=3"); - const oldest = await mon(ctx, "oldest first", "/logs?preset=last-24h&sortKey=timestamp&sortDirection=asc&limit=2"); - const newest = await mon(ctx, "newest first", "/logs?preset=last-24h&limit=2"); - await ctx.verify("paging", [ - same("the total", first.total, 60), - same("the first page holds 50 rows", first.paths.length, 50), - same("the second page holds the other 10", second.paths.length, 10), - same("status ascending starts with a 200", list(byStatus.body, "results").map((row) => text(row, "status")), ["200", "200", "200"]), - same("status descending starts with the 404s", list(byStatusDesc.body, "results").map((row) => text(row, "status")).slice(0, 2), ["404", "404"]), - check("newest first", Date.parse(text(list(newest.body, "results")[0], "timestamp")) > Date.parse(text(list(oldest.body, "results")[0], "timestamp")), pick(newest.body, "total")), - same("the oldest rows are the two 404s", list(oldest.body, "results").map((row) => text(row, "status")), ["404", "404"]), - ]); - }, - }, { id: "pause-holds-the-table-and-counts-what-was-missed", title: "Pause stops the reads: the toolbar counts the new requests and the 5xx among them, the table keeps its rows and offers Show; Show reads them", diff --git a/tests/logs.journeys.test.ts b/tests/logs.journeys.test.ts new file mode 100644 index 0000000..822f215 --- /dev/null +++ b/tests/logs.journeys.test.ts @@ -0,0 +1,300 @@ +import { describe, expect } from "vitest"; +import { z } from "zod"; +import { type Dms, test } from "./helpers/dms"; +import { type Json, resultsOf } from "./helpers/json"; +import { seedLogs } from "./helpers/seed"; + +const MS_PER_DAY = 86_400_000; + +const listSchema = z.object({ + total: z.number(), + results: z.array(z.object({ path: z.string() })), +}); +const statusSchema = z.object({ status: z.string() }); +const timestampSchema = z.object({ timestamp: z.string() }); +const liveSchema = z.object({ + total: z.number(), + clientErrors: z.number(), + serverErrors: z.number(), + buckets: z.array( + z.object({ + start: z.number(), + success: z.number().default(0), + clientErrors: z.number().default(0), + serverErrors: z.number().default(0), + }), + ), +}); + +type Bucket = z.infer["buckets"][number]; + +async function listed( + dms: Dms, + query: string, +): Promise<{ total: number; paths: string[] }> { + const answer = await dms.mon(`/logs?preset=last-24h&${query}`); + const parsed = listSchema.parse(answer.body); + return { total: parsed.total, paths: parsed.results.map((row) => row.path) }; +} + +function byClass(buckets: Bucket[]): number[] { + return [ + buckets.reduce((sum, bucket) => sum + bucket.success, 0), + buckets.reduce((sum, bucket) => sum + bucket.clientErrors, 0), + buckets.reduce((sum, bucket) => sum + bucket.serverErrors, 0), + ]; +} + +function totalsOf(body: Json): number[] { + const live = liveSchema.parse(body); + return [live.total, live.clientErrors, live.serverErrors]; +} + +interface Pages { + first: { total: number; paths: string[] }; + second: { total: number; paths: string[] }; + ascending: string[]; + descending: string[]; + oldest: { statuses: string[]; at: number }; + newestAt: number; +} + +async function statusesAndTimes( + dms: Dms, + query: string, +): Promise<{ statuses: string[]; at: number }> { + const answer = await dms.mon(`/logs?preset=last-24h&${query}`); + return { + statuses: resultsOf(answer.body, statusSchema).map((row) => row.status), + at: Date.parse(resultsOf(answer.body, timestampSchema)[0]?.timestamp ?? ""), + }; +} + +async function readPages(dms: Dms): Promise { + await seedLogs(dms, { daysAgo: 0.01, count: 58 }); + await seedLogs(dms, { + daysAgo: 0.02, + count: 2, + statusCode: 404, + uri: "/fx/missing", + message: "seeded", + }); + return { + first: await listed(dms, ""), + second: await listed(dms, "offset=50"), + ascending: ( + await statusesAndTimes(dms, "sortKey=status&sortDirection=asc&limit=3") + ).statuses, + descending: ( + await statusesAndTimes(dms, "sortKey=status&sortDirection=desc&limit=3") + ).statuses, + oldest: await statusesAndTimes( + dms, + "sortKey=timestamp&sortDirection=asc&limit=2", + ), + newestAt: (await statusesAndTimes(dms, "limit=2")).at, + }; +} + +async function seedPastDays(dms: Dms): Promise { + await seedLogs(dms, { daysAgo: 3, count: 4, uri: "/fx/ok" }); + await seedLogs(dms, { + daysAgo: 3, + count: 2, + uri: "/fx/missing", + statusCode: 404, + }); + await seedLogs(dms, { + daysAgo: 3, + count: 1, + uri: "/fx/boom", + statusCode: 500, + }); + await seedLogs(dms, { daysAgo: 10, count: 2, uri: "/fx/ok" }); +} + +async function barsOf( + dms: Dms, + startDays: number, + endDays: number, +): Promise { + const from = new Date(Date.now() - startDays * MS_PER_DAY).toISOString(); + const to = new Date(Date.now() - endDays * MS_PER_DAY).toISOString(); + const answer = await dms.mon( + `/logs/live?from=${encodeURIComponent(from)}&to=${encodeURIComponent(to)}`, + ); + return liveSchema.parse(answer.body).buckets.length; +} + +async function sendEightCalls(dms: Dms): Promise { + await dms.saveSettings({ requestSlownessThresholdMs: 50 }); + await dms.traffic([ + { path: "/fx/ok", count: 3 }, + { path: "/fx/slow?ms=80", count: 1 }, + { + path: "/fx/echo", + count: 2, + method: "POST", + headers: { "content-type": "application/json" }, + body: "{}", + }, + { path: "/fx/boom", count: 1 }, + { path: "/fx/missing", count: 1 }, + ]); + await dms.settled( + "/logs?preset=last-24h", + (body) => listSchema.safeParse(body).data?.total === 8, + ); +} + +const ECHO = encodeURIComponent("POST /fx/echo"); + +const FILTER_CASES: { + label: string; + query: string; + total: number; + paths: string[]; +}[] = [ + { + label: "2xx and POST", + query: "filter_statusClass=is:2xx&filter_method=is:POST", + total: 2, + paths: ["/fx/echo"], + }, + { + label: "2xx and GET", + query: "filter_statusClass=is:2xx&filter_method=is:GET", + total: 4, + paths: ["/fx/ok", "/fx/slow"], + }, + { + label: "4xx and GET", + query: "filter_statusClass=is:4xx&filter_method=is:GET", + total: 1, + paths: ["/fx/missing"], + }, + { + label: "4xx and POST", + query: "filter_statusClass=is:4xx&filter_method=is:POST", + total: 0, + paths: [], + }, + { + label: "5xx and slow", + query: "filter_statusClass=is:5xx&filter_slow=is:true", + total: 0, + paths: [], + }, + { + label: "2xx and slow", + query: "filter_statusClass=is:2xx&filter_slow=is:true", + total: 1, + paths: ["/fx/slow"], + }, + { + label: "slow and POST", + query: "filter_slow=is:true&filter_method=is:POST", + total: 0, + paths: [], + }, + { + label: "2xx and a route", + query: `filter_statusClass=is:2xx&filter_route=is:${ECHO}`, + total: 2, + paths: ["/fx/echo"], + }, + { + label: "a route under another tab", + query: `filter_statusClass=is:5xx&filter_route=is:${ECHO}`, + total: 0, + paths: [], + }, +]; + +const BAR_RANGES: [number, number][] = [ + [3, 2], + [6, 1], + [20, 10], + [50, 2], + [0.1, 0.05], +]; + +describe("logs", () => { + test("the-list-is-newest-first-sorts-and-pages", async ({ dms }) => { + const pages = await readPages(dms); + expect.soft(pages.first.total, "the total").toBe(60); + expect + .soft(pages.first.paths.length, "the first page holds 50 rows") + .toBe(50); + expect + .soft(pages.second.paths.length, "the second page holds the other 10") + .toBe(10); + expect + .soft(pages.ascending, "status ascending starts with a 200") + .toEqual(["200", "200", "200"]); + expect + .soft( + pages.descending.slice(0, 2), + "status descending starts with the 404s", + ) + .toEqual(["404", "404"]); + expect + .soft(pages.newestAt, "newest first") + .toBeGreaterThan(pages.oldest.at); + expect + .soft(pages.oldest.statuses, "the oldest rows are the two 404s") + .toEqual(["404", "404"]); + }); + + test("the-histogram-counts-past-days-by-class", async ({ dms }) => { + await seedPastDays(dms); + const at = Date.now() - 3 * MS_PER_DAY; + const week = await dms.mon("/logs/live?preset=last-7-days"); + const month = await dms.mon("/logs/live?preset=last-30-days"); + const weekBuckets = liveSchema.parse(week.body).buckets; + const near = weekBuckets.filter( + (bucket) => Math.abs(bucket.start - at) <= 2 * 7_200_000, + ); + expect.soft(totalsOf(week.body), "7D: the totals").toEqual([7, 2, 1]); + expect + .soft(byClass(weekBuckets), "7D: the bars by class") + .toEqual([4, 2, 1]); + expect + .soft(byClass(near), "7D: the bars around 3 days ago hold them") + .toEqual([4, 2, 1]); + expect + .soft( + totalsOf(month.body), + "30D: the totals, the calls of 10 days ago included", + ) + .toEqual([9, 2, 1]); + expect + .soft( + byClass(liveSchema.parse(month.body).buckets), + "30D: the bars by class", + ) + .toEqual([6, 2, 1]); + }); + + test("a-past-range-draws-at-most-120-bars", async ({ dms }) => { + for (const [startDays, endDays] of BAR_RANGES) { + const bars = await barsOf(dms, startDays, endDays); + expect + .soft( + bars >= 50 && bars <= 120, + `${startDays} to ${endDays} days ago: 50 to 120 bars (${bars})`, + ) + .toBe(true); + } + }); + + test("filters-combine-with-the-status-tab", async ({ dms }) => { + await sendEightCalls(dms); + for (const { label, query, total, paths } of FILTER_CASES) { + const got = await listed(dms, query); + expect + .soft([got.total, [...new Set(got.paths)].toSorted()], label) + .toEqual([total, paths]); + } + }); +}); From c72a86e3fb8fb4990ff60222d4b01c85304414bd Mon Sep 17 00:00:00 2001 From: Antony Rizzitelli Date: Thu, 19 Nov 2026 08:43:18 +0000 Subject: [PATCH 05/16] test(overview): run the overview journeys as integration tests --- e2e/journeys/overview/blocks-and-period.mjs | 30 ----- e2e/journeys/overview/comparison.mjs | 20 ---- tests/overview.journeys.test.ts | 116 ++++++++++++++++++++ 3 files changed, 116 insertions(+), 50 deletions(-) create mode 100644 tests/overview.journeys.test.ts diff --git a/e2e/journeys/overview/blocks-and-period.mjs b/e2e/journeys/overview/blocks-and-period.mjs index c73e050..c77a721 100644 --- a/e2e/journeys/overview/blocks-and-period.mjs +++ b/e2e/journeys/overview/blocks-and-period.mjs @@ -1,4 +1,3 @@ -import { seedLogs } from "../../seed.mjs"; import { check, isOn, list, mon, num, pick, same, saveSettings, settled, text, traffic, openPage } from "../../api.mjs"; /** @typedef {import("../../api.mjs").Ctx} Ctx */ @@ -93,35 +92,6 @@ export const blocksAndPeriod = [ await ctx.verify("?live=0", [isOn(tab, { role: "button", name: "Pause" }), isOn(tab, { role: "button", name: "Live" }, false)]); }, }, - { - id: "down-reads-the-last-five-minutes-and-not-the-last-fifteen", - title: "10 failing calls out of 20 in the last 5 minutes read Down although 100 good calls 10 minutes ago keep the 15-minute rate under 20%; the same errors 10 minutes ago and 20 good calls now read Degraded, not Down", - entryPoint: "GET /api/monitoring/overview/health", - async run(/** @type {Ctx} */ ctx) { - await seedLogs(ctx, "100 good calls 10 minutes ago", { minutesAgo: 10, count: 100 }); - await traffic(ctx, "10 good and 10 failing now", [{ path: "/fx/ok", count: 10 }, { path: "/fx/boom", count: 10 }]); - const recent = await settled(ctx, "health with a recent burst", "/overview/health", (body) => num(body, "metrics", "requests") === 120); - await ctx.verify("a recent burst", [ - same("the verdict", text(recent.body, "verdict"), "down"), - same("the 15-minute rate is the 10 errors over 120 calls", [num(recent.body, "metrics", "requests"), Math.round((num(recent.body, "metrics", "serverErrorRate") ?? 0) * 10) / 10], [120, 8.3]), - ]); - }, - }, - { - id: "errors-older-than-five-minutes-do-not-make-the-api-down", - title: "40 failing calls out of 120 ten minutes ago and 20 good calls now (28.6% over 15 minutes, 0% over 5) read Degraded, not Down", - entryPoint: "GET /api/monitoring/overview/health", - async run(/** @type {Ctx} */ ctx) { - await seedLogs(ctx, "80 good calls 10 minutes ago", { minutesAgo: 10, count: 80 }); - await seedLogs(ctx, "40 failing calls 10 minutes ago", { minutesAgo: 10, count: 40, uri: "/fx/boom", statusCode: 500, message: "seeded" }); - await traffic(ctx, "20 good calls now", [{ path: "/fx/ok", count: 20 }]); - const aged = await settled(ctx, "health with aged errors", "/overview/health", (body) => num(body, "metrics", "requests") === 140); - await ctx.verify("aged errors", [ - same("the verdict", text(aged.body, "verdict"), "degraded"), - same("the 15-minute rate is the 40 errors over 140 calls", Math.round((num(aged.body, "metrics", "serverErrorRate") ?? 0) * 10) / 10, 28.6), - ]); - }, - }, { id: "the-module-path-redirects-to-the-overview", title: "opening /modules/api lands on /modules/api/overview, and the menu lists Overview, Request logs, Routes and API settings under Monitor, Explore and Configure", diff --git a/e2e/journeys/overview/comparison.mjs b/e2e/journeys/overview/comparison.mjs index 5428109..a43aaef 100644 --- a/e2e/journeys/overview/comparison.mjs +++ b/e2e/journeys/overview/comparison.mjs @@ -116,26 +116,6 @@ export const comparison = [ ]); }, }, - { - id: "a-comparison-the-page-does-not-know-compares-with-nothing", - title: "a year back or any name the page does not know gives no previous period, no compared line and no previous-period figure, where the two names it knows do", - entryPoint: "GET /api/monitoring/overview/traffic?comparison=", - async run(/** @type {Ctx} */ ctx) { - await traffic(ctx, "some calls", [{ path: "/fx/ok", count: 3 }]); - await settled(ctx, "counted", "/overview/traffic?preset=last-24h", (body) => num(body, "value") === 3); - const windowOf = async (/** @type {string} */ name) => { - const answer = await mon(ctx, `comparison ${name}`, `/overview/traffic?preset=last-24h&comparison=${name}`); - return [pick(answer.body, "window", "comparison"), pick(answer.body, "window", "previous") === null, pick(answer.body, "window", "previousAvailable"), list(answer.body, "series").length]; - }; - await ctx.verify("the names", [ - same("previous-year", await windowOf("previous-year"), ["none", true, false, 3]), - same("year", await windowOf("year"), ["none", true, false, 3]), - same("an empty name", await windowOf(""), ["none", true, false, 3]), - same("none", await windowOf("none"), ["none", true, false, 3]), - same("previous-period", await windowOf("previous-period"), ["previous-period", false, true, 3]), - ]); - }, - }, { id: "every-figure-compared-with-the-period-before-says-its-change-and-its-value", title: "with 12 calls in the period before and 14 now, Traffic reads +16.7% vs 12, Error share 14.3% vs 0.0%, the slowest route's average is compared with the average before (100 ms against 50), and the status classes against theirs", diff --git a/tests/overview.journeys.test.ts b/tests/overview.journeys.test.ts new file mode 100644 index 0000000..71abaa5 --- /dev/null +++ b/tests/overview.journeys.test.ts @@ -0,0 +1,116 @@ +import { describe, expect } from "vitest"; +import { z } from "zod"; +import { test } from "./helpers/dms"; +import type { Json } from "./helpers/json"; +import { seedLogs } from "./helpers/seed"; + +const healthSchema = z.object({ + verdict: z.string(), + metrics: z.object({ requests: z.number(), serverErrorRate: z.number() }), +}); +const trafficSchema = z.object({ + value: z.number(), + series: z.array(z.object({ name: z.string() })), + window: z.object({ + comparison: z.string(), + previous: z.union([z.null(), z.object({})]), + previousAvailable: z.boolean(), + }), +}); + +function requestsOf(body: Json): number | undefined { + return healthSchema.safeParse(body).data?.metrics.requests; +} + +function tenthsOf(rate: number): number { + return Math.round(rate * 10) / 10; +} + +describe("overview", () => { + test("down-reads-the-last-five-minutes-and-not-the-last-fifteen", async ({ + dms, + }) => { + await seedLogs(dms, { minutesAgo: 10, count: 100 }); + await dms.traffic([ + { path: "/fx/ok", count: 10 }, + { path: "/fx/boom", count: 10 }, + ]); + const recent = await dms.settled( + "/overview/health", + (body) => requestsOf(body) === 120, + ); + const health = healthSchema.parse(recent.body); + expect.soft(health.verdict, "the verdict").toBe("down"); + expect + .soft( + [health.metrics.requests, tenthsOf(health.metrics.serverErrorRate)], + "the 15-minute rate is the 10 errors over 120 calls", + ) + .toEqual([120, 8.3]); + }); + + test("errors-older-than-five-minutes-do-not-make-the-api-down", async ({ + dms, + }) => { + await seedLogs(dms, { minutesAgo: 10, count: 80 }); + await seedLogs(dms, { + minutesAgo: 10, + count: 40, + uri: "/fx/boom", + statusCode: 500, + message: "seeded", + }); + await dms.traffic([{ path: "/fx/ok", count: 20 }]); + const aged = await dms.settled( + "/overview/health", + (body) => requestsOf(body) === 140, + ); + const health = healthSchema.parse(aged.body); + expect.soft(health.verdict, "the verdict").toBe("degraded"); + expect + .soft( + tenthsOf(health.metrics.serverErrorRate), + "the 15-minute rate is the 40 errors over 140 calls", + ) + .toBe(28.6); + }); + + test("a-comparison-the-page-does-not-know-compares-with-nothing", async ({ + dms, + }) => { + await dms.traffic([{ path: "/fx/ok", count: 3 }]); + await dms.settled( + "/overview/traffic?preset=last-24h", + (body) => trafficSchema.safeParse(body).data?.value === 3, + ); + const windowOf = async ( + name: string, + ): Promise<[string, boolean, boolean, number]> => { + const answer = await dms.mon( + `/overview/traffic?preset=last-24h&comparison=${name}`, + ); + const traffic = trafficSchema.parse(answer.body); + return [ + traffic.window.comparison, + traffic.window.previous === null, + traffic.window.previousAvailable, + traffic.series.length, + ]; + }; + expect + .soft(await windowOf("previous-year"), "previous-year") + .toEqual(["none", true, false, 3]); + expect + .soft(await windowOf("year"), "year") + .toEqual(["none", true, false, 3]); + expect + .soft(await windowOf(""), "an empty name") + .toEqual(["none", true, false, 3]); + expect + .soft(await windowOf("none"), "none") + .toEqual(["none", true, false, 3]); + expect + .soft(await windowOf("previous-period"), "previous-period") + .toEqual(["previous-period", false, true, 3]); + }); +}); From ec2ab0140ac21dde6968d42fcf2a122b25ed626b Mon Sep 17 00:00:00 2001 From: Antony Rizzitelli Date: Thu, 19 Nov 2026 08:48:18 +0000 Subject: [PATCH 06/16] test(access): run the access journeys as integration tests --- e2e/journeys/access.mjs | 96 +------------ e2e/journeys/capture/redaction.mjs | 161 --------------------- e2e/journeys/capture/shared.mjs | 5 - tests/access.journeys.test.ts | 222 +++++++++++++++++++++++++++++ 4 files changed, 223 insertions(+), 261 deletions(-) delete mode 100644 e2e/journeys/capture/redaction.mjs delete mode 100644 e2e/journeys/capture/shared.mjs create mode 100644 tests/access.journeys.test.ts diff --git a/e2e/journeys/access.mjs b/e2e/journeys/access.mjs index ddc1fdc..20605b4 100644 --- a/e2e/journeys/access.mjs +++ b/e2e/journeys/access.mjs @@ -1,106 +1,12 @@ import { guarded } from "../guard.mjs"; -import { check, isText, list, member, mon, raw, same, saveSettings, settled, text, traffic, openPage } from "../api.mjs"; +import { check, list, member, mon, raw, same, saveSettings, settled, text, traffic, openPage } from "../api.mjs"; /** @typedef {import("../api.mjs").Ctx} Ctx */ /** @typedef {import("../api.mjs").Row} Row */ const TESTER_HEADER = "x-antelopejs-api-source"; -/** Every route of the console: method and path under /api/monitoring. */ -const ROUTES = [ - ["GET", "/overview/health"], ["GET", "/overview/traffic"], ["GET", "/overview/classes"], ["GET", "/overview/error-share"], ["GET", "/overview/attention"], ["GET", "/overview/slowest"], ["GET", "/overview/activity"], - ["GET", "/logs"], ["GET", "/logs/recent"], ["GET", "/logs/routes?route=GET%20%2Ffx%2Fok"], ["GET", "/logs/tester?requestId=x"], ["GET", "/logs/live"], - ["GET", "/logs/missing"], ["GET", "/logs/recent/missing"], ["GET", "/logs/tester/missing"], ["GET", "/logs/routes/missing"], - ["GET", "/routes/list"], ["GET", "/routes/detail?route=GET%20%2Ffx%2Fok"], ["GET", "/routes/statistics/activity?route=GET%20%2Ffx%2Fok"], ["GET", "/routes/statistics/summary?route=GET%20%2Ffx%2Fok"], - ["GET", "/routes/statistics/status?route=GET%20%2Ffx%2Fok"], ["GET", "/routes/statistics/latency-chart?route=GET%20%2Ffx%2Fok"], ["GET", "/routes/statistics/errors?route=GET%20%2Ffx%2Fok"], - ["GET", "/settings"], ["POST", "/settings"], ["GET", "/settings/retention-impact?field=requestLogRetentionDays&days=30"], ["GET", "/settings/summary"], ["GET", "/tester/source"], -]; - -/** The permission each route of the console names in a refusal: one block, one id. */ -const BLOCK_OF = new Map([ - ["GET /overview/health", "modules.api.monitor.overview.health"], - ["GET /overview/traffic", "modules.api.monitor.overview.dashboard.traffic.chart"], - ["GET /overview/classes", "modules.api.monitor.overview.dashboard.traffic.classes"], - ["GET /overview/error-share", "modules.api.monitor.overview.dashboard.health.errorShare"], - ["GET /overview/attention", "modules.api.monitor.overview.dashboard.health.attention"], - ["GET /overview/slowest", "modules.api.monitor.overview.dashboard.health.slowest"], - ["GET /overview/activity", "modules.api.monitor.overview.dashboard.recent.activity"], - ["GET /logs", "modules.api.monitor.logs.requests"], - ["GET /logs/recent", "modules.api.monitor.overview.dashboard.recent.requests.table"], - ["GET /logs/routes", "modules.api.explore.routes.explorer.tabs.statistics.recent.requests.table"], - ["GET /logs/tester", "modules.api.explore.routes.explorer.tabs.tester"], - ["GET /logs/live", "modules.api.monitor.logs.live"], - ["GET /logs/missing", "modules.api.monitor.logs.requests"], - ["GET /logs/recent/missing", "modules.api.monitor.overview.dashboard.recent.requests.table"], - ["GET /logs/tester/missing", "modules.api.explore.routes.explorer.tabs.tester"], - ["GET /logs/routes/missing", "modules.api.explore.routes.explorer.tabs.statistics.recent.requests.table"], - ["GET /routes/list", "modules.api.explore.routes.explorer.tree"], - ["GET /routes/detail", "modules.api.explore.routes.explorer"], - ["GET /routes/statistics/activity", "modules.api.explore.routes.explorer.tabs.statistics"], - ["GET /routes/statistics/summary", "modules.api.explore.routes.explorer.tabs.statistics.grid.summary.summary"], - ["GET /routes/statistics/status", "modules.api.explore.routes.explorer.tabs.statistics.grid.charts.status"], - ["GET /routes/statistics/latency-chart", "modules.api.explore.routes.explorer.tabs.statistics.grid.charts.latency"], - ["GET /routes/statistics/errors", "modules.api.explore.routes.explorer.tabs.statistics.grid.detail.errors"], - ["GET /settings", "modules.api.configure.settings.form"], - ["POST /settings", "modules.api.configure.settings.form"], - ["GET /settings/retention-impact", "modules.api.configure.settings.form"], - ["GET /tester/source", "modules.api.explore.routes.explorer.tabs.tester"], -]); - -/** - * @param {Ctx} ctx The journey - * @param {string} as The account - * @returns {Promise<{ route: string, status: number | null, message: string }[]>} What each route answered - */ -async function sweep(ctx, as) { - const seen = []; - for (const [method, path] of ROUTES) { - const answer = await mon(ctx, `${method} ${path} as ${as}`, path, { method, as, body: method === "POST" ? {} : undefined }); - const body = answer.body; - seen.push({ route: `${method} ${path.split("?")[0]}`, status: answer.status, message: isText(body) ? body : text(body, "message") || text(body, "error") }); - } - return seen; -} - export const journeys = guarded([ - { - id: "every-console-route-needs-a-session-and-the-block-it-serves", - tag: "@smoke", - title: "every route of the console answers 401 without a session and 403 naming the missing permission to a member whose role holds none, while the owner passes every one", - entryPoint: "every /api/monitoring route", - async run(/** @type {Ctx} */ ctx) { - await traffic(ctx, "a request to read", [{ path: "/fx/ok", count: 1 }]); - await member(ctx, "plain", []); - const anonymous = await sweep(ctx, "anon"); - const plain = await sweep(ctx, "plain"); - const owner = await sweep(ctx, "admin"); - await ctx.verify("without a session", anonymous.map((entry) => same(`${entry.route} is 401`, entry.status, 401))); - await ctx.verify("a member with no permission", plain.map((entry) => same(`${entry.route} refuses with its own block's permission`, entry.message, entry.route === "GET /settings/summary" ? "Owner-only" : `Forbidden: missing permission ${BLOCK_OF.get(entry.route)}`))); - await ctx.verify("the owner", owner.map((entry) => check(`${entry.route} passes the guard`, entry.status !== 401 && entry.status !== 403, entry))); - }, - }, - { - id: "the-console-is-the-owners-and-a-role-holding-a-block-is-refused", - title: "a role holding the Request logs table's permission, or the Tester's, is still refused on every route of the console, while the owner opens both", - entryPoint: "GET /api/monitoring/logs and /api/monitoring/tester/source as members", - async run(/** @type {Ctx} */ ctx) { - await traffic(ctx, "a request to read", [{ path: "/fx/ok", count: 1 }]); - await member(ctx, "plain", []); - const refused = await sweep(ctx, "plain"); - const needed = (/** @type {string} */ route) => /missing permission ([\w.-]+)/.exec(refused.find((entry) => entry.route === route)?.message ?? "")?.[1] ?? ""; - const logsPermission = needed("GET /logs"); - const testerPermission = needed("GET /tester/source"); - await ctx.verify("the permissions the refusals name", [check("the logs table has its own permission", logsPermission.startsWith("modules.api."), logsPermission), check("the Tester has its own permission", testerPermission.startsWith("modules.api.") && testerPermission !== logsPermission, testerPermission)]); - await member(ctx, "reader", [logsPermission]); - await member(ctx, "tester", [testerPermission]); - const reader = await sweep(ctx, "reader"); - const tester = await sweep(ctx, "tester"); - const owner = await sweep(ctx, "admin"); - await ctx.verify("the reader", reader.map((entry) => check(`${entry.route} is refused`, entry.status === 403, entry))); - await ctx.verify("the tester", tester.map((entry) => check(`${entry.route} is refused`, entry.status === 403, entry))); - await ctx.verify("the owner", ["GET /logs", "GET /tester/source"].map((route) => check(`${route} opens`, owner.find((entry) => entry.route === route)?.status === 200, owner.find((entry) => entry.route === route)))); - }, - }, { id: "a-member-without-the-modules-permissions-is-sent-to-their-settings", title: "every page of the console sends a member whose role holds none of its permissions to their own Settings page, where none of the console's headings show, while the owner stays on the page", diff --git a/e2e/journeys/capture/redaction.mjs b/e2e/journeys/capture/redaction.mjs deleted file mode 100644 index f68c6d2..0000000 --- a/e2e/journeys/capture/redaction.mjs +++ /dev/null @@ -1,161 +0,0 @@ -import { check, isRow, isText, mon, parseJson, pick, same, saveSettings, sendAndRead, text } from "../../api.mjs"; -import { DEFAULT_HEADERS, REDACTED_WORDS, JSON_HEADERS } from "./shared.mjs"; - -/** @typedef {import("../../api.mjs").Ctx} Ctx */ - -/** The capture journeys about redaction. */ -export const redaction = [ - { - id: "only-the-listed-headers-are-stored-and-secret-ones-never", - title: "the default allow-list stores eight headers; a header added in the settings is stored from then on; authorization, cookie and API-key headers are refused in the list and never stored; an empty list stores none", - entryPoint: "POST /api/monitoring/settings then /fx/ok with headers", - async run(/** @type {Ctx} */ ctx) { - const before = await sendAndRead(ctx, "default list", { path: "/fx/ok", headers: JSON_HEADERS }); - const stored = Object.keys(pick(before.log, "requestHeaders") ?? {}).sort(); - await ctx.verify("the default list", [ - same("the stored request headers", stored, ["accept", "accept-encoding", "content-type", "host", "origin", "referer", "user-agent"]), - same("a header outside the list is absent", pick(before.log, "requestHeaders", "x-webhook-signature"), undefined), - same("the allow-list in force", (await mon(ctx, "settings", "/settings")).body?.requestLogCaptureHeaders, DEFAULT_HEADERS), - ]); - const refusedList = await saveSettings(ctx, "a secret header in the list", { requestLogCaptureHeaders: ["x-webhook-signature", "authorization"] }); - await ctx.verify("a secret header is refused", [same("HTTP 400", refusedList.status, 400)]); - const refusals = []; - for (const header of ["cookie", "set-cookie", "x-api-key", "proxy-authorization", "x-antelopejs-api-source"]) refusals.push([header, (await saveSettings(ctx, `${header} in the list`, { requestLogCaptureHeaders: [header] })).status]); - await ctx.verify("every credential header is refused", refusals.map(([header, status]) => same(`${String(header)} is refused`, status, 400))); - const saved = await saveSettings(ctx, "add the webhook signature, in capitals", { requestLogCaptureHeaders: ["X-Webhook-Signature", "x-webhook-signature", "user-agent"] }); - await ctx.verify("stored lower case without repeats", [same("HTTP 200", saved.status, 200), same("the list", pick(saved.body, "requestLogCaptureHeaders"), ["x-webhook-signature", "user-agent"])]); - const after = await sendAndRead(ctx, "custom list", { path: "/fx/ok", headers: JSON_HEADERS }); - await ctx.verify("the custom list", [ - same("the stored request headers", Object.keys(pick(after.log, "requestHeaders") ?? {}).sort(), ["user-agent", "x-webhook-signature"]), - same("the signature value", pick(after.log, "requestHeaders", "x-webhook-signature"), "sig-1"), - same("authorization is never stored", pick(after.log, "requestHeaders", "authorization"), undefined), - ]); - await saveSettings(ctx, "an empty list", { requestLogCaptureHeaders: [] }); - const none = await sendAndRead(ctx, "empty list", { path: "/fx/ok", headers: JSON_HEADERS }); - await ctx.verify("the empty list", [same("no request header is stored", pick(none.log, "requestHeaders"), {}), same("the override stays explicit", (await mon(ctx, "settings", "/settings")).body?.requestLogCaptureHeaders, [])]); - }, - }, - { - id: "values-under-secret-looking-keys-are-redacted-before-storage", - title: "every key of the fixed list, in request bodies, response bodies and query strings, is stored as [REDACTED] and counted; other keys keep their values", - entryPoint: "POST /fx/echo with one key per redacted word", - async run(/** @type {Ctx} */ ctx) { - const settings = await mon(ctx, "the fixed list", "/settings"); - const listed = pick(settings.body, "redactedKeys"); - const words = Array.isArray(listed) && listed.every(isText) ? listed : []; - const fields = new Map([...words.map((word) => /** @type {const} */ ([word, `value-of-${word}`])), /** @type {const} */ (["name", "plain"])]); - const body = { ...Object.fromEntries(fields), nested: { token: "deep", keep: "kept" } }; - const query = words.map((word) => `${word}=q-${word}`).join("&"); - const sent = await sendAndRead(ctx, "all the words", { method: "POST", path: `/fx/echo?${query}&page=2`, headers: { "content-type": "application/json" }, body: JSON.stringify(body) }); - const stored = parseJson(text(sent.log, "requestBody") || "{}"); - const echoed = parseJson(text(sent.log, "responseBody") || "{}"); - await ctx.verify("the list", [same("it is the 23 words the settings page shows", words, REDACTED_WORDS)]); - await ctx.verify("request body", [ - ...words.map((word) => same(`${word} in the request body`, pick(stored, word), "[REDACTED]")), - same("a plain key keeps its value", pick(stored, "name"), "plain"), - same("a nested secret is redacted and its sibling kept", pick(stored, "nested"), { token: "[REDACTED]", keep: "kept" }), - same("the response body echoes it redacted too", pick(echoed, "password"), "[REDACTED]"), - ]); - await ctx.verify("query string", [ - ...words.map((word) => same(`${word} in the query`, pick(sent.log, "query", word), "[REDACTED]")), - same("a plain parameter keeps its value", pick(sent.log, "query", "page"), "2"), - ]); - await ctx.verify("the counts", [same("query", pick(sent.log, "redactions", "query"), words.length), same("request body", pick(sent.log, "redactions", "requestBody"), words.length + 1), same("response body", pick(sent.log, "redactions", "responseBody"), words.length + 1)]); - }, - }, - { - id: "bodies-above-the-cap-are-stored-truncated-and-flagged", - title: "a body of exactly the cap is stored whole, one byte more is stored cut with the full size, the cap applies to responses too, and a cap of 0 turns body capture off", - entryPoint: "POST /api/monitoring/settings requestLogMaxBodyKb then /fx/echo", - async run(/** @type {Ctx} */ ctx) { - await saveSettings(ctx, "cap at 1 KB", { requestLogMaxBodyKb: 1 }); - const exact = JSON.stringify({ a: "x".repeat(1024 - 8) }); - const over = JSON.stringify({ a: "x".repeat(1024 - 7) }); - const atCap = await sendAndRead(ctx, "exactly 1024 bytes", { method: "POST", path: "/fx/echo", headers: { "content-type": "application/json" }, body: exact }); - const aboveCap = await sendAndRead(ctx, "1025 bytes", { method: "POST", path: "/fx/echo", headers: { "content-type": "application/json" }, body: over }); - await ctx.verify("the cap", [ - same("the body at the cap is whole", [Buffer.byteLength(exact), text(atCap.log, "requestBody") === exact, pick(atCap.log, "requestBodyTruncated")], [1024, true, false]), - same("the body above the cap is flagged", pick(aboveCap.log, "requestBodyTruncated"), true), - same("it keeps the first 1024 bytes", text(aboveCap.log, "requestBody"), over.slice(0, 1024)), - same("it records the whole size", [Buffer.byteLength(over), pick(aboveCap.log, "requestBodySize")], [1025, 1025]), - same("the echoed response is cut too", [pick(aboveCap.log, "responseBodyTruncated"), text(aboveCap.log, "responseBody").length], [true, 1024]), - ]); - await saveSettings(ctx, "no body capture", { requestLogMaxBodyKb: 0 }); - const off = await sendAndRead(ctx, "capture off", { method: "POST", path: "/fx/echo", headers: { "content-type": "application/json" }, body: exact }); - await ctx.verify("a cap of 0", [same("no request body is stored", pick(off.log, "requestBody"), null), same("no response body is stored", pick(off.log, "responseBody"), null), check("the status is still stored", pick(off.log, "statusCode") === 201, pick(off.log, "statusCode"))]); - }, - }, - { - id: "a-key-is-secret-by-the-words-it-is-made-of", - title: "keys such as clientSecret, x-api-key, accessToken, userPin, creditCard and card_cvc2 are redacted, short look-alikes such as spin, footprint, company and cardBrand are not, and a payment card number is redacted whatever its key", - entryPoint: "POST /fx/echo with keys and values", - async run(/** @type {Ctx} */ ctx) { - const redactedKeys = ["clientSecret", "x-api-key", "accessToken", "access_token", "accesstoken", "refreshtoken", "userPin", "creditCard", "card_cvc2", "myPassword", "iban", "totp", "X-Authorization"]; - const keptKeys = ["spin", "footprint", "isNotPaid", "company", "cardBrand", "tokenizer", "pinned", "panda", "cvcs"]; - const body = { ...Object.fromEntries(redactedKeys.map((key) => [key, "v-" + key])), ...Object.fromEntries(keptKeys.map((key) => [key, "v-" + key])), memo: "4111 1111 1111 1111", ref: "4111111111111111", short: "411111111111", note: "order 12345" }; - const sent = await sendAndRead(ctx, "keys", { method: "POST", path: "/fx/echo", headers: { "content-type": "application/json" }, body: JSON.stringify(body) }); - const stored = parseJson(text(sent.log, "requestBody") || "{}"); - await ctx.verify("secret keys", redactedKeys.map((key) => same(`${key} is redacted`, pick(stored, key), "[REDACTED]"))); - await ctx.verify("look-alikes", keptKeys.map((key) => same(`${key} keeps its value`, pick(stored, key), "v-" + key))); - await ctx.verify("card numbers", [ - same("a spaced card number is redacted whatever its key", pick(stored, "memo"), "[REDACTED]"), - same("a plain card number is redacted", pick(stored, "ref"), "[REDACTED]"), - same("twelve digits are not a card number", pick(stored, "short"), "411111111111"), - same("other text is kept", pick(stored, "note"), "order 12345"), - ]); - }, - }, - { - id: "secrets-and-card-numbers-are-redacted-wherever-they-sit", - title: "card numbers and credential keys are redacted in query values (repeated ones too), form bodies, JSON arrays and nested arrays, at their count; invalid or look-alike digits are kept; bodies that are not JSON or form data, malformed JSON and bare JSON strings are not stored", - entryPoint: "GET /api/monitoring/logs/:id after calls to /fx/ok and /fx/echo", - async run(/** @type {Ctx} */ ctx) { - const query = await sendAndRead(ctx, "query values", { path: "/fx/ok?memo=4111111111111111&dash=4111-1111-1111-1111&sp=4111%201111%201111%201111&bad=4111111111111112&ms=1700000000000&amex=378282246310005&mc=5555555555554444&rep=1&rep=5555555555554444&token=abc&page=2&text=order%204111%201111%201111%201111%20paid" }); - await ctx.verify("the query string", [ - same("a plain card number", pick(query.log, "query", "memo"), "[REDACTED]"), - same("a card number with dashes", pick(query.log, "query", "dash"), "[REDACTED]"), - same("a card number with spaces", pick(query.log, "query", "sp"), "[REDACTED]"), - same("an American Express number", pick(query.log, "query", "amex"), "[REDACTED]"), - same("a Mastercard number", pick(query.log, "query", "mc"), "[REDACTED]"), - same("16 digits that fail the Luhn check are kept", pick(query.log, "query", "bad"), "4111111111111112"), - same("an epoch in milliseconds is kept", pick(query.log, "query", "ms"), "1700000000000"), - same("a repeated parameter keeps its plain value and redacts its card", pick(query.log, "query", "rep"), ["1", "[REDACTED]"]), - same("a credential key", pick(query.log, "query", "token"), "[REDACTED]"), - same("a plain parameter", pick(query.log, "query", "page"), "2"), - same("a card number inside a sentence", pick(query.log, "query", "text"), "order [REDACTED] paid"), - same("the count: memo, dash, sp, amex, mc, rep, token, text", pick(query.log, "redactions", "query"), 8), - ]); - const form = await sendAndRead(ctx, "form body", { method: "POST", path: "/fx/echo", headers: { "content-type": "application/x-www-form-urlencoded" }, body: "memo=4111111111111111&name=plain&password=x&amex=378282246310005&bad=4111111111111112" }); - await ctx.verify("a form body", [ - same("the stored form", text(form.log, "requestBody"), "memo=%5BREDACTED%5D&name=plain&password=%5BREDACTED%5D&amex=%5BREDACTED%5D&bad=4111111111111112"), - same("the count", pick(form.log, "redactions", "requestBody"), 3), - ]); - const nested = await sendAndRead(ctx, "arrays", { method: "POST", path: "/fx/echo", headers: { "content-type": "application/json" }, body: JSON.stringify({ items: [{ password: "p", keep: "k" }], cards: ["4111 1111 1111 1111", "plain"], n: [4111111111111111], m: 1700000000000, nest: [[{ token: "t", ok: 1 }]] }) }); - await ctx.verify("arrays", [ - same("the stored body", parseJson(text(nested.log, "requestBody")), { items: [{ password: "[REDACTED]", keep: "k" }], cards: ["[REDACTED]", "plain"], n: ["[REDACTED]"], m: 1700000000000, nest: [[{ token: "[REDACTED]", ok: 1 }]] }), - same("the request count", pick(nested.log, "redactions", "requestBody"), 4), - same("the response is redacted the same way", pick(nested.log, "redactions", "responseBody"), 4), - same("the stored response", parseJson(text(nested.log, "responseBody")), parseJson(text(nested.log, "requestBody"))), - ]); - /** @type {Record} */ - let deep = { leaf: "x" }; - for (let level = 0; level < 40; level += 1) deep = { next: deep }; - const tall = await sendAndRead(ctx, "forty levels", { method: "POST", path: "/fx/echo", headers: { "content-type": "application/json" }, body: JSON.stringify(deep) }); - let walk = parseJson(text(tall.log, "requestBody")); - let levels = 0; - while (isRow(walk) && "next" in walk) { - walk = walk.next; - levels += 1; - } - await ctx.verify("a body nested deeper than 32 levels", [same("it stops at level 32 with a redaction", [levels, walk], [32, "[REDACTED]"])]); - const notJson = await sendAndRead(ctx, "text body", { method: "POST", path: "/fx/echo", headers: { "content-type": "text/plain" }, body: "token=abc" }); - const malformed = await sendAndRead(ctx, "malformed JSON", { method: "POST", path: "/fx/echo", headers: { "content-type": "application/json" }, body: '{"password":' }); - const bare = await sendAndRead(ctx, "bare string", { method: "POST", path: "/fx/echo", headers: { "content-type": "application/json" }, body: '"sk_live_abc"' }); - await ctx.verify("bodies that cannot be redacted by key", [ - same("text/plain is not stored", [pick(notJson.log, "requestBody"), pick(notJson.log, "requestBodyOmitted")], [null, "not-json"]), - same("malformed JSON is not stored", [pick(malformed.log, "requestBody"), pick(malformed.log, "requestBodyOmitted")], [null, "malformed"]), - same("a bare JSON string is not stored", [pick(bare.log, "requestBody"), pick(bare.log, "requestBodyOmitted")], [null, "not-json"]), - ]); - }, - }, -]; diff --git a/e2e/journeys/capture/shared.mjs b/e2e/journeys/capture/shared.mjs deleted file mode 100644 index ae7acb8..0000000 --- a/e2e/journeys/capture/shared.mjs +++ /dev/null @@ -1,5 +0,0 @@ - - -export const DEFAULT_HEADERS = ["content-type", "content-length", "user-agent", "accept", "accept-encoding", "host", "referer", "origin"]; -export const REDACTED_WORDS = ["password", "passwd", "secret", "authorization", "cookie", "apikey", "privatekey", "credential", "recoverycode", "backupcode", "cardnumber", "securitycode", "pincode", "token", "otp", "cvc", "cvv", "cvn", "iban", "pin", "pan", "ssn", "card"]; -export const JSON_HEADERS = { "content-type": "application/json", "user-agent": "verify-agent/1.0", "x-request-id": "req-capture-1", "x-webhook-signature": "sig-1", authorization: "Bearer top-secret", cookie: "sid=1", "proxy-authorization": "Basic cHJveHk6cHc=", "set-cookie": "sid=2", "x-api-key": "k-1", origin: "https://client.example.org", referer: "https://client.example.org/page" }; diff --git a/tests/access.journeys.test.ts b/tests/access.journeys.test.ts new file mode 100644 index 0000000..f3cd5a8 --- /dev/null +++ b/tests/access.journeys.test.ts @@ -0,0 +1,222 @@ +import { describe, expect } from "vitest"; +import { z } from "zod"; +import { type Dms, test } from "./helpers/dms"; + +const ROUTES: [string, string][] = [ + ["GET", "/overview/health"], + ["GET", "/overview/traffic"], + ["GET", "/overview/classes"], + ["GET", "/overview/error-share"], + ["GET", "/overview/attention"], + ["GET", "/overview/slowest"], + ["GET", "/overview/activity"], + ["GET", "/logs"], + ["GET", "/logs/recent"], + ["GET", "/logs/routes?route=GET%20%2Ffx%2Fok"], + ["GET", "/logs/tester?requestId=x"], + ["GET", "/logs/live"], + ["GET", "/logs/missing"], + ["GET", "/logs/recent/missing"], + ["GET", "/logs/tester/missing"], + ["GET", "/logs/routes/missing"], + ["GET", "/routes/list"], + ["GET", "/routes/detail?route=GET%20%2Ffx%2Fok"], + ["GET", "/routes/statistics/activity?route=GET%20%2Ffx%2Fok"], + ["GET", "/routes/statistics/summary?route=GET%20%2Ffx%2Fok"], + ["GET", "/routes/statistics/status?route=GET%20%2Ffx%2Fok"], + ["GET", "/routes/statistics/latency-chart?route=GET%20%2Ffx%2Fok"], + ["GET", "/routes/statistics/errors?route=GET%20%2Ffx%2Fok"], + ["GET", "/settings"], + ["POST", "/settings"], + ["GET", "/settings/retention-impact?field=requestLogRetentionDays&days=30"], + ["GET", "/settings/summary"], + ["GET", "/tester/source"], +]; + +const BLOCK_OF = new Map([ + ["GET /overview/health", "modules.api.monitor.overview.health"], + [ + "GET /overview/traffic", + "modules.api.monitor.overview.dashboard.traffic.chart", + ], + [ + "GET /overview/classes", + "modules.api.monitor.overview.dashboard.traffic.classes", + ], + [ + "GET /overview/error-share", + "modules.api.monitor.overview.dashboard.health.errorShare", + ], + [ + "GET /overview/attention", + "modules.api.monitor.overview.dashboard.health.attention", + ], + [ + "GET /overview/slowest", + "modules.api.monitor.overview.dashboard.health.slowest", + ], + [ + "GET /overview/activity", + "modules.api.monitor.overview.dashboard.recent.activity", + ], + ["GET /logs", "modules.api.monitor.logs.requests"], + [ + "GET /logs/recent", + "modules.api.monitor.overview.dashboard.recent.requests.table", + ], + [ + "GET /logs/routes", + "modules.api.explore.routes.explorer.tabs.statistics.recent.requests.table", + ], + ["GET /logs/tester", "modules.api.explore.routes.explorer.tabs.tester"], + ["GET /logs/live", "modules.api.monitor.logs.live"], + ["GET /logs/missing", "modules.api.monitor.logs.requests"], + [ + "GET /logs/recent/missing", + "modules.api.monitor.overview.dashboard.recent.requests.table", + ], + [ + "GET /logs/tester/missing", + "modules.api.explore.routes.explorer.tabs.tester", + ], + [ + "GET /logs/routes/missing", + "modules.api.explore.routes.explorer.tabs.statistics.recent.requests.table", + ], + ["GET /routes/list", "modules.api.explore.routes.explorer.tree"], + ["GET /routes/detail", "modules.api.explore.routes.explorer"], + [ + "GET /routes/statistics/activity", + "modules.api.explore.routes.explorer.tabs.statistics", + ], + [ + "GET /routes/statistics/summary", + "modules.api.explore.routes.explorer.tabs.statistics.grid.summary.summary", + ], + [ + "GET /routes/statistics/status", + "modules.api.explore.routes.explorer.tabs.statistics.grid.charts.status", + ], + [ + "GET /routes/statistics/latency-chart", + "modules.api.explore.routes.explorer.tabs.statistics.grid.charts.latency", + ], + [ + "GET /routes/statistics/errors", + "modules.api.explore.routes.explorer.tabs.statistics.grid.detail.errors", + ], + ["GET /settings", "modules.api.configure.settings.form"], + ["POST /settings", "modules.api.configure.settings.form"], + ["GET /settings/retention-impact", "modules.api.configure.settings.form"], + ["GET /tester/source", "modules.api.explore.routes.explorer.tabs.tester"], +]); + +const refusalSchema = z + .union([ + z.string(), + z + .object({ message: z.string().optional(), error: z.string().optional() }) + .transform((body) => body.message || body.error || ""), + ]) + .catch(""); + +interface Sweep { + route: string; + status: number; + message: string; +} + +/** Calls every route of the console as an account. */ +async function sweep(dms: Dms, as: string): Promise { + const seen: Sweep[] = []; + for (const [method, path] of ROUTES) { + const answer = await dms.mon(path, { + method, + as, + body: method === "POST" ? {} : undefined, + }); + seen.push({ + route: `${method} ${path.split("?")[0]}`, + status: answer.status, + message: refusalSchema.parse(answer.body), + }); + } + return seen; +} + +function permissionNamedFor(refused: Sweep[], route: string): string { + const message = refused.find((entry) => entry.route === route)?.message; + return /missing permission ([\w.-]+)/.exec(message ?? "")?.[1] ?? ""; +} + +describe("access", () => { + test("every-console-route-needs-a-session-and-the-block-it-serves", async ({ + dms, + }) => { + await dms.traffic([{ path: "/fx/ok", count: 1 }]); + await dms.member("plain", []); + const anonymous = await sweep(dms, "anon"); + const plain = await sweep(dms, "plain"); + const owner = await sweep(dms, "admin"); + + for (const entry of anonymous) + expect.soft(entry.status, `${entry.route} is 401`).toBe(401); + for (const entry of plain) + expect + .soft( + entry.message, + `${entry.route} refuses with its own block's permission`, + ) + .toBe( + entry.route === "GET /settings/summary" + ? "Owner-only" + : `Forbidden: missing permission ${BLOCK_OF.get(entry.route)}`, + ); + for (const entry of owner) + expect + .soft( + [401, 403].includes(entry.status), + `${entry.route} passes the guard`, + ) + .toBe(false); + }); + + test("the-console-is-the-owners-and-a-role-holding-a-block-is-refused", async ({ + dms, + }) => { + await dms.traffic([{ path: "/fx/ok", count: 1 }]); + await dms.member("plain", []); + const refused = await sweep(dms, "plain"); + const logsPermission = permissionNamedFor(refused, "GET /logs"); + const testerPermission = permissionNamedFor(refused, "GET /tester/source"); + expect + .soft( + logsPermission.startsWith("modules.api."), + "the logs table has its own permission", + ) + .toBe(true); + expect + .soft( + testerPermission.startsWith("modules.api.") && + testerPermission !== logsPermission, + "the Tester has its own permission", + ) + .toBe(true); + + await dms.member("reader", [logsPermission]); + await dms.member("tester", [testerPermission]); + for (const account of ["reader", "tester"]) + for (const entry of await sweep(dms, account)) + expect + .soft(entry.status, `${account}: ${entry.route} is refused`) + .toBe(403); + const owner = await sweep(dms, "admin"); + for (const route of ["GET /logs", "GET /tester/source"]) + expect + .soft( + owner.find((entry) => entry.route === route)?.status, + `${route} opens`, + ) + .toBe(200); + }); +}); From ebdc0696f10f8af8b626f4d6d99fc563e2e0fc99 Mon Sep 17 00:00:00 2001 From: Antony Rizzitelli Date: Thu, 19 Nov 2026 08:48:23 +0000 Subject: [PATCH 07/16] test(tester): run the tester token journey as an integration test --- e2e/journeys/tester.mjs | 33 +--------------- tests/tester.journeys.test.ts | 71 +++++++++++++++++++++++++++++++++++ 2 files changed, 72 insertions(+), 32 deletions(-) create mode 100644 tests/tester.journeys.test.ts diff --git a/e2e/journeys/tester.mjs b/e2e/journeys/tester.mjs index dca1302..13918e7 100644 --- a/e2e/journeys/tester.mjs +++ b/e2e/journeys/tester.mjs @@ -1,12 +1,10 @@ import { guarded } from "../guard.mjs"; import { forwarding } from "./tester/forwarding.mjs"; -import { saveSettings, check, list, mon, num, pick, raw, same, settled, text, traffic, withConfig, openPage } from "../api.mjs"; +import { saveSettings, check, list, mon, num, pick, raw, same, settled, text, openPage } from "../api.mjs"; /** @typedef {import("../api.mjs").Ctx} Ctx */ /** @typedef {import("../api.mjs").Tab} Tab */ -const TESTER_HEADER = "x-antelopejs-api-source"; - /** * Opens the Tester of a route in a signed-in tab. * @@ -53,35 +51,6 @@ export const journeys = guarded([ ]); }, }, - { - id: "a-secret-keeps-the-testers-token-valid-across-restarts", - title: "with testerSourceSecret set the Tester's token survives a restart and still marks the call; without it a token issued before the restart is no longer recognised, and its call counts as traffic", - entryPoint: "GET /api/monitoring/tester/source then a call with the header", - async run(/** @type {Ctx} */ ctx) { - const tokenAnswer = await mon(ctx, "ask a token", "/tester/source"); - const expiresAt = Date.parse(text(tokenAnswer.body, "expiresAt")); - await ctx.verify("the token", [ - same("the header name", text(tokenAnswer.body, "header"), TESTER_HEADER), - check("valid 15 minutes", Math.abs(expiresAt - Date.now() - 15 * 60_000) < 60_000, text(tokenAnswer.body, "expiresAt")), - check("a value is given", text(tokenAnswer.body, "value").length > 10, "value"), - ]); - /** @type {string} */ - let signedWithSecret = ""; - await withConfig(ctx, { testerSourceSecret: "one-secret-for-every-instance" }, async () => { - signedWithSecret = text((await mon(ctx, "a token under the secret", "/tester/source")).body, "value"); - }); - await withConfig(ctx, { testerSourceSecret: "one-secret-for-every-instance" }, async () => { - await raw(ctx, "a call with the token of before the restart", { path: "/fx/ok", headers: { [TESTER_HEADER]: signedWithSecret } }); - const marked = await settled(ctx, "the logs", "/logs?limit=5", (body) => list(body, "results").length === 1); - await ctx.verify("under the same secret", [same("the call is the Tester's", text(list(marked.body, "results")[0], "source"), "tester")]); - }); - await raw(ctx, "a call with a token signed under the secret, secret now unset", { path: "/fx/ok", headers: { [TESTER_HEADER]: signedWithSecret } }); - const logs = await settled(ctx, "the logs", "/logs?limit=5", (body) => list(body, "results").length === 2); - const newest = list(logs.body, "results")[0]; - await ctx.verify("without the secret", [same("the call is no longer the Tester's", pick(newest, "source"), null)]); - await traffic(ctx, "a plain call", [{ path: "/fx/ok", count: 1 }]); - }, - }, { id: "the-tester-sends-the-form-and-keeps-twenty-five-calls", title: "Send calls the route without credentials until Send with my session is on, a header added in the form reaches the handler, Ctrl Enter sends too, and the History keeps the last 25 calls with their credential note", diff --git a/tests/tester.journeys.test.ts b/tests/tester.journeys.test.ts new file mode 100644 index 0000000..ae9ed3b --- /dev/null +++ b/tests/tester.journeys.test.ts @@ -0,0 +1,71 @@ +import { describe, expect } from "vitest"; +import { z } from "zod"; +import { test } from "./helpers/dms"; +import { resultsOf } from "./helpers/json"; + +const TESTER_HEADER = "x-antelopejs-api-source"; +const SECRET = { testerSourceSecret: "one-secret-for-every-instance" }; +const FIFTEEN_MINUTES_MS = 15 * 60_000; + +const tokenSchema = z.object({ + header: z.string(), + expiresAt: z.string(), + value: z.string(), +}); +const sourceSchema = z.object({ source: z.string().nullable() }); + +describe("tester", () => { + test("a-secret-keeps-the-testers-token-valid-across-restarts", async ({ + dms, + }) => { + const issued = tokenSchema.parse((await dms.mon("/tester/source")).body); + expect.soft(issued.header, "the header name").toBe(TESTER_HEADER); + expect + .soft( + Math.abs( + Date.parse(issued.expiresAt) - Date.now() - FIFTEEN_MINUTES_MS, + ) < 60_000, + "valid 15 minutes", + ) + .toBe(true); + expect.soft(issued.value.length > 10, "a value is given").toBe(true); + + let signedWithSecret = ""; + await dms.withConfig(SECRET, async () => { + signedWithSecret = tokenSchema.parse( + (await dms.mon("/tester/source")).body, + ).value; + }); + await dms.withConfig(SECRET, async () => { + await dms.raw({ + path: "/fx/ok", + headers: { [TESTER_HEADER]: signedWithSecret }, + }); + const marked = await dms.settled( + "/logs?limit=5", + (body) => resultsOf(body, sourceSchema).length === 1, + ); + expect + .soft( + resultsOf(marked.body, sourceSchema)[0]?.source, + "under the same secret the call is the Tester's", + ) + .toBe("tester"); + }); + + await dms.raw({ + path: "/fx/ok", + headers: { [TESTER_HEADER]: signedWithSecret }, + }); + const logs = await dms.settled( + "/logs?limit=5", + (body) => resultsOf(body, sourceSchema).length === 2, + ); + expect + .soft( + resultsOf(logs.body, sourceSchema)[0]?.source, + "without the secret the call is no longer the Tester's", + ) + .toBeNull(); + }); +}); From df8a7c3a50ca7f56bb33b2b1bb847652ad87c86c Mon Sep 17 00:00:00 2001 From: Antony Rizzitelli Date: Thu, 19 Nov 2026 08:48:28 +0000 Subject: [PATCH 08/16] test(routes): run the routes contract journey as an integration test --- e2e/journeys/routes/documentation.mjs | 38 +-------- tests/helpers/json.ts | 2 +- tests/routes.journeys.test.ts | 111 ++++++++++++++++++++++++++ 3 files changed, 113 insertions(+), 38 deletions(-) create mode 100644 tests/routes.journeys.test.ts diff --git a/e2e/journeys/routes/documentation.mjs b/e2e/journeys/routes/documentation.mjs index 74e94f5..24a4541 100644 --- a/e2e/journeys/routes/documentation.mjs +++ b/e2e/journeys/routes/documentation.mjs @@ -1,5 +1,4 @@ -import { seedLogs } from "../../seed.mjs"; -import { list, mon, num, pick, raw, same, settled, text, withConfig, openPage } from "../../api.mjs"; +import { list, num, pick, raw, same, settled, text, withConfig, openPage } from "../../api.mjs"; import { named, detail } from "./shared.mjs"; /** @typedef {import("../../api.mjs").Ctx} Ctx */ @@ -86,39 +85,4 @@ export const documentation = [ }); }, }, - { - id: "the-contract-reads-seven-days-of-real-calls-and-never-the-testers", - title: "a call 6 days old shapes the documentation, one 8 days old does not, and a call the Tester made leaves its totals, fields, statuses and examples unchanged", - entryPoint: "GET /api/monitoring/routes/detail", - async run(/** @type {Ctx} */ ctx) { - const ref = "POST /fx/orders/:shop/lines"; - const uri = "/fx/orders/:shop/lines"; - await seedLogs(ctx, "8 days old", { daysAgo: 8, count: 1, method: "POST", uri, statusCode: 201, requestBody: JSON.stringify({ sku: "OLD", legacy: true }), responseBody: JSON.stringify({ shop: "old" }) }); - await seedLogs(ctx, "6 days old", { daysAgo: 6, count: 1, method: "POST", uri, statusCode: 201, requestBody: JSON.stringify({ sku: "R6", recent: true }), responseBody: JSON.stringify({ shop: "recent" }) }); - await raw(ctx, "a real call", { method: "POST", path: "/fx/orders/shop1/lines?limit=5", headers: { "content-type": "application/json" }, body: JSON.stringify({ sku: "A1", quantity: 2 }) }); - const before = await settled(ctx, "the contract", `/routes/detail?${named(ref)}`, (body) => num(body, "contract", "total") === 2); - const token = text((await mon(ctx, "ask a Tester token", "/tester/source")).body, "value"); - await raw(ctx, "a Tester call with its own fields", { method: "POST", path: "/fx/orders/shop9/lines?viaTester=1", headers: { "content-type": "application/json", "x-antelopejs-api-source": token }, body: JSON.stringify({ sku: "T9", quantity: 9, viaTester: true }) }); - await settled(ctx, "the Tester call is stored", "/logs?limit=50", (body) => list(body, "results").some((row) => text(row, "source") === "tester")); - await new Promise((resolve) => setTimeout(resolve, 16_000)); - const after = await mon(ctx, "the contract after the Tester call", `/routes/detail?${named(ref)}`); - const summarize = (/** @type {unknown} */ body) => ({ - total: num(body, "contract", "total"), - fields: list(body, "contract", "bodyFields").map((entry) => [text(entry, "name"), pick(entry, "required")]).toSorted((left, right) => String(left[0]).localeCompare(String(right[0]))), - statuses: list(body, "contract", "statuses").map((entry) => [num(entry, "status"), num(entry, "count")]), - query: pick(body, "contract", "queryParams"), - example: pick(body, "contract", "exampleBody"), - }); - await ctx.verify("seven days", [ - same("only the calls of the last 7 days count", summarize(before.body).total, 2), - same("the 6 day old call's field is documented, the 8 day old one's is not, and a field missing from a call is not required", summarize(before.body).fields, [["quantity", false], ["recent", false], ["sku", true]]), - same("the statuses seen", summarize(before.body).statuses, [[201, 2]]), - ]); - await ctx.verify("the Tester", [ - same("the contract is the same after a Tester call", summarize(after.body), summarize(before.body)), - same("the Tester's field is not documented", summarize(after.body).fields.map((entry) => entry[0]).includes("viaTester"), false), - same("the Tester's query parameter is not documented", summarize(after.body).query, ["limit"]), - ]); - }, - }, ]; diff --git a/tests/helpers/json.ts b/tests/helpers/json.ts index 3e541f0..5c9ceba 100644 --- a/tests/helpers/json.ts +++ b/tests/helpers/json.ts @@ -9,7 +9,7 @@ export type Json = | Json[] | { [key: string]: Json }; -const jsonSchema: z.ZodType = z.lazy(() => +export const jsonSchema: z.ZodType = z.lazy(() => z.union([ z.string(), z.number(), diff --git a/tests/routes.journeys.test.ts b/tests/routes.journeys.test.ts new file mode 100644 index 0000000..1123f07 --- /dev/null +++ b/tests/routes.journeys.test.ts @@ -0,0 +1,111 @@ +import { describe, expect } from "vitest"; +import { z } from "zod"; +import { test } from "./helpers/dms"; +import { type Json, jsonSchema, resultsOf } from "./helpers/json"; +import { seedLogs } from "./helpers/seed"; + +const REF = "POST /fx/orders/:shop/lines"; +const URI = "/fx/orders/:shop/lines"; +const DETAIL = `/routes/detail?route=${encodeURIComponent(REF)}`; +const TESTER_SETTLE_MS = 16_000; + +const contractSchema = z.object({ + contract: z.object({ + total: z.number(), + bodyFields: z.array(z.object({ name: z.string(), required: z.boolean() })), + statuses: z.array(z.object({ status: z.number(), count: z.number() })), + queryParams: jsonSchema, + exampleBody: jsonSchema.optional(), + }), +}); + +const sourceSchema = z.object({ source: z.string().nullable() }); +const tokenSchema = z.object({ value: z.string() }); + +/** The contract's totals, body fields, statuses and examples, in a fixed order. */ +function summarize(body: Json) { + const { contract } = contractSchema.parse(body); + return { + total: contract.total, + fields: contract.bodyFields + .map((field) => [field.name, field.required]) + .sort((left, right) => String(left[0]).localeCompare(String(right[0]))), + statuses: contract.statuses.map((entry) => [entry.status, entry.count]), + query: contract.queryParams, + example: contract.exampleBody, + }; +} + +function seeded(daysAgo: number, sku: string, flag: string, shop: string) { + return { + daysAgo, + count: 1, + method: "POST", + uri: URI, + statusCode: 201, + requestBody: JSON.stringify({ sku, [flag]: true }), + responseBody: JSON.stringify({ shop }), + }; +} + +describe("routes", () => { + test("the-contract-reads-seven-days-of-real-calls-and-never-the-testers", async ({ + dms, + }) => { + await seedLogs(dms, seeded(8, "OLD", "legacy", "old")); + await seedLogs(dms, seeded(6, "R6", "recent", "recent")); + await dms.raw({ + method: "POST", + path: "/fx/orders/shop1/lines?limit=5", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ sku: "A1", quantity: 2 }), + }); + const before = await dms.settled( + DETAIL, + (body) => contractSchema.safeParse(body).data?.contract.total === 2, + ); + const token = tokenSchema.parse((await dms.mon("/tester/source")).body); + await dms.raw({ + method: "POST", + path: "/fx/orders/shop9/lines?viaTester=1", + headers: { + "content-type": "application/json", + "x-antelopejs-api-source": token.value, + }, + body: JSON.stringify({ sku: "T9", quantity: 9, viaTester: true }), + }); + await dms.settled("/logs?limit=50", (body) => + resultsOf(body, sourceSchema).some((row) => row.source === "tester"), + ); + await new Promise((done) => setTimeout(done, TESTER_SETTLE_MS)); + const after = await dms.mon(DETAIL); + + const seen = summarize(before.body); + expect.soft(seen.total, "only the calls of the last 7 days count").toBe(2); + expect + .soft( + seen.fields, + "the 6 day old call's field is documented, the 8 day old one's is not, and a field missing from a call is not required", + ) + .toEqual([ + ["quantity", false], + ["recent", false], + ["sku", true], + ]); + expect.soft(seen.statuses, "the statuses seen").toEqual([[201, 2]]); + + const later = summarize(after.body); + expect + .soft(later, "the contract is the same after a Tester call") + .toEqual(seen); + expect + .soft( + later.fields.map((entry) => entry[0]).includes("viaTester"), + "the Tester's field is not documented", + ) + .toBe(false); + expect + .soft(later.query, "the Tester's query parameter is not documented") + .toEqual(["limit"]); + }); +}); From 804e47810c9a56030433bd5410d65fb2f76e52a5 Mon Sep 17 00:00:00 2001 From: Antony Rizzitelli Date: Thu, 19 Nov 2026 08:48:33 +0000 Subject: [PATCH 09/16] test(capture): run the capture redaction journeys as integration tests --- e2e/journeys/capture.mjs | 3 +- tests/capture-redaction.journeys.test.ts | 482 +++++++++++++++++++++++ 2 files changed, 483 insertions(+), 2 deletions(-) create mode 100644 tests/capture-redaction.journeys.test.ts diff --git a/e2e/journeys/capture.mjs b/e2e/journeys/capture.mjs index 5b9a94c..e5cb694 100644 --- a/e2e/journeys/capture.mjs +++ b/e2e/journeys/capture.mjs @@ -1,4 +1,3 @@ import { guarded } from "../guard.mjs"; -import { redaction } from "./capture/redaction.mjs"; -export const journeys = guarded(redaction); +export const journeys = guarded([]); diff --git a/tests/capture-redaction.journeys.test.ts b/tests/capture-redaction.journeys.test.ts new file mode 100644 index 0000000..0f328d2 --- /dev/null +++ b/tests/capture-redaction.journeys.test.ts @@ -0,0 +1,482 @@ +import { describe, expect } from "vitest"; +import { z } from "zod"; +import { type Dms, test } from "./helpers/dms"; +import { type Json, parseBody } from "./helpers/json"; + +const DEFAULT_HEADERS = [ + "content-type", + "content-length", + "user-agent", + "accept", + "accept-encoding", + "host", + "referer", + "origin", +]; + +const REDACTED_WORDS = [ + "password", + "passwd", + "secret", + "authorization", + "cookie", + "apikey", + "privatekey", + "credential", + "recoverycode", + "backupcode", + "cardnumber", + "securitycode", + "pincode", + "token", + "otp", + "cvc", + "cvv", + "cvn", + "iban", + "pin", + "pan", + "ssn", + "card", +]; + +const JSON_HEADERS = { + "content-type": "application/json", + "user-agent": "verify-agent/1.0", + "x-request-id": "req-capture-1", + "x-webhook-signature": "sig-1", + authorization: "Bearer top-secret", + cookie: "sid=1", + "proxy-authorization": "Basic cHJveHk6cHc=", + "set-cookie": "sid=2", + "x-api-key": "k-1", + origin: "https://client.example.org", + referer: "https://client.example.org/page", +}; + +const REDACTED = "[REDACTED]"; + +const DEFAULT_STORED = [ + "accept", + "accept-encoding", + "content-type", + "host", + "origin", + "referer", + "user-agent", +]; + +const capSchema = z.object({ + requestBody: z.string().nullable(), + requestBodyTruncated: z.boolean(), + requestBodySize: z.number().nullable(), + responseBody: z.string().nullable(), + responseBodyTruncated: z.boolean(), +}); + +const bodiesSchema = z.object({ + requestBody: z.string().nullish(), + responseBody: z.string().nullish(), +}); + +interface Depth { + levels: number; + leaf: string; +} + +const depthSchema: z.ZodType = z.lazy(() => + z.union([ + z.string().transform((leaf) => ({ levels: 0, leaf })), + z + .object({ next: depthSchema }) + .transform(({ next }) => ({ levels: next.levels + 1, leaf: next.leaf })), + ]), +); + +/** The body a stored request log holds, parsed as the module stored it; `{}` when it stored none. */ +function storedBody(log: Json, field: "requestBody" | "responseBody"): Json { + const stored = bodiesSchema.parse(log)[field]; + return stored ? parseBody(stored) : {}; +} + +function storedHeaders(log: Json): string[] { + return Object.keys( + z.object({ requestHeaders: z.record(z.string()) }).parse(log) + .requestHeaders, + ).sort(); +} + +function echo(dms: Dms, body: string, contentType = "application/json") { + return dms.sendAndRead({ + method: "POST", + path: "/fx/echo", + headers: { "content-type": contentType }, + body, + }); +} + +const CREDENTIAL_HEADERS = [ + "cookie", + "set-cookie", + "x-api-key", + "proxy-authorization", + "x-antelopejs-api-source", +]; + +/** What the settings route answers to each header offered alone as the capture allow-list. */ +async function offeredAlone( + dms: Dms, + headers: string[], +): Promise> { + const statuses = new Map(); + for (const header of headers) { + const answer = await dms.saveSettings({ + requestLogCaptureHeaders: [header], + }); + statuses.set(header, answer.status); + } + return statuses; +} + +const KEPT_KEYS = [ + "spin", + "footprint", + "isNotPaid", + "company", + "cardBrand", + "tokenizer", + "pinned", + "panda", + "cvcs", +]; + +const SECRET_KEYS = [ + "clientSecret", + "x-api-key", + "accessToken", + "access_token", + "accesstoken", + "refreshtoken", + "userPin", + "creditCard", + "card_cvc2", + "myPassword", + "iban", + "totp", + "X-Authorization", +]; + +const CARD_QUERY = + "/fx/ok?memo=4111111111111111&dash=4111-1111-1111-1111&sp=4111%201111%201111%201111&bad=4111111111111112&ms=1700000000000&amex=378282246310005&mc=5555555555554444&rep=1&rep=5555555555554444&token=abc&page=2&text=order%204111%201111%201111%201111%20paid"; + +const QUERY_VALUES: [string, string, string][] = [ + ["a plain card number", "memo", REDACTED], + ["a card number with dashes", "dash", REDACTED], + ["a card number with spaces", "sp", REDACTED], + ["an American Express number", "amex", REDACTED], + ["a Mastercard number", "mc", REDACTED], + ["16 digits that fail the Luhn check are kept", "bad", "4111111111111112"], + ["an epoch in milliseconds is kept", "ms", "1700000000000"], + ["a credential key", "token", REDACTED], + ["a plain parameter", "page", "2"], + ["a card number inside a sentence", "text", `order ${REDACTED} paid`], +]; + +const FORM_BODY = + "memo=4111111111111111&name=plain&password=x&amex=378282246310005&bad=4111111111111112"; + +const FORM_STORED = + "memo=%5BREDACTED%5D&name=plain&password=%5BREDACTED%5D&amex=%5BREDACTED%5D&bad=4111111111111112"; + +const ARRAYS_BODY = JSON.stringify({ + items: [{ password: "p", keep: "k" }], + cards: ["4111 1111 1111 1111", "plain"], + n: [4111111111111111], + m: 1700000000000, + nest: [[{ token: "t", ok: 1 }]], +}); + +const ARRAYS_STORED = { + items: [{ password: REDACTED, keep: "k" }], + cards: [REDACTED, "plain"], + n: [REDACTED], + m: 1700000000000, + nest: [[{ token: REDACTED, ok: 1 }]], +}; + +function nestedDeeperThanTheLimit(): string { + let deep: Json = { leaf: "x" }; + for (let level = 0; level < 40; level += 1) deep = { next: deep }; + return JSON.stringify(deep); +} + +describe("capture", () => { + test("only-the-listed-headers-are-stored-and-secret-ones-never", async ({ + dms, + }) => { + const send = () => + dms.sendAndRead({ path: "/fx/ok", headers: JSON_HEADERS }); + const before = await send(); + expect + .soft(storedHeaders(before.log), "the stored request headers") + .toEqual(DEFAULT_STORED); + expect + .soft(before.log, "a header outside the list is absent") + .not.toHaveProperty(["requestHeaders", "x-webhook-signature"]); + expect + .soft((await dms.mon("/settings")).body, "the allow-list in force") + .toMatchObject({ requestLogCaptureHeaders: DEFAULT_HEADERS }); + + const refusedList = await dms.saveSettings({ + requestLogCaptureHeaders: ["x-webhook-signature", "authorization"], + }); + expect + .soft(refusedList.status, "a secret header is refused: HTTP 400") + .toBe(400); + for (const [header, status] of await offeredAlone(dms, CREDENTIAL_HEADERS)) + expect.soft(status, `${header} is refused`).toBe(400); + + const saved = await dms.saveSettings({ + requestLogCaptureHeaders: [ + "X-Webhook-Signature", + "x-webhook-signature", + "user-agent", + ], + }); + expect.soft(saved.status, "HTTP 200").toBe(200); + expect.soft(saved.body, "stored lower case without repeats").toMatchObject({ + requestLogCaptureHeaders: ["x-webhook-signature", "user-agent"], + }); + const after = await send(); + expect + .soft(storedHeaders(after.log), "the stored request headers") + .toEqual(["user-agent", "x-webhook-signature"]); + expect + .soft(after.log, "the signature value") + .toHaveProperty(["requestHeaders", "x-webhook-signature"], "sig-1"); + expect + .soft(after.log, "authorization is never stored") + .not.toHaveProperty(["requestHeaders", "authorization"]); + + await dms.saveSettings({ requestLogCaptureHeaders: [] }); + expect + .soft(storedHeaders((await send()).log), "no request header is stored") + .toEqual([]); + expect + .soft((await dms.mon("/settings")).body, "the override stays explicit") + .toMatchObject({ requestLogCaptureHeaders: [] }); + }); +}); + +describe("capture", () => { + test("values-under-secret-looking-keys-are-redacted-before-storage", async ({ + dms, + }) => { + const words = z + .object({ redactedKeys: z.array(z.string()) }) + .parse((await dms.mon("/settings")).body).redactedKeys; + const body = { + ...Object.fromEntries(words.map((word) => [word, `value-of-${word}`])), + name: "plain", + nested: { token: "deep", keep: "kept" }, + }; + const query = words.map((word) => `${word}=q-${word}`).join("&"); + const sent = await dms.sendAndRead({ + method: "POST", + path: `/fx/echo?${query}&page=2`, + headers: { "content-type": "application/json" }, + body: JSON.stringify(body), + }); + const stored = storedBody(sent.log, "requestBody"); + + expect + .soft(words, "it is the 23 words the settings page shows") + .toEqual(REDACTED_WORDS); + for (const word of words) + expect + .soft(stored, `${word} in the request body`) + .toHaveProperty([word], REDACTED); + expect + .soft(stored, "a plain key keeps its value") + .toHaveProperty(["name"], "plain"); + expect + .soft(stored, "a nested secret is redacted and its sibling kept") + .toHaveProperty(["nested"], { token: REDACTED, keep: "kept" }); + expect + .soft( + storedBody(sent.log, "responseBody"), + "the response body echoes it redacted too", + ) + .toHaveProperty(["password"], REDACTED); + for (const word of words) + expect + .soft(sent.log, `${word} in the query`) + .toHaveProperty(["query", word], REDACTED); + expect + .soft(sent.log, "a plain parameter keeps its value") + .toHaveProperty(["query", "page"], "2"); + expect + .soft(sent.log, "the count of query") + .toHaveProperty(["redactions", "query"], words.length); + expect + .soft(sent.log, "the count of request body") + .toHaveProperty(["redactions", "requestBody"], words.length + 1); + expect + .soft(sent.log, "the count of response body") + .toHaveProperty(["redactions", "responseBody"], words.length + 1); + }); +}); + +describe("capture", () => { + test("bodies-above-the-cap-are-stored-truncated-and-flagged", async ({ + dms, + }) => { + await dms.saveSettings({ requestLogMaxBodyKb: 1 }); + const exact = JSON.stringify({ a: "x".repeat(1024 - 8) }); + const over = JSON.stringify({ a: "x".repeat(1024 - 7) }); + const atCap = capSchema.parse((await echo(dms, exact)).log); + const cut = capSchema.parse((await echo(dms, over)).log); + + expect + .soft( + [ + Buffer.byteLength(exact), + atCap.requestBody === exact, + atCap.requestBodyTruncated, + ], + "the body at the cap is whole", + ) + .toEqual([1024, true, false]); + expect + .soft(cut.requestBodyTruncated, "the body above the cap is flagged") + .toBe(true); + expect + .soft(cut.requestBody, "it keeps the first 1024 bytes") + .toBe(over.slice(0, 1024)); + expect + .soft( + [Buffer.byteLength(over), cut.requestBodySize], + "it records the whole size", + ) + .toEqual([1025, 1025]); + expect + .soft( + [cut.responseBodyTruncated, cut.responseBody?.length], + "the echoed response is cut too", + ) + .toEqual([true, 1024]); + + await dms.saveSettings({ requestLogMaxBodyKb: 0 }); + const off = (await echo(dms, exact)).log; + expect + .soft(off, "no request body is stored") + .toMatchObject({ requestBody: null }); + expect + .soft(off, "no response body is stored") + .toMatchObject({ responseBody: null }); + expect + .soft(off, "the status is still stored") + .toMatchObject({ statusCode: 201 }); + }); +}); + +describe("capture", () => { + test("a-key-is-secret-by-the-words-it-is-made-of", async ({ dms }) => { + const body = { + ...Object.fromEntries(SECRET_KEYS.map((key) => [key, `v-${key}`])), + ...Object.fromEntries(KEPT_KEYS.map((key) => [key, `v-${key}`])), + memo: "4111 1111 1111 1111", + ref: "4111111111111111", + short: "411111111111", + note: "order 12345", + }; + const stored = storedBody( + (await echo(dms, JSON.stringify(body))).log, + "requestBody", + ); + + for (const key of SECRET_KEYS) + expect.soft(stored, `${key} is redacted`).toHaveProperty([key], REDACTED); + for (const key of KEPT_KEYS) + expect + .soft(stored, `${key} keeps its value`) + .toHaveProperty([key], `v-${key}`); + expect + .soft(stored, "a spaced card number is redacted whatever its key") + .toHaveProperty(["memo"], REDACTED); + expect + .soft(stored, "a plain card number is redacted") + .toHaveProperty(["ref"], REDACTED); + expect + .soft(stored, "twelve digits are not a card number") + .toHaveProperty(["short"], "411111111111"); + expect + .soft(stored, "other text is kept") + .toHaveProperty(["note"], "order 12345"); + }); +}); + +describe("capture", () => { + test("secrets-and-card-numbers-are-redacted-wherever-they-sit", async ({ + dms, + }) => { + const query = await dms.sendAndRead({ path: CARD_QUERY }); + for (const [name, key, value] of QUERY_VALUES) + expect.soft(query.log, name).toHaveProperty(["query", key], value); + expect + .soft( + query.log, + "a repeated parameter keeps its plain value and redacts its card", + ) + .toHaveProperty(["query", "rep"], ["1", REDACTED]); + expect + .soft(query.log, "the count: memo, dash, sp, amex, mc, rep, token, text") + .toHaveProperty(["redactions", "query"], 8); + + const form = await echo( + dms, + FORM_BODY, + "application/x-www-form-urlencoded", + ); + expect + .soft(form.log, "the stored form") + .toHaveProperty(["requestBody"], FORM_STORED); + expect + .soft(form.log, "the count") + .toHaveProperty(["redactions", "requestBody"], 3); + + const nested = await echo(dms, ARRAYS_BODY); + const nestedBody = storedBody(nested.log, "requestBody"); + expect.soft(nestedBody, "the stored body").toEqual(ARRAYS_STORED); + expect + .soft(nested.log, "the request count") + .toHaveProperty(["redactions", "requestBody"], 4); + expect + .soft(nested.log, "the response is redacted the same way") + .toHaveProperty(["redactions", "responseBody"], 4); + expect + .soft(storedBody(nested.log, "responseBody"), "the stored response") + .toEqual(nestedBody); + + const tall = await echo(dms, nestedDeeperThanTheLimit()); + expect + .soft( + depthSchema.parse(storedBody(tall.log, "requestBody")), + "it stops at level 32 with a redaction", + ) + .toEqual({ levels: 32, leaf: REDACTED }); + + const notJson = await echo(dms, "token=abc", "text/plain"); + const malformed = await echo(dms, '{"password":'); + const bare = await echo(dms, '"sk_live_abc"'); + expect + .soft(notJson.log, "text/plain is not stored") + .toMatchObject({ requestBody: null, requestBodyOmitted: "not-json" }); + expect + .soft(malformed.log, "malformed JSON is not stored") + .toMatchObject({ requestBody: null, requestBodyOmitted: "malformed" }); + expect + .soft(bare.log, "a bare JSON string is not stored") + .toMatchObject({ requestBody: null, requestBodyOmitted: "not-json" }); + }); +}); From 14637ec17bb477c6a00b74128c275eb0886db982 Mon Sep 17 00:00:00 2001 From: Antony Rizzitelli Date: Thu, 19 Nov 2026 08:49:14 +0000 Subject: [PATCH 10/16] test(e2e): tag the access redirect journey @smoke in place of the ported one --- e2e/journeys/access.mjs | 1 + 1 file changed, 1 insertion(+) diff --git a/e2e/journeys/access.mjs b/e2e/journeys/access.mjs index 20605b4..b9dc16d 100644 --- a/e2e/journeys/access.mjs +++ b/e2e/journeys/access.mjs @@ -9,6 +9,7 @@ const TESTER_HEADER = "x-antelopejs-api-source"; export const journeys = guarded([ { id: "a-member-without-the-modules-permissions-is-sent-to-their-settings", + tag: "@smoke", title: "every page of the console sends a member whose role holds none of its permissions to their own Settings page, where none of the console's headings show, while the owner stays on the page", entryPoint: "/modules/api, /modules/api/overview, /modules/api/logs, /modules/api/routes and /modules/api/settings", async run(/** @type {Ctx} */ ctx) { From 168b7f549ac244740605a8c5d06f1645364c5a44 Mon Sep 17 00:00:00 2001 From: Antony Rizzitelli Date: Thu, 19 Nov 2026 08:53:58 +0000 Subject: [PATCH 11/16] test(automation): run the automation, retention and settings journeys as integration tests --- e2e/api.mjs | 25 -- e2e/journeys/automation.mjs | 219 +--------------- e2e/journeys/retention.mjs | 83 +----- e2e/journeys/settings.mjs | 130 +-------- e2e/seed.mjs | 13 - tests/automation.journeys.test.ts | 423 ++++++++++++++++++++++++++++++ tests/helpers/automation.ts | 140 ++++++++++ tests/retention.journeys.test.ts | 183 +++++++++++++ tests/settings.journeys.test.ts | 247 +++++++++++++++++ 9 files changed, 997 insertions(+), 466 deletions(-) create mode 100644 tests/automation.journeys.test.ts create mode 100644 tests/helpers/automation.ts create mode 100644 tests/retention.journeys.test.ts create mode 100644 tests/settings.journeys.test.ts diff --git a/e2e/api.mjs b/e2e/api.mjs index d183245..863f048 100644 --- a/e2e/api.mjs +++ b/e2e/api.mjs @@ -17,19 +17,6 @@ export const FIXTURE_ROUTES = 23; /** The console's HTTP API, which the dashboard calls. */ export const MONITOR = "/api/monitoring"; -/** - * The log file of the journey's backend: dms-control names it after the - * journey's database, so each journey's backend writes its own. - * - * @param {Ctx} ctx The journey - * @returns {string} Its path - */ -export function backendLog(ctx) { - const backend = ctx.state.processes.find((entry) => entry.name === "backend"); - if (!backend) throw new Error("the run has no backend"); - return backend.log; -} - /** * @param {unknown} value Anything a JSON answer holds * @returns {value is Row} Whether it is an object @@ -425,15 +412,3 @@ export async function withEnv(ctx, env, body) { await ctx.restartBackend(); } } - -/** - * Runs an automation action the module registered with the stand-in provider. - * - * @param {Ctx} ctx The journey - * @param {string} id The action's id - * @param {unknown} input Its input - * @returns {Promise} The action's output - */ -export function runAction(ctx, id, input) { - return ctx.api(`run ${id} ${JSON.stringify(input).slice(0, 60)}`, { method: "POST", path: `/dms/verify-api/nodes/action/${id}`, body: input, as: "anon" }); -} diff --git a/e2e/journeys/automation.mjs b/e2e/journeys/automation.mjs index 146a93c..e5cb694 100644 --- a/e2e/journeys/automation.mjs +++ b/e2e/journeys/automation.mjs @@ -1,220 +1,3 @@ -import { readFileSync } from "node:fs"; import { guarded } from "../guard.mjs"; -import { FIXTURE_ROUTES, backendLog, check, isRow, isText, list, mon, num, pick, runAction, same, saveSettings, settled, text, traffic, withEnv } from "../api.mjs"; -import { seedLogs, seedStatistics } from "../seed.mjs"; -/** @typedef {import("../api.mjs").Ctx} Ctx */ - -/** - * @param {unknown} value A JSON object - * @returns {string[]} Its keys - */ -const keysOf = (value) => (isRow(value) ? Object.keys(value) : []); - -const WITH_NODES = { VERIFY_API_AUTOMATION: "1" }; - -/** - * @param {Ctx} ctx The journey - * @param {string} id A trigger's id - * @param {unknown} config Its configuration - * @returns {Promise} The activation's key - */ -async function arm(ctx, id, config) { - const answer = await ctx.api(`arm ${id}`, { method: "POST", path: `/dms/verify-api/nodes/trigger/${id}/activate`, body: config, as: "anon" }); - return text(answer.body, "activation"); -} - -/** - * @param {Ctx} ctx The journey - * @param {string} key An activation - * @returns {Promise<{ flags: string[], rows: Record[], ageMs: number }>} What it emitted so far - */ -async function emitted(ctx, key) { - const answer = await ctx.api(`emissions of ${key}`, { method: "GET", path: `/dms/verify-api/nodes/trigger/activation/${key}`, as: "anon" }); - const rows = list(answer.body, "emissions"); - return { flags: rows.map((row) => `${text(row, "uri")}:${text(row, "flag")}`).toSorted((left, right) => left.localeCompare(right)), rows, ageMs: num(answer.body, "ageMs") ?? 0 }; -} - -export const journeys = guarded([ - { - id: "without-the-automation-module-the-registration-is-a-silent-no-op", - title: "with no @antelopejs/dms-automation in the project the module starts, says only that the optional interface has no provider, logs no error about the nodes, and the console keeps working", - entryPoint: "the backend log of the start and GET /api/monitoring/overview/health", - async run(/** @type {Ctx} */ ctx) { - await ctx.restartBackend(); - await traffic(ctx, "a call", [{ path: "/fx/ok", count: 1 }]); - const log = readFileSync(backendLog(ctx), "utf8").split("\n"); - const start = log.map((line) => /Server started/.test(line)).lastIndexOf(true); - const lines = log.slice(Math.max(0, start - 60)); - const health = await mon(ctx, "health", "/overview/health"); - const settings = await mon(ctx, "settings", "/settings"); - await ctx.verify("the start", [ - check("the log says the optional automation interface has no provider", lines.some((line) => /Optional interface '@antelopejs\/interface-dms-automation' has no provider/.test(line)), lines.filter((line) => /automation/i.test(line))), - same("no error line names the automation nodes", lines.filter((line) => /\[ERROR\]/.test(line) && /automation|api\.route-unhealthy|api\.get-api-health|api\.query-request-logs/i.test(line)), []), - same("no line says dms-api failed", lines.filter((line) => /dms-api/.test(line) && /(failed|did not construct|Failed to)/i.test(line)), []), - same("the console answers", [health.status, settings.status, num(settings.body, "statisticsLifetimeDays")], [200, 200, 90]), - check("the health reads the call", text(health.body, "verdict") !== "", text(health.body, "verdict")), - ]); - }, - }, - { - id: "the-module-registers-its-three-nodes-with-their-documented-inputs", - title: "with an automation provider the module registers api.get-api-health, api.query-request-logs and the singleton trigger api.route-unhealthy, whose inputs declare days (minimum 1), limit (default 50, maximum 200), the three status classes, intervalSeconds (default 60, minimum 10) and the flags broken, slow and warn (default broken and slow)", - entryPoint: "the nodes registered with the provider", - async run(/** @type {Ctx} */ ctx) { - await withEnv(ctx, WITH_NODES, async () => { - const nodes = await ctx.api("the registered nodes", { method: "GET", path: "/dms/verify-api/nodes", as: "anon" }); - const actions = list(nodes.body, "actions").filter((node) => text(node, "id").startsWith("api.")); - const triggers = list(nodes.body, "triggers").filter((node) => text(node, "id").startsWith("api.")); - const property = (/** @type {unknown} */ node, /** @type {string} */ schema, /** @type {string} */ name) => pick(node, schema, "properties", name); - const health = actions.find((node) => text(node, "id") === "api.get-api-health"); - const logs = actions.find((node) => text(node, "id") === "api.query-request-logs"); - const unhealthy = triggers.find((node) => text(node, "id") === "api.route-unhealthy"); - await ctx.verify("the nodes", [ - same("actions", actions.map((node) => text(node, "id")).toSorted((left, right) => left.localeCompare(right)), ["api.get-api-health", "api.query-request-logs"]), - same("triggers", triggers.map((node) => text(node, "id")), ["api.route-unhealthy"]), - same("the trigger runs on one node of a cluster", pick(unhealthy, "cluster"), "singleton"), - same("days", [pick(property(health, "inputSchema", "days"), "type"), pick(property(health, "inputSchema", "days"), "minimum")], ["number", 1]), - same("limit", [pick(property(logs, "inputSchema", "limit"), "default"), pick(property(logs, "inputSchema", "limit"), "maximum")], [50, 200]), - same("statusClass", pick(property(logs, "inputSchema", "statusClass"), "enum"), ["success", "client-error", "server-error"]), - same("the other filters", ["routeId", "uri", "method", "statusMin", "statusMax"].map((name) => pick(property(logs, "inputSchema", name), "type")), ["string", "string", "string", "number", "number"]), - same("intervalSeconds", [pick(property(unhealthy, "configSchema", "intervalSeconds"), "default"), pick(property(unhealthy, "configSchema", "intervalSeconds"), "minimum")], [60, 10]), - same("flags", [pick(property(unhealthy, "configSchema", "flags"), "items", "enum"), pick(property(unhealthy, "configSchema", "flags"), "default")], [["broken", "slow", "warn"], ["broken", "slow"]]), - same("the trigger's output", keysOf(pick(unhealthy, "outputSchema", "properties")), ["routeKey", "method", "uri", "flag", "errorRate", "averageLatencyMs", "requestsLast24h"]), - same("the health action's output", keysOf(pick(health, "outputSchema", "properties")), ["totalRoutes", "failingRoutes", "slowRoutesCount", "averageLatencyMs", "callsToday", "errors24h", "statusBreakdown"]), - ]); - }); - }, - }, - { - id: "get-api-health-restricts-its-breakdown-to-the-last-days", - title: "the status breakdown covers every retained day without days and the last N days with it (4 calls with 1 server error 10 days ago and 6 calls today), while the KPIs keep their fixed windows", - entryPoint: "api.get-api-health.execute", - async run(/** @type {Ctx} */ ctx) { - await seedStatistics(ctx, "10 days ago", { method: "GET", uri: "/fx/ok", daysAgo: 10, requests: 4, serverErrors: 1 }); - await seedStatistics(ctx, "today", { method: "GET", uri: "/fx/ok", daysAgo: 0, requests: 6 }); - await withEnv(ctx, WITH_NODES, async () => { - const breakdown = async (/** @type {unknown} */ input) => { - const answer = await runAction(ctx, "api.get-api-health", input); - return { all: pick(answer.body, "statusBreakdown"), kpis: answer.body }; - }; - const everything = await breakdown({}); - const one = await breakdown({ days: 1 }); - const five = await breakdown({ days: 5 }); - const thirty = await breakdown({ days: 30 }); - const figures = (/** @type {unknown} */ value) => [num(value, "total"), num(value, "serverErrors"), num(value, "clientErrors"), num(value, "success")]; - await ctx.verify("the breakdown", [ - same("without days: every retained day", figures(everything.all), [10, 1, 0, 9]), - same("the last day", figures(one.all), [6, 0, 0, 6]), - same("the last 5 days", figures(five.all), [6, 0, 0, 6]), - same("the last 30 days", figures(thirty.all), [10, 1, 0, 9]), - ]); - await ctx.verify("the KPIs", [ - same("totalRoutes counts the routes of the scope", num(everything.kpis, "totalRoutes"), FIXTURE_ROUTES), - same("the KPIs do not follow days", [num(one.kpis, "callsToday"), num(one.kpis, "totalRoutes")], [num(everything.kpis, "callsToday"), num(everything.kpis, "totalRoutes")]), - same("callsToday", num(everything.kpis, "callsToday"), 6), - same("no route is failing", [num(everything.kpis, "failingRoutes"), num(everything.kpis, "errors24h")], [0, 0]), - ]); - }); - }, - }, - { - id: "query-request-logs-pages-caps-and-filters", - title: "api.query-request-logs returns 50 rows by default, at most 200 whatever limit is asked, the rows of a status class, status range, uri or method, and a nextCursor while rows remain", - entryPoint: "api.query-request-logs.execute", - async run(/** @type {Ctx} */ ctx) { - await seedLogs(ctx, "250 good calls", { minutesAgo: 100, count: 250, uri: "/fx/ok" }); - await seedLogs(ctx, "20 client errors", { minutesAgo: 50, count: 20, uri: "/fx/missing", statusCode: 404, message: "seeded gone" }); - await seedLogs(ctx, "3 server errors", { minutesAgo: 20, count: 3, uri: "/fx/boom", statusCode: 500, message: "seeded boom" }); - await withEnv(ctx, WITH_NODES, async () => { - const ask = async (/** @type {unknown} */ input) => (await runAction(ctx, "api.query-request-logs", input)).body; - const defaults = await ask({}); - const capped = await ask({ limit: 250 }); - const ten = await ask({ limit: 10 }); - const client = await ask({ statusClass: "client-error", limit: 200 }); - const server = await ask({ statusClass: "server-error" }); - const success = await ask({ statusClass: "success" }); - const range = await ask({ statusMin: 500, statusMax: 599 }); - const exact = await ask({ statusMin: 404, statusMax: 404, limit: 200 }); - const byUri = await ask({ uri: "/fx/missing", limit: 200 }); - const byMethod = await ask({ method: "POST" }); - const statuses = (/** @type {unknown} */ page) => [...new Set(list(page, "results").map((row) => num(row, "statusCode")))]; - await ctx.verify("the page size", [ - same("the default is 50 rows and more remain", [list(defaults, "results").length, text(defaults, "nextCursor") !== ""], [50, true]), - same("a limit of 250 is capped to 200 and more remain", [list(capped, "results").length, text(capped, "nextCursor") !== ""], [200, true]), - same("a limit of 10", [list(ten, "results").length, text(ten, "nextCursor") !== ""], [10, true]), - ]); - await ctx.verify("the filters", [ - same("client errors only", [list(client, "results").length, statuses(client), text(client, "nextCursor")], [20, [404], ""]), - same("server errors only", [list(server, "results").length, statuses(server)], [3, [500]]), - same("success only", [list(success, "results").length, statuses(success)], [50, [200]]), - same("a status range", [list(range, "results").length, statuses(range)], [3, [500]]), - same("a range of one status", [list(exact, "results").length, statuses(exact)], [20, [404]]), - same("a uri", [list(byUri, "results").length, [...new Set(list(byUri, "results").map((row) => text(row, "uri")))]], [20, ["/fx/missing"]]), - same("a method with none of it", list(byMethod, "results").length, 0), - same("each row carries an ISO timestamp", isText(pick(list(defaults, "results")[0], "timestamp")) && !Number.isNaN(Date.parse(text(list(defaults, "results")[0], "timestamp"))), true), - ]); - }); - }, - }, - { - id: "route-unhealthy-fires-on-the-flags-it-watches-and-never-faster-than-ten-seconds", - title: "the trigger fires once for a route that enters broken or warn after it was armed, only for the flags it watches (default broken and slow), not for a state it saw when armed, and an interval under 10 seconds waits the 10", - entryPoint: "api.route-unhealthy.activate", - async run(/** @type {Ctx} */ ctx) { - await saveSettings(ctx, "slow from 50 ms", { requestSlownessThresholdMs: 50 }); - await withEnv(ctx, WITH_NODES, async () => { - const id = "api.route-unhealthy"; - const watching = { - broken: await arm(ctx, id, { intervalSeconds: 10, flags: ["broken"] }), - warn: await arm(ctx, id, { intervalSeconds: 10, flags: ["warn"] }), - defaults: await arm(ctx, id, { intervalSeconds: 10 }), - quick: await arm(ctx, id, { intervalSeconds: 1, flags: ["broken", "slow", "warn"] }), - }; - await traffic(ctx, "a failing, a noisy and a slow route", [{ path: "/fx/boom", count: 25 }, { path: "/fx/missing", count: 25 }, { path: "/fx/slow?ms=80", count: 20 }]); - await settled(ctx, "the routes are flagged", "/overview/attention", (body) => list(body, "items").length >= 3); - const early = await emitted(ctx, watching.quick); - await new Promise((resolve) => setTimeout(resolve, 12_000)); - const results = { - broken: await emitted(ctx, watching.broken), - warn: await emitted(ctx, watching.warn), - defaults: await emitted(ctx, watching.defaults), - quick: await emitted(ctx, watching.quick), - }; - const boom = results.broken.rows[0]; - await ctx.verify("the flags", [ - same("broken only", results.broken.flags, ["/fx/boom:broken"]), - same("warn only", results.warn.flags, ["/fx/missing:warn"]), - same("the default flags are broken and slow", results.defaults.flags, ["/fx/boom:broken", "/fx/slow:slow"]), - same("all three flags", results.quick.flags, ["/fx/boom:broken", "/fx/missing:warn", "/fx/slow:slow"]), - same("the output of a broken route", [text(boom, "routeKey"), text(boom, "method"), text(boom, "uri"), text(boom, "flag"), num(boom, "requestsLast24h")], ["GET:/fx/boom", "GET", "/fx/boom", "broken", 25]), - same("the error rate of a route failing every call, as a ratio", num(boom, "errorRate"), 1), - ]); - await ctx.verify("the interval", [ - same("an interval of 1 second fires nothing before 10 seconds", early.rows.length, 0), - check("and fires after them", results.quick.rows.length === 3, results.quick.ageMs), - ]); - for (const key of Object.values(watching)) await ctx.api(`stop ${key}`, { method: "POST", path: `/dms/verify-api/nodes/trigger/${id}/deactivate/${key}`, as: "anon" }); - }); - }, - }, - { - id: "query-request-logs-next-cursor-reads-the-next-page", - knownFailures: { "asking with it returns other rows": "api.query-request-logs returns a nextCursor but takes no cursor input, so a second call returns the first page again" }, - title: "the nextCursor api.query-request-logs returns reads the next page: asked again with it, the action returns other rows (dms-api issue: the action takes no cursor input, so it returns the first page again)", - entryPoint: "api.query-request-logs.execute", - async run(/** @type {Ctx} */ ctx) { - await seedLogs(ctx, "30 good calls", { minutesAgo: 100, count: 30, uri: "/fx/ok" }); - await withEnv(ctx, WITH_NODES, async () => { - const ask = async (/** @type {unknown} */ input) => (await runAction(ctx, "api.query-request-logs", input)).body; - const first = await ask({ limit: 10 }); - const second = await ask({ limit: 10, cursor: text(first, "nextCursor") }); - const ids = (/** @type {unknown} */ page) => list(page, "results").map((row) => text(row, "_id")); - await ctx.verify("the next page", [ - check("a cursor comes back while rows remain", text(first, "nextCursor") !== "", text(first, "nextCursor")), - check("asking with it returns other rows", ids(second).length === 10 && ids(second).every((id) => !ids(first).includes(id)), { first: ids(first), second: ids(second) }), - ]); - }); - }, - }, -]); +export const journeys = guarded([]); diff --git a/e2e/journeys/retention.mjs b/e2e/journeys/retention.mjs index 625f8f9..2d397d7 100644 --- a/e2e/journeys/retention.mjs +++ b/e2e/journeys/retention.mjs @@ -1,89 +1,10 @@ -import { fileURLToPath } from "node:url"; import { guarded } from "../guard.mjs"; -import { list, mon, num, same, saveSettings, settled, openPage } from "../api.mjs"; -import { dayStart, readDocuments, seedLogs, seedStatistics } from "../seed.mjs"; +import { mon, num, same, settled, openPage } from "../api.mjs"; +import { seedLogs } from "../seed.mjs"; /** @typedef {import("../api.mjs").Ctx} Ctx */ export const journeys = guarded([ - { - id: "lowering-a-retention-tells-what-the-next-prune-deletes", - title: "the retention impact counts the logs and the days of statistics older than the value, and refuses a value outside the bounds or an unknown field", - entryPoint: "GET /api/monitoring/settings/retention-impact", - async run(/** @type {Ctx} */ ctx) { - await seedLogs(ctx, "40 days old", { daysAgo: 40, count: 3 }); - await seedLogs(ctx, "99.5 days old", { daysAgo: 99.5, count: 2 }); - await seedStatistics(ctx, "50 days old", { method: "GET", uri: "/fx/ok", daysAgo: 50, requests: 4 }); - await seedStatistics(ctx, "70 days old", { method: "GET", uri: "/fx/ok", daysAgo: 70, requests: 6 }); - const read = (/** @type {string} */ field, /** @type {number} */ days) => mon(ctx, `${field} at ${days}`, `/settings/retention-impact?field=${field}&days=${days}`); - const figures = (/** @type {import("../api.mjs").Answer} */ answer) => [num(answer.body, "days"), num(answer.body, "requests")]; - const logs30 = await read("requestLogRetentionDays", 30); - const logs60 = await read("requestLogRetentionDays", 60); - const logs366 = await read("requestLogRetentionDays", 366); - const stats30 = await read("statisticsLifetimeDays", 30); - const stats60 = await read("statisticsLifetimeDays", 60); - const stats90 = await read("statisticsLifetimeDays", 90); - const unknown = await read("nope", 30); - const zero = await read("requestLogRetentionDays", 0); - const over = await read("requestLogRetentionDays", 367); - await ctx.verify("impact", [ - same("logs older than 30 days: the days back they reach and the requests", figures(logs30), [70, 5]), - same("logs older than 60 days", figures(logs60), [40, 2]), - same("logs older than 366 days", figures(logs366), [0, 0]), - same("statistics older than 30 days: the 50 and 70 day old rollups", figures(stats30), [2, 10]), - same("statistics older than 60 days: the 70 day old rollup", figures(stats60), [1, 6]), - same("statistics older than 90 days", figures(stats90), [0, 0]), - same("an unknown field is refused", unknown.status, 400), - same("0 days is refused", zero.status, 400), - same("367 days is refused", over.status, 400), - ]); - }, - }, - { - id: "the-hourly-prune-deletes-what-the-retention-passed", - title: "at the turn of the hour the prune deletes the request logs older than the log retention and the days of statistics older than the statistics retention, and keeps everything newer", - entryPoint: "the hourly job of the module", - async run(/** @type {Ctx} */ ctx) { - await saveSettings(ctx, "keep logs 20 days and statistics 30 days", { requestLogRetentionDays: 20, statisticsLifetimeDays: 30 }); - await seedLogs(ctx, "39.5 days old", { daysAgo: 39.5, count: 3 }); - await seedLogs(ctx, "25 days old", { daysAgo: 25, count: 2, uri: "/fx/boom", statusCode: 500 }); - await seedLogs(ctx, "10 days old", { daysAgo: 10, count: 4 }); - await seedLogs(ctx, "an hour old", { daysAgo: 1 / 24, count: 1 }); - await seedStatistics(ctx, "100 days old", { method: "GET", uri: "/fx/ok", daysAgo: 100, requests: 4 }); - await seedStatistics(ctx, "40 days old", { method: "GET", uri: "/fx/ok", daysAgo: 40, requests: 6 }); - await seedStatistics(ctx, "10 days old", { method: "GET", uri: "/fx/ok", daysAgo: 10, requests: 8 }); - const impact = await mon(ctx, "what the next prune deletes", "/settings/retention-impact?field=requestLogRetentionDays&days=20"); - const seeded = () => readDocuments(ctx, "dms-api__request_log", { userAgent: "seed" }); - const before = (await seeded()).length; - const toNextHour = 3_600_000 - (Date.now() % 3_600_000); - const offset = (toNextHour > 20_000 ? toNextHour : toNextHour + 3_600_000) - 12_000; - const previous = { ...ctx.state.extraEnv }; - ctx.state.extraEnv = { ...previous, NODE_OPTIONS: `--require ${fileURLToPath(new URL("../clock.cjs", import.meta.url))}`, VERIFY_API_CLOCK_OFFSET_MS: String(offset) }; - try { - await ctx.restartBackend(); - const deadline = Date.now() + 90_000; - let after = before; - while (Date.now() < deadline && after > 5) { - await new Promise((resolve) => setTimeout(resolve, 1000)); - after = (await seeded()).length; - } - await new Promise((resolve) => setTimeout(resolve, 3000)); - } finally { - ctx.state.extraEnv = previous; - } - const logs = await seeded(); - const ages = logs.map((row) => Math.round((Date.now() - new Date(String(row.timestamp)).getTime()) / 86_400_000)).toSorted((left, right) => left - right); - const stats = await readDocuments(ctx, "dms-api__routes_statistics", { _id: "GET:/fx/ok" }); - const kept = stats[0]?.statistics; - const days = (Array.isArray(kept) ? list(kept) : []).map((entry) => num(entry, "day") ?? 0).toSorted((left, right) => left - right); - await ctx.verify("the prune", [ - same("the impact said 5 requests reaching back 20 days before", [num(impact.body, "days"), num(impact.body, "requests")], [20, 5]), - same("the logs before", before, 10), - same("the logs left are the four 10 days old and the hour old one", ages, [0, 10, 10, 10, 10]), - same("the statistics left are the 10 days old and the 40 days old are gone with the 100 days old", days, [dayStart(10)]), - ]); - }, - }, { id: "lowering-a-retention-asks-before-it-deletes", title: "lowering the log retention shows what the next prune deletes under the field, Save changes asks \"Delete stored data?\" with the old and new values and the count, Cancel keeps 30 days, Save and delete writes 7", diff --git a/e2e/journeys/settings.mjs b/e2e/journeys/settings.mjs index b384c91..9b14157 100644 --- a/e2e/journeys/settings.mjs +++ b/e2e/journeys/settings.mjs @@ -1,46 +1,10 @@ -import { existsSync, readFileSync, statSync } from "node:fs"; import { guarded } from "../guard.mjs"; -import { FIXTURE_ROUTES, backendLog, check, list, mon, pick, same, saveSettings, settled, text, traffic, withConfig, openPage } from "../api.mjs"; +import { FIXTURE_ROUTES, check, list, mon, pick, same, saveSettings, settled, text, traffic, withConfig, openPage } from "../api.mjs"; /** @typedef {import("../api.mjs").Ctx} Ctx */ const DEFAULT_HEADERS = ["content-type", "content-length", "user-agent", "accept", "accept-encoding", "host", "referer", "origin"]; -/** - * Saves one field and reads the status back. - * - * @param {Ctx} ctx The journey - * @param {string} field The settings field - * @param {unknown} value The value - * @returns {Promise} The HTTP status - */ -async function statusOf(ctx, field, value) { - return (await saveSettings(ctx, `${field} = ${JSON.stringify(value).slice(0, 40)}`, { [field]: value })).status; -} - -/** - * Restarts the backend with the module configured, and reports whether it came back. - * - * @param {Ctx} ctx The journey - * @param {Record} config The module options - * @returns {Promise<{ started: boolean, log: string }>} Whether it started, and the backend log - */ -async function startWith(ctx, config) { - const previous = { ...ctx.state.extraEnv }; - const file = backendLog(ctx); - const before = existsSync(file) ? statSync(file).size : 0; - ctx.state.extraEnv = { ...previous, VERIFY_API_CONFIG: JSON.stringify(config) }; - let started = true; - try { - await ctx.restartBackend(); - } catch { - started = false; - } - const log = readFileSync(file).subarray(before).toString("utf8"); - ctx.state.extraEnv = previous; - return { started, log }; -} - export const journeys = guarded([ { id: "the-form-shows-the-documented-defaults", @@ -78,85 +42,6 @@ export const journeys = guarded([ ]); }, }, - { - id: "every-bound-of-the-form-is-enforced-on-its-edge", - title: "the retentions, body cap, slow threshold and header list accept their documented edge values and refuse the value just outside", - entryPoint: "POST /api/monitoring/settings", - async run(/** @type {Ctx} */ ctx) { - const cases = [ - ["requestLogRetentionDays", [[0, 400], [1, 200], [366, 200], [367, 400], [1.5, 400]]], - ["statisticsLifetimeDays", [[45, 400], [30, 200], [60, 200], [90, 200], [120, 400]]], - ["requestLogMaxBodyKb", [[-1, 400], [0, 200], [10240, 200], [10241, 400]]], - ["requestSlownessThresholdMs", [[49, 400], [50, 200], [75, 400], [100, 200], [600000, 200], [600050, 400]]], - ["scope", [["own", 200], ["modules", 200], ["all", 200], ["everything", 400]]], - ]; - for (const [field, expectations] of cases) { - const seen = []; - for (const [value, expected] of expectations) seen.push([value, await statusOf(ctx, String(field), value), expected]); - await ctx.verify(String(field), seen.map(([value, got, expected]) => same(`${String(field)} = ${String(value)}`, got, expected))); - } - const fifty = Array.from({ length: 50 }, (_, index) => `x-h${index}`); - await ctx.verify("the header list", [ - same("50 headers are accepted", await statusOf(ctx, "requestLogCaptureHeaders", fifty), 200), - same("51 headers are refused", await statusOf(ctx, "requestLogCaptureHeaders", [...fifty, "x-one-more"]), 400), - ]); - const secretHeaders = ["authorization", "cookie", "set-cookie", "x-api-key", "proxy-authorization", "X-Authorization", "x-antelopejs-api-source"]; - const refusals = []; - for (const header of secretHeaders) refusals.push([header, await statusOf(ctx, "requestLogCaptureHeaders", [header])]); - await ctx.verify("secret-looking headers", refusals.map(([header, status]) => same(`${String(header)} is refused`, status, 400))); - const kept = []; - for (const header of ["x-request-trace", "accept-language", "if-none-match"]) kept.push([header, await statusOf(ctx, "requestLogCaptureHeaders", [header])]); - await ctx.verify("ordinary headers", kept.map(([header, status]) => same(`${String(header)} is accepted`, status, 200))); - }, - }, - { - id: "use-default-returns-a-field-to-the-module-option", - title: "a saved value overrides the module option without a restart; null (Use default) removes the override and the field follows the option again; the override survives a restart", - entryPoint: "POST /api/monitoring/settings with null", - async run(/** @type {Ctx} */ ctx) { - await withConfig(ctx, { requestSlownessThreshold: 500, requestLogMaxBodySize: 2048 }, async () => { - const base = await mon(ctx, "module options", "/settings"); - await saveSettings(ctx, "override the slow threshold", { requestSlownessThresholdMs: 250 }); - const overridden = await mon(ctx, "after the override", "/settings"); - await ctx.restartBackend(); - const restarted = await mon(ctx, "after a restart", "/settings"); - await saveSettings(ctx, "use default", { requestSlownessThresholdMs: null }); - const back = await mon(ctx, "after Use default", "/settings"); - await ctx.verify("overrides", [ - same("the module options are read: slow", pick(base.body, "requestSlownessThresholdMs"), 500), - same("the module options are read: body cap in KB", pick(base.body, "requestLogMaxBodyKb"), 2), - same("the override wins at once", pick(overridden.body, "requestSlownessThresholdMs"), 250), - same("the override survives a restart", pick(restarted.body, "requestSlownessThresholdMs"), 250), - same("Use default follows the module option again", pick(back.body, "requestSlownessThresholdMs"), 500), - same("the other field was never overridden", pick(back.body, "requestLogMaxBodyKb"), 2), - ]); - }); - }, - }, - { - id: "the-module-refuses-to-start-with-a-value-the-form-would-refuse", - title: "a module option the settings form would refuse (statistics kept 45 days, a slow threshold off its step, a body cap or log retention out of bounds, a secret header) stops the module at start with the reasons; valid options start it", - entryPoint: "the dms-api entry of antelope.config.ts", - async run(/** @type {Ctx} */ ctx) { - const DAY = 86_400_000; - const refused = [ - { name: "statisticsLifetime", config: { statisticsLifetime: 45 * DAY } }, - { name: "requestSlownessThreshold off its step", config: { requestSlownessThreshold: 75 } }, - { name: "requestLogMaxBodySize above 10 MB", config: { requestLogMaxBodySize: 10_241 * 1024 } }, - { name: "requestLogRetention of 367 days", config: { requestLogRetention: 367 * DAY } }, - { name: "a secret header", config: { requestLogCaptureHeaders: ["authorization"] } }, - ]; - const results = []; - for (const { name, config } of refused) { - const outcome = await startWith(ctx, config); - results.push({ name, started: outcome.started, said: /\[dms-api\] invalid configuration/.test(outcome.log) }); - await ctx.restartBackend(); - } - await ctx.verify("refused options", results.map((entry) => check(`${entry.name}: the module stops and says so`, !entry.started && entry.said, entry))); - const accepted = await startWith(ctx, { statisticsLifetime: 60 * DAY, requestLogRetention: 14 * DAY, requestSlownessThreshold: 500, requestLogMaxBodySize: 0, requestLogCaptureHeaders: ["x-webhook-signature"] }); - await ctx.verify("valid options", [check("the module starts", accepted.started, accepted.started)]); - }, - }, { id: "the-scope-decides-which-routes-and-requests-every-page-watches", title: "Own routes lists the project's routes, Module routes the installed modules' routes, All routes both; the Request logs and the Routes list follow, and a scope change applies at once", @@ -191,19 +76,6 @@ export const journeys = guarded([ await ctx.verify("the scope survives a restart", [same("scope", pick(kept.body, "scope"), "all")]); }, }, - { - id: "a-log-retention-of-no-day-is-refused-at-start", - title: "a requestLogRetention of 0 is out of the 1 to 366 days the settings page accepts, so the module refuses to start with it", - entryPoint: "the dms-api entry of antelope.config.ts", - async run(/** @type {Ctx} */ ctx) { - const outcome = await startWith(ctx, { requestLogRetention: 0 }); - const form = outcome.started ? await mon(ctx, "the form's retention", "/settings") : undefined; - if (outcome.started) await ctx.restartBackend(); - await ctx.verify("a retention of 0", [ - check("the module stops and says so", !outcome.started && /\[dms-api\] invalid configuration/.test(outcome.log), { started: outcome.started, form: form?.body }), - ]); - }, - }, { id: "retention-fields-have-no-use-default-but-the-others-do", title: "once overridden, the slow threshold and the body size show Use default beside their module default; the two retentions never do, since lowering a retention deletes data", diff --git a/e2e/seed.mjs b/e2e/seed.mjs index c0d4967..7f64cb6 100644 --- a/e2e/seed.mjs +++ b/e2e/seed.mjs @@ -138,16 +138,3 @@ export async function seedStatistics(ctx, label, day) { ), ); } - -/** - * Reads the run's database with the MongoDB driver. - * - * @param {Ctx} ctx The journey - * @param {string} collection The collection - * @param {Record} filter The filter - * @returns {Promise[]>} The documents - */ -export async function readDocuments(ctx, collection, filter = {}) { - const found = await withDatabase(ctx, (db) => db.collection(collection).find(filter).toArray()); - return found.filter(isRow); -} diff --git a/tests/automation.journeys.test.ts b/tests/automation.journeys.test.ts new file mode 100644 index 0000000..2c897f3 --- /dev/null +++ b/tests/automation.journeys.test.ts @@ -0,0 +1,423 @@ +import { describe, expect } from "vitest"; +import type { z } from "zod"; +import { + HEALTH, + INTERVAL_WAIT_MS, + LOGS, + TRIGGER_PATH, + UNHEALTHY, + WITH_NODES, + apiNodes, + arm, + breakdownSchema, + byName, + emitted, + attentionSchema, + hasMore, + healthSchema, + lifetimeSchema, + queryLogs, + statusesOf, + verdictSchema, +} from "./helpers/automation"; +import { type Dms, FIXTURE_ROUTES, test } from "./helpers/dms"; +import { seedLogs, seedStatistics } from "./helpers/seed"; + +async function assertWithoutTheAutomationModuleTheRegistrationIsASilentNoOp( + dms: Dms, +): Promise { + const from = dms.backend.logSize(); + await dms.restart(); + await dms.traffic([{ path: "/fx/ok", count: 1 }]); + const lines = dms.backend.logSince(from).split("\n"); + const health = await dms.mon("/overview/health"); + const settings = await dms.mon("/settings"); + expect + .soft( + lines.some((line) => + /Optional interface '@antelopejs\/interface-dms-automation' has no provider/.test( + line, + ), + ), + "the log says the optional automation interface has no provider", + ) + .toBe(true); + expect + .soft( + lines.filter( + (line) => + /\[ERROR\]/.test(line) && + /automation|api\.route-unhealthy|api\.get-api-health|api\.query-request-logs/i.test( + line, + ), + ), + "no error line names the automation nodes", + ) + .toEqual([]); + expect + .soft( + lines.filter( + (line) => + /dms-api/.test(line) && + /(failed|did not construct|Failed to)/i.test(line), + ), + "no line says dms-api failed", + ) + .toEqual([]); + expect + .soft( + [ + health.status, + settings.status, + lifetimeSchema.parse(settings.body).statisticsLifetimeDays, + ], + "the console answers", + ) + .toEqual([200, 200, 90]); + expect + .soft(verdictSchema.parse(health.body).verdict, "the health reads the call") + .not.toBe(""); +} + +async function assertTheModuleRegistersItsThreeNodesWithTheirDocumentedInputs( + dms: Dms, +): Promise { + await dms.withEnv(WITH_NODES, async () => { + const { actions, triggers } = apiNodes( + (await dms.api("GET", "/dms/verify-api/nodes", { as: "anon" })).body, + ); + const health = actions.find((node) => node.id === HEALTH); + const logs = actions.find((node) => node.id === LOGS); + const unhealthy = triggers.find((node) => node.id === UNHEALTHY); + const input = (node: typeof health, name: string) => + node?.inputSchema?.get(name); + const config = (node: typeof health, name: string) => + node?.configSchema?.get(name); + expect + .soft(actions.map((node) => node.id).toSorted(byName), "actions") + .toEqual([HEALTH, LOGS]); + expect + .soft( + triggers.map((node) => node.id), + "triggers", + ) + .toEqual([UNHEALTHY]); + expect + .soft(unhealthy?.cluster, "the trigger runs on one node of a cluster") + .toBe("singleton"); + expect + .soft(input(health, "days"), "days") + .toMatchObject({ type: "number", minimum: 1 }); + expect + .soft(input(logs, "limit"), "limit") + .toMatchObject({ default: 50, maximum: 200 }); + expect + .soft(input(logs, "statusClass")?.enum, "statusClass") + .toEqual(["success", "client-error", "server-error"]); + expect + .soft( + ["routeId", "uri", "method", "statusMin", "statusMax"].map( + (name) => input(logs, name)?.type, + ), + "the other filters", + ) + .toEqual(["string", "string", "string", "number", "number"]); + expect + .soft(config(unhealthy, "intervalSeconds"), "intervalSeconds") + .toMatchObject({ default: 60, minimum: 10 }); + expect.soft(config(unhealthy, "flags"), "flags").toMatchObject({ + items: { enum: ["broken", "slow", "warn"] }, + default: ["broken", "slow"], + }); + expect + .soft([...(unhealthy?.outputSchema.keys() ?? [])], "the trigger's output") + .toEqual([ + "routeKey", + "method", + "uri", + "flag", + "errorRate", + "averageLatencyMs", + "requestsLast24h", + ]); + expect + .soft( + [...(health?.outputSchema.keys() ?? [])], + "the health action's output", + ) + .toEqual([ + "totalRoutes", + "failingRoutes", + "slowRoutesCount", + "averageLatencyMs", + "callsToday", + "errors24h", + "statusBreakdown", + ]); + }); +} + +async function assertGetApiHealthRestrictsItsBreakdownToTheLastDays( + dms: Dms, +): Promise { + await seedStatistics(dms, { + method: "GET", + uri: "/fx/ok", + daysAgo: 10, + requests: 4, + serverErrors: 1, + }); + await seedStatistics(dms, { + method: "GET", + uri: "/fx/ok", + daysAgo: 0, + requests: 6, + }); + await dms.withEnv(WITH_NODES, async () => { + const health = async (input: { days?: number }) => + healthSchema.parse((await dms.runAction(HEALTH, input)).body); + const everything = await health({}); + const one = await health({ days: 1 }); + const five = await health({ days: 5 }); + const thirty = await health({ days: 30 }); + const figures = (breakdown: z.infer) => [ + breakdown.total, + breakdown.serverErrors, + breakdown.clientErrors, + breakdown.success, + ]; + expect + .soft( + figures(everything.statusBreakdown), + "without days: every retained day", + ) + .toEqual([10, 1, 0, 9]); + expect + .soft(figures(one.statusBreakdown), "the last day") + .toEqual([6, 0, 0, 6]); + expect + .soft(figures(five.statusBreakdown), "the last 5 days") + .toEqual([6, 0, 0, 6]); + expect + .soft(figures(thirty.statusBreakdown), "the last 30 days") + .toEqual([10, 1, 0, 9]); + expect + .soft( + everything.totalRoutes, + "totalRoutes counts the routes of the scope", + ) + .toBe(FIXTURE_ROUTES); + expect + .soft([one.callsToday, one.totalRoutes], "the KPIs do not follow days") + .toEqual([everything.callsToday, everything.totalRoutes]); + expect.soft(everything.callsToday, "callsToday").toBe(6); + expect + .soft( + [everything.failingRoutes, everything.errors24h], + "no route is failing", + ) + .toEqual([0, 0]); + }); +} + +async function assertQueryRequestLogsPagesCapsAndFilters( + dms: Dms, +): Promise { + await seedLogs(dms, { minutesAgo: 100, count: 250, uri: "/fx/ok" }); + await seedLogs(dms, { + minutesAgo: 50, + count: 20, + uri: "/fx/missing", + statusCode: 404, + message: "seeded gone", + }); + await seedLogs(dms, { + minutesAgo: 20, + count: 3, + uri: "/fx/boom", + statusCode: 500, + message: "seeded boom", + }); + await dms.withEnv(WITH_NODES, async () => { + const defaults = await queryLogs(dms, {}); + const capped = await queryLogs(dms, { limit: 250 }); + const ten = await queryLogs(dms, { limit: 10 }); + const client = await queryLogs(dms, { + statusClass: "client-error", + limit: 200, + }); + const server = await queryLogs(dms, { statusClass: "server-error" }); + const success = await queryLogs(dms, { statusClass: "success" }); + const range = await queryLogs(dms, { statusMin: 500, statusMax: 599 }); + const exact = await queryLogs(dms, { + statusMin: 404, + statusMax: 404, + limit: 200, + }); + const byUri = await queryLogs(dms, { uri: "/fx/missing", limit: 200 }); + const byMethod = await queryLogs(dms, { method: "POST" }); + expect + .soft( + [defaults.results.length, hasMore(defaults)], + "the default is 50 rows and more remain", + ) + .toEqual([50, true]); + expect + .soft( + [capped.results.length, hasMore(capped)], + "a limit of 250 is capped to 200 and more remain", + ) + .toEqual([200, true]); + expect + .soft([ten.results.length, hasMore(ten)], "a limit of 10") + .toEqual([10, true]); + expect + .soft( + [client.results.length, statusesOf(client), client.nextCursor ?? ""], + "client errors only", + ) + .toEqual([20, [404], ""]); + expect + .soft([server.results.length, statusesOf(server)], "server errors only") + .toEqual([3, [500]]); + expect + .soft([success.results.length, statusesOf(success)], "success only") + .toEqual([50, [200]]); + expect + .soft([range.results.length, statusesOf(range)], "a status range") + .toEqual([3, [500]]); + expect + .soft([exact.results.length, statusesOf(exact)], "a range of one status") + .toEqual([20, [404]]); + expect + .soft( + [ + byUri.results.length, + [...new Set(byUri.results.map((row) => row.uri))], + ], + "a uri", + ) + .toEqual([20, ["/fx/missing"]]); + expect.soft(byMethod.results.length, "a method with none of it").toBe(0); + expect + .soft( + Number.isNaN(Date.parse(defaults.results[0]?.timestamp ?? "")), + "each row carries an ISO timestamp", + ) + .toBe(false); + }); +} + +async function assertRouteUnhealthyFiresOnTheFlagsItWatchesAndNeverFasterThanTenSeconds( + dms: Dms, +): Promise { + await dms.saveSettings({ requestSlownessThresholdMs: 50 }); + await dms.withEnv(WITH_NODES, async () => { + const watching = { + broken: await arm(dms, { intervalSeconds: 10, flags: ["broken"] }), + warn: await arm(dms, { intervalSeconds: 10, flags: ["warn"] }), + defaults: await arm(dms, { intervalSeconds: 10 }), + quick: await arm(dms, { + intervalSeconds: 1, + flags: ["broken", "slow", "warn"], + }), + }; + await dms.traffic([ + { path: "/fx/boom", count: 25 }, + { path: "/fx/missing", count: 25 }, + { path: "/fx/slow?ms=80", count: 20 }, + ]); + await dms.settled("/overview/attention", (body) => { + const attention = attentionSchema.safeParse(body); + return attention.success && attention.data.items.length >= 3; + }); + const early = await emitted(dms, watching.quick); + await new Promise((done) => setTimeout(done, INTERVAL_WAIT_MS)); + const results = { + broken: await emitted(dms, watching.broken), + warn: await emitted(dms, watching.warn), + defaults: await emitted(dms, watching.defaults), + quick: await emitted(dms, watching.quick), + }; + expect + .soft(results.broken.flags, "broken only") + .toEqual(["/fx/boom:broken"]); + expect.soft(results.warn.flags, "warn only").toEqual(["/fx/missing:warn"]); + expect + .soft(results.defaults.flags, "the default flags are broken and slow") + .toEqual(["/fx/boom:broken", "/fx/slow:slow"]); + expect + .soft(results.quick.flags, "all three flags") + .toEqual(["/fx/boom:broken", "/fx/missing:warn", "/fx/slow:slow"]); + expect + .soft(results.broken.rows[0], "the output of a broken route") + .toMatchObject({ + routeKey: "GET:/fx/boom", + method: "GET", + uri: "/fx/boom", + flag: "broken", + requestsLast24h: 25, + }); + expect + .soft( + results.broken.rows[0]?.errorRate, + "the error rate of a route failing every call, as a ratio", + ) + .toBe(1); + expect + .soft( + early.rows.length, + "an interval of 1 second fires nothing before 10 seconds", + ) + .toBe(0); + expect.soft(results.quick.rows.length, "and fires after them").toBe(3); + for (const key of Object.values(watching)) + await dms.api("POST", `${TRIGGER_PATH}/${UNHEALTHY}/deactivate/${key}`, { + as: "anon", + }); + }); +} + +async function assertQueryRequestLogsNextCursorReadsTheNextPage( + dms: Dms, +): Promise { + await seedLogs(dms, { minutesAgo: 100, count: 30, uri: "/fx/ok" }); + await dms.withEnv(WITH_NODES, async () => { + const first = await queryLogs(dms, { limit: 10 }); + const cursor = first.nextCursor ?? ""; + const second = await queryLogs(dms, { limit: 10, cursor }); + const firstIds = first.results.map((row) => row._id); + const secondIds = second.results.map((row) => row._id); + expect + .soft(cursor !== "", "a cursor comes back while rows remain") + .toBe(true); + expect + .soft( + secondIds.length === 10 && + secondIds.every((id) => !firstIds.includes(id)), + "api.query-request-logs returns a nextCursor but takes no cursor input, so a second call returns the first page again", + ) + .toBe(true); + }); +} + +describe("automation", () => { + test("without-the-automation-module-the-registration-is-a-silent-no-op", ({ + dms, + }) => assertWithoutTheAutomationModuleTheRegistrationIsASilentNoOp(dms)); + test("the-module-registers-its-three-nodes-with-their-documented-inputs", ({ + dms, + }) => assertTheModuleRegistersItsThreeNodesWithTheirDocumentedInputs(dms)); + test("get-api-health-restricts-its-breakdown-to-the-last-days", ({ dms }) => + assertGetApiHealthRestrictsItsBreakdownToTheLastDays(dms)); + test("query-request-logs-pages-caps-and-filters", ({ dms }) => + assertQueryRequestLogsPagesCapsAndFilters(dms)); + test("route-unhealthy-fires-on-the-flags-it-watches-and-never-faster-than-ten-seconds", ({ + dms, + }) => + assertRouteUnhealthyFiresOnTheFlagsItWatchesAndNeverFasterThanTenSeconds( + dms, + )); + test.fails("query-request-logs-next-cursor-reads-the-next-page", ({ dms }) => + assertQueryRequestLogsNextCursorReadsTheNextPage(dms)); +}); diff --git a/tests/helpers/automation.ts b/tests/helpers/automation.ts new file mode 100644 index 0000000..996f19d --- /dev/null +++ b/tests/helpers/automation.ts @@ -0,0 +1,140 @@ +import { z } from "zod"; +import type { Dms } from "./dms"; +import type { Json } from "./json"; + +export const WITH_NODES = { VERIFY_API_AUTOMATION: "1" }; +export const HEALTH = "api.get-api-health"; +export const LOGS = "api.query-request-logs"; +export const UNHEALTHY = "api.route-unhealthy"; +export const TRIGGER_PATH = "/dms/verify-api/nodes/trigger"; +export const INTERVAL_WAIT_MS = 12_000; + +const propertySchema = z.object({ + type: z.union([z.string(), z.array(z.string())]).optional(), + minimum: z.number().optional(), + maximum: z.number().optional(), + default: z.union([z.number(), z.array(z.string())]).optional(), + enum: z.array(z.string()).optional(), + items: z.object({ enum: z.array(z.string()).optional() }).optional(), +}); +const propertiesSchema = z + .object({ + properties: z + .record(z.string(), propertySchema) + .transform((record) => new Map(Object.entries(record))), + }) + .transform((schema) => schema.properties); +const nodeSchema = z.object({ + id: z.string(), + cluster: z.string().nullish(), + inputSchema: propertiesSchema.optional(), + configSchema: propertiesSchema.optional(), + outputSchema: propertiesSchema, +}); +const listedSchema = z.object({ + actions: z.array(z.object({ id: z.string() }).passthrough()), + triggers: z.array(z.object({ id: z.string() }).passthrough()), +}); + +/** The nodes of this module (ids under `api.`) that the playground's provider holds, with their schemas parsed. */ +export function apiNodes(body: Json) { + const listed = listedSchema.parse(body); + const own = (node: { id: string }) => node.id.startsWith("api."); + return { + actions: listed.actions.filter(own).map((node) => nodeSchema.parse(node)), + triggers: listed.triggers.filter(own).map((node) => nodeSchema.parse(node)), + }; +} + +export const breakdownSchema = z.object({ + total: z.number(), + serverErrors: z.number(), + clientErrors: z.number(), + success: z.number(), +}); +export const healthSchema = z.object({ + totalRoutes: z.number(), + failingRoutes: z.number(), + callsToday: z.number(), + errors24h: z.number(), + statusBreakdown: breakdownSchema, +}); + +export const logPageSchema = z.object({ + results: z.array( + z.object({ + _id: z.string(), + uri: z.string(), + statusCode: z.number(), + timestamp: z.string(), + }), + ), + nextCursor: z.string().nullish(), +}); + +const emissionsSchema = z.object({ + emissions: z.array( + z.object({ + routeKey: z.string(), + method: z.string(), + uri: z.string(), + flag: z.string(), + errorRate: z.number(), + requestsLast24h: z.number(), + }), + ), + ageMs: z.number(), +}); +const activationSchema = z.object({ activation: z.string() }); +export const attentionSchema = z.object({ + items: z.array(z.object({}).passthrough()), +}); +export const verdictSchema = z.object({ verdict: z.string() }); +export const lifetimeSchema = z.object({ statisticsLifetimeDays: z.number() }); + +export function byName(left: string, right: string): number { + return left.localeCompare(right); +} + +export async function arm( + dms: Dms, + config: { intervalSeconds: number; flags?: string[] }, +): Promise { + const answer = await dms.api( + "POST", + `${TRIGGER_PATH}/${UNHEALTHY}/activate`, + { + body: config, + as: "anon", + }, + ); + return activationSchema.parse(answer.body).activation; +} + +export async function emitted(dms: Dms, key: string) { + const answer = await dms.api("GET", `${TRIGGER_PATH}/activation/${key}`, { + as: "anon", + }); + const { emissions, ageMs } = emissionsSchema.parse(answer.body); + return { + flags: emissions.map((row) => `${row.uri}:${row.flag}`).toSorted(byName), + rows: emissions, + ageMs, + }; +} + +export async function queryLogs( + dms: Dms, + input: { [key: string]: string | number }, +) { + const answer = await dms.runAction(LOGS, input); + return logPageSchema.parse(answer.body); +} + +export function hasMore(page: z.infer): boolean { + return (page.nextCursor ?? "") !== ""; +} + +export function statusesOf(page: z.infer): number[] { + return [...new Set(page.results.map((row) => row.statusCode))]; +} diff --git a/tests/retention.journeys.test.ts b/tests/retention.journeys.test.ts new file mode 100644 index 0000000..68553d9 --- /dev/null +++ b/tests/retention.journeys.test.ts @@ -0,0 +1,183 @@ +import { resolve } from "node:path"; +import { describe, expect } from "vitest"; +import { z } from "zod"; +import { type Dms, type Answer, test } from "./helpers/dms"; +import { + dayStart, + readDocuments, + seedLogs, + seedStatistics, +} from "./helpers/seed"; + +const MS_PER_DAY = 86_400_000; +const MS_PER_HOUR = 3_600_000; +const CLOCK = resolve(__dirname, "../e2e/clock.cjs"); +const PRUNE_DEADLINE_MS = 90_000; +const PRUNE_POLL_MS = 1000; +const PRUNE_SETTLE_MS = 3000; + +const impactSchema = z.object({ days: z.number(), requests: z.number() }); +const loggedSchema = z.object({ timestamp: z.date() }); +const statisticsSchema = z.object({ + statistics: z.array(z.object({ day: z.number() })), +}); + +function sleep(ms: number): Promise { + return new Promise((done) => setTimeout(done, ms)); +} + +function figures(answer: Answer): number[] { + const { days, requests } = impactSchema.parse(answer.body); + return [days, requests]; +} + +async function assertLoweringARetentionTellsWhatTheNextPruneDeletes( + dms: Dms, +): Promise { + await seedLogs(dms, { daysAgo: 40, count: 3 }); + await seedLogs(dms, { daysAgo: 99.5, count: 2 }); + await seedStatistics(dms, { + method: "GET", + uri: "/fx/ok", + daysAgo: 50, + requests: 4, + }); + await seedStatistics(dms, { + method: "GET", + uri: "/fx/ok", + daysAgo: 70, + requests: 6, + }); + const read = (field: string, days: number) => + dms.mon(`/settings/retention-impact?field=${field}&days=${days}`); + const logs30 = await read("requestLogRetentionDays", 30); + const logs60 = await read("requestLogRetentionDays", 60); + const logs366 = await read("requestLogRetentionDays", 366); + const stats30 = await read("statisticsLifetimeDays", 30); + const stats60 = await read("statisticsLifetimeDays", 60); + const stats90 = await read("statisticsLifetimeDays", 90); + const unknown = await read("nope", 30); + const zero = await read("requestLogRetentionDays", 0); + const over = await read("requestLogRetentionDays", 367); + expect + .soft( + figures(logs30), + "logs older than 30 days: the days back they reach and the requests", + ) + .toEqual([70, 5]); + expect.soft(figures(logs60), "logs older than 60 days").toEqual([40, 2]); + expect.soft(figures(logs366), "logs older than 366 days").toEqual([0, 0]); + expect + .soft( + figures(stats30), + "statistics older than 30 days: the 50 and 70 day old rollups", + ) + .toEqual([2, 10]); + expect + .soft( + figures(stats60), + "statistics older than 60 days: the 70 day old rollup", + ) + .toEqual([1, 6]); + expect + .soft(figures(stats90), "statistics older than 90 days") + .toEqual([0, 0]); + expect.soft(unknown.status, "an unknown field is refused").toBe(400); + expect.soft(zero.status, "0 days is refused").toBe(400); + expect.soft(over.status, "367 days is refused").toBe(400); +} + +async function assertTheHourlyPruneDeletesWhatTheRetentionPassed( + dms: Dms, +): Promise { + await dms.saveSettings({ + requestLogRetentionDays: 20, + statisticsLifetimeDays: 30, + }); + await seedLogs(dms, { daysAgo: 39.5, count: 3 }); + await seedLogs(dms, { + daysAgo: 25, + count: 2, + uri: "/fx/boom", + statusCode: 500, + }); + await seedLogs(dms, { daysAgo: 10, count: 4 }); + await seedLogs(dms, { daysAgo: 1 / 24, count: 1 }); + for (const [daysAgo, requests] of [ + [100, 4], + [40, 6], + [10, 8], + ] as const) + await seedStatistics(dms, { + method: "GET", + uri: "/fx/ok", + daysAgo, + requests, + }); + const impact = await dms.mon( + "/settings/retention-impact?field=requestLogRetentionDays&days=20", + ); + const seeded = () => + readDocuments(dms, "dms-api__request_log", loggedSchema, { + userAgent: "seed", + }); + const before = (await seeded()).length; + const toNextHour = MS_PER_HOUR - (Date.now() % MS_PER_HOUR); + const offset = + (toNextHour > 20_000 ? toNextHour : toNextHour + MS_PER_HOUR) - 12_000; + const previous = { ...dms.backend.extraEnv }; + dms.backend.extraEnv = { + ...previous, + NODE_OPTIONS: `--require ${CLOCK}`, + VERIFY_API_CLOCK_OFFSET_MS: String(offset), + }; + try { + await dms.restart(); + const deadline = Date.now() + PRUNE_DEADLINE_MS; + let after = before; + while (Date.now() < deadline && after > 5) { + await sleep(PRUNE_POLL_MS); + after = (await seeded()).length; + } + await sleep(PRUNE_SETTLE_MS); + } finally { + dms.backend.extraEnv = previous; + } + const ages = (await seeded()) + .map((row) => + Math.round((Date.now() - row.timestamp.getTime()) / MS_PER_DAY), + ) + .toSorted((left, right) => left - right); + const stats = await readDocuments( + dms, + "dms-api__routes_statistics", + statisticsSchema, + { _id: "GET:/fx/ok" }, + ); + const days = (stats[0]?.statistics ?? []) + .map((entry) => entry.day) + .toSorted((left, right) => left - right); + expect + .soft( + figures(impact), + "the impact said 5 requests reaching back 20 days before", + ) + .toEqual([20, 5]); + expect.soft(before, "the logs before").toBe(10); + expect + .soft(ages, "the logs left are the four 10 days old and the hour old one") + .toEqual([0, 10, 10, 10, 10]); + expect + .soft( + days, + "the statistics left are the 10 days old and the 40 days old are gone with the 100 days old", + ) + .toEqual([dayStart(10)]); +} + +describe("retention", () => { + test("lowering-a-retention-tells-what-the-next-prune-deletes", ({ dms }) => + assertLoweringARetentionTellsWhatTheNextPruneDeletes(dms)); + test("the-hourly-prune-deletes-what-the-retention-passed", ({ dms }) => + assertTheHourlyPruneDeletesWhatTheRetentionPassed(dms)); +}); diff --git a/tests/settings.journeys.test.ts b/tests/settings.journeys.test.ts new file mode 100644 index 0000000..66edee5 --- /dev/null +++ b/tests/settings.journeys.test.ts @@ -0,0 +1,247 @@ +import { describe, expect } from "vitest"; +import { z } from "zod"; +import { type Dms, type Fields, test } from "./helpers/dms"; +import type { Json } from "./helpers/json"; + +const DAY = 86_400_000; +const KB = 1024; +const FIFTY_HEADERS = Array.from({ length: 50 }, (_, index) => `x-h${index}`); +const SECRET_HEADERS = [ + "authorization", + "cookie", + "set-cookie", + "x-api-key", + "proxy-authorization", + "X-Authorization", + "x-antelopejs-api-source", +]; +const ORDINARY_HEADERS = [ + "x-request-trace", + "accept-language", + "if-none-match", +]; +const BOUNDS: { field: string; edges: [Json, number][] }[] = [ + { + field: "requestLogRetentionDays", + edges: [ + [0, 400], + [1, 200], + [366, 200], + [367, 400], + [1.5, 400], + ], + }, + { + field: "statisticsLifetimeDays", + edges: [ + [45, 400], + [30, 200], + [60, 200], + [90, 200], + [120, 400], + ], + }, + { + field: "requestLogMaxBodyKb", + edges: [ + [-1, 400], + [0, 200], + [10240, 200], + [10241, 400], + ], + }, + { + field: "requestSlownessThresholdMs", + edges: [ + [49, 400], + [50, 200], + [75, 400], + [100, 200], + [600000, 200], + [600050, 400], + ], + }, + { + field: "scope", + edges: [ + ["own", 200], + ["modules", 200], + ["all", 200], + ["everything", 400], + ], + }, +]; +const REFUSED: { name: string; config: Fields }[] = [ + { name: "statisticsLifetime", config: { statisticsLifetime: 45 * DAY } }, + { + name: "requestSlownessThreshold off its step", + config: { requestSlownessThreshold: 75 }, + }, + { + name: "requestLogMaxBodySize above 10 MB", + config: { requestLogMaxBodySize: 10_241 * KB }, + }, + { + name: "requestLogRetention of 367 days", + config: { requestLogRetention: 367 * DAY }, + }, + { + name: "a secret header", + config: { requestLogCaptureHeaders: ["authorization"] }, + }, +]; + +const thresholdsSchema = z.object({ + requestSlownessThresholdMs: z.number(), + requestLogMaxBodyKb: z.number(), +}); + +async function statusOf(dms: Dms, field: string, value: Json): Promise { + return (await dms.saveSettings({ [field]: value })).status; +} + +function refusedAtStart(outcome: { started: boolean; log: string }) { + return { + started: outcome.started, + said: /\[dms-api\] invalid configuration/.test(outcome.log), + }; +} + +async function assertEveryBoundOfTheFormIsEnforcedOnItsEdge( + dms: Dms, +): Promise { + for (const { field, edges } of BOUNDS) + for (const [value, expected] of edges) + expect + .soft( + await statusOf(dms, field, value), + `${field} = ${JSON.stringify(value)}`, + ) + .toBe(expected); + expect + .soft( + await statusOf(dms, "requestLogCaptureHeaders", FIFTY_HEADERS), + "50 headers are accepted", + ) + .toBe(200); + expect + .soft( + await statusOf(dms, "requestLogCaptureHeaders", [ + ...FIFTY_HEADERS, + "x-one-more", + ]), + "51 headers are refused", + ) + .toBe(400); + for (const header of SECRET_HEADERS) + expect + .soft( + await statusOf(dms, "requestLogCaptureHeaders", [header]), + `${header} is refused`, + ) + .toBe(400); + for (const header of ORDINARY_HEADERS) + expect + .soft( + await statusOf(dms, "requestLogCaptureHeaders", [header]), + `${header} is accepted`, + ) + .toBe(200); +} + +async function assertUseDefaultReturnsAFieldToTheModuleOption( + dms: Dms, +): Promise { + await dms.withConfig( + { requestSlownessThreshold: 500, requestLogMaxBodySize: 2048 }, + async () => { + const read = async () => + thresholdsSchema.parse((await dms.mon("/settings")).body); + const base = await read(); + await dms.saveSettings({ requestSlownessThresholdMs: 250 }); + const overridden = await read(); + await dms.restart(); + const restarted = await read(); + await dms.saveSettings({ requestSlownessThresholdMs: null }); + const back = await read(); + expect + .soft( + base.requestSlownessThresholdMs, + "the module options are read: slow", + ) + .toBe(500); + expect + .soft( + base.requestLogMaxBodyKb, + "the module options are read: body cap in KB", + ) + .toBe(2); + expect + .soft( + overridden.requestSlownessThresholdMs, + "the override wins at once", + ) + .toBe(250); + expect + .soft( + restarted.requestSlownessThresholdMs, + "the override survives a restart", + ) + .toBe(250); + expect + .soft( + back.requestSlownessThresholdMs, + "Use default follows the module option again", + ) + .toBe(500); + expect + .soft(back.requestLogMaxBodyKb, "the other field was never overridden") + .toBe(2); + }, + ); +} + +async function assertTheModuleRefusesToStartWithAValueTheFormWouldRefuse( + dms: Dms, +): Promise { + for (const { name, config } of REFUSED) { + expect + .soft( + refusedAtStart(await dms.startWith(config)), + `${name}: the module stops and says so`, + ) + .toEqual({ started: false, said: true }); + await dms.restart(); + } + const accepted = await dms.startWith({ + statisticsLifetime: 60 * DAY, + requestLogRetention: 14 * DAY, + requestSlownessThreshold: 500, + requestLogMaxBodySize: 0, + requestLogCaptureHeaders: ["x-webhook-signature"], + }); + expect.soft(accepted.started, "the module starts").toBe(true); +} + +async function assertALogRetentionOfNoDayIsRefusedAtStart( + dms: Dms, +): Promise { + expect + .soft( + refusedAtStart(await dms.startWith({ requestLogRetention: 0 })), + "the module stops and says so", + ) + .toEqual({ started: false, said: true }); +} + +describe("settings", () => { + test("every-bound-of-the-form-is-enforced-on-its-edge", ({ dms }) => + assertEveryBoundOfTheFormIsEnforcedOnItsEdge(dms)); + test("use-default-returns-a-field-to-the-module-option", ({ dms }) => + assertUseDefaultReturnsAFieldToTheModuleOption(dms)); + test("the-module-refuses-to-start-with-a-value-the-form-would-refuse", ({ + dms, + }) => assertTheModuleRefusesToStartWithAValueTheFormWouldRefuse(dms)); + test("a-log-retention-of-no-day-is-refused-at-start", ({ dms }) => + assertALogRetentionOfNoDayIsRefusedAtStart(dms)); +}); From a08e6b705f1881fa710d1fc47f86e49023ffeb76 Mon Sep 17 00:00:00 2001 From: Antony Rizzitelli Date: Thu, 19 Nov 2026 08:58:01 +0000 Subject: [PATCH 12/16] test: let knip read the journey tests and name each journey's database after its test --- knip.config.ts | 9 +++------ tests/helpers/dms.ts | 19 +++++++++++++------ 2 files changed, 16 insertions(+), 12 deletions(-) diff --git a/knip.config.ts b/knip.config.ts index e8ce3a3..f8e746b 100644 --- a/knip.config.ts +++ b/knip.config.ts @@ -2,13 +2,10 @@ import { antelopeKnipConfig } from "@antelopejs/tooling-configs/knip"; export default { workspaces: { - /** - * `e2e/seed.mjs` loads `mongodb-memory-server-core` through - * `createRequire`, for the MongoDB driver it ships, and dms-control - * resolves it from the checkout to fetch `mongod`. - */ + /** The journey tests run under their own Vitest config, which Knip does not read. */ ".": antelopeKnipConfig({ - ignoreDependencies: ["mongodb-memory-server-core"], + entry: ["tests/**/*.journeys.test.ts", "tests/helpers/journeys-setup.ts"], + project: ["tests/**/*.ts"], }), /** * The front end ships as source and is built by the DMS loader, which diff --git a/tests/helpers/dms.ts b/tests/helpers/dms.ts index 04eac7e..a713821 100644 --- a/tests/helpers/dms.ts +++ b/tests/helpers/dms.ts @@ -75,12 +75,20 @@ export class Dms { private constructor(readonly backend: Backend) {} - /** Starts a backend on a fresh database and onboards its administrator. */ - static async start(mongoUrl: string): Promise { + /** + * Starts a backend on a fresh database and onboards its administrator. + * The database and the backend's log are named after `label` and a + * random suffix. + */ + static async start(mongoUrl: string, label: string): Promise { + const name = label + .toLowerCase() + .replace(/[^a-z0-9]+/g, "_") + .slice(0, 40); const dms = new Dms( new Backend( mongoUrl, - `journey_${randomUUID().replaceAll("-", "").slice(0, 12)}`, + `${name}_${randomUUID().replaceAll("-", "").slice(0, 8)}`, ), ); await dms.backend.start(); @@ -367,9 +375,8 @@ export class Dms { /** `test` with `dms`: a DMS of the test's own, stopped once the test ends. */ export const test = base.extend<{ dms: Dms }>({ - // oxlint-disable-next-line no-empty-pattern -- Vitest reads a fixture's dependencies from its first parameter's pattern. - dms: async ({}, use) => { - const dms = await Dms.start(inject("mongoUrl")); + dms: async ({ task }, use) => { + const dms = await Dms.start(inject("mongoUrl"), task.name); try { await use(dms); } finally { From 90a0cd697bd7bdd7baedccddd4434be9de211890 Mon Sep 17 00:00:00 2001 From: Antony Rizzitelli Date: Thu, 19 Nov 2026 08:58:41 +0000 Subject: [PATCH 13/16] test(e2e): drop the journey helpers the ported journeys alone used --- e2e/api.mjs | 39 --------------------------------------- e2e/seed.mjs | 46 ---------------------------------------------- 2 files changed, 85 deletions(-) diff --git a/e2e/api.mjs b/e2e/api.mjs index 863f048..3578f5a 100644 --- a/e2e/api.mjs +++ b/e2e/api.mjs @@ -259,18 +259,6 @@ export async function raw(ctx, label, request) { return { folder, status: response.status, headers: received, bytes, text: body, json: parseJson(body) }; } -/** - * Records checks against the evidence folder `raw` made. - * - * @param {Ctx} ctx The journey - * @param {Raw} answer The answer - * @param {Check[]} checks The checks - * @returns {boolean} Whether they all passed - */ -export function recordRaw(ctx, answer, checks) { - return ctx.record(answer.folder, checks); -} - /** * @param {Uint8Array} bytes Some bytes * @returns {string} Their SHA-256, hex @@ -292,33 +280,6 @@ export function saveSettings(ctx, label, values, as) { return mon(ctx, `settings: ${label}`, "/settings", { method: "POST", body: values, as }); } -/** - * Sends one call to the project's routes with the headers, query and body given, and reads the request log it made. - * - * @param {Ctx} ctx The journey - * @param {string} label What the call is - * @param {{ method?: string, path: string, headers?: Record, body?: string }} request What to send - * @returns {Promise<{ status: number, log: Row }>} The status answered and the captured request in full - */ -export async function sendAndRead(ctx, label, request) { - const sent = await raw(ctx, label, request); - const marker = `${request.method ?? "GET"} ${request.path.split("?")[0]}`; - const deadline = Date.now() + 15_000; - /** @type {Row} */ - let found = {}; - while (Date.now() < deadline) { - const listed = await mon(ctx, `${label}: find it`, "/logs?limit=50"); - const row = list(listed.body, "results").find((entry) => entry.requestId === request.headers?.["x-request-id"] || (text(entry, "path") === request.path.split("?")[0] && `${text(entry, "method")} ${text(entry, "path")}` === marker)); - if (row) { - const detail = await mon(ctx, `${label}: open it`, `/logs/${text(row, "_id")}`); - found = isRow(detail.body) ? detail.body : {}; - break; - } - await new Promise((resolve) => setTimeout(resolve, 300)); - } - return { status: sent.status ?? 0, log: found }; -} - /** * A check that a radio, switch or toggle button is on (or off), once it shows. * diff --git a/e2e/seed.mjs b/e2e/seed.mjs index 7f64cb6..b4b3b42 100644 --- a/e2e/seed.mjs +++ b/e2e/seed.mjs @@ -47,17 +47,6 @@ async function withDatabase(ctx, body) { } } -/** - * The first millisecond of the local day `daysAgo` days back, how the module keys its per-day statistics. - * - * @param {number} daysAgo How many days back - * @returns {number} The day - */ -export function dayStart(daysAgo) { - const now = new Date(); - return new Date(now.getFullYear(), now.getMonth(), now.getDate() - daysAgo).getTime(); -} - /** * Inserts request logs aged `daysAgo` days, the way the capture stores them, and keeps what was written as evidence. * @@ -103,38 +92,3 @@ export async function seedLogs(ctx, label, batch) { await withDatabase(ctx, (db) => db.collection("dms-api__request_log").insertMany(rows)); } -/** - * Adds a day of statistics to a route, `daysAgo` days back. - * - * @param {Ctx} ctx The journey - * @param {string} label What the day is - * @param {{ method: string, uri: string, daysAgo: number, requests: number, serverErrors?: number, clientErrors?: number, averageMs?: number }} day The day - * @returns {Promise} - */ -export async function seedStatistics(ctx, label, day) { - const average = day.averageMs ?? 10; - const entry = { - day: dayStart(day.daysAgo), - requestsCount: day.requests, - averageResponseTime: average, - minResponseTime: average, - maxResponseTime: average, - totalResponseTime: average * day.requests, - clientErrorsCount: day.clientErrors ?? 0, - serverErrorsCount: day.serverErrors ?? 0, - timedRequestsCount: day.requests, - latencyHistogram: [day.requests, ...Array.from({ length: 17 }, () => 0)], - }; - const folder = ctx.step(`seed statistics ${label}`); - mkdirSync(folder, { recursive: true }); - writeFileSync(join(folder, "seed.json"), `${JSON.stringify({ day, entry }, null, 2)}\n`); - await withDatabase(ctx, (db) => - db - .collection("dms-api__routes_statistics") - .updateOne( - { _id: `${day.method}:${day.uri}` }, - { $push: { statistics: entry }, $setOnInsert: { method: day.method, uri: day.uri, revision: randomUUID(), _instance: null } }, - { upsert: true }, - ), - ); -} From 4aec4d63c0c3f0f7666c293810a64fcef0c6dc30 Mon Sep 17 00:00:00 2001 From: Antony Rizzitelli Date: Thu, 19 Nov 2026 09:00:11 +0000 Subject: [PATCH 14/16] ci: start the checks on the ready pull request From a3623e507a91784c6b4dae2936a633a1359bfaf6 Mon Sep 17 00:00:00 2001 From: Antony Rizzitelli Date: Thu, 19 Nov 2026 09:03:53 +0000 Subject: [PATCH 15/16] build: type-check the playground's declaration files so e2e:build passes the guardrails --- .agents/skills/verify-dms-api/playground/tsconfig.build.json | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.agents/skills/verify-dms-api/playground/tsconfig.build.json b/.agents/skills/verify-dms-api/playground/tsconfig.build.json index 1392878..3b31236 100644 --- a/.agents/skills/verify-dms-api/playground/tsconfig.build.json +++ b/.agents/skills/verify-dms-api/playground/tsconfig.build.json @@ -11,8 +11,7 @@ "experimentalDecorators": true, "emitDecoratorMetadata": true, "useDefineForClassFields": true, - "types": ["node"], - "skipLibCheck": true + "types": ["node"] }, "include": ["./src/**/*.ts"] } From 50eb0d57ff4dd7c8141ae1d66fcfdea53c5e163e Mon Sep 17 00:00:00 2001 From: Antony Rizzitelli Date: Thu, 19 Nov 2026 09:05:54 +0000 Subject: [PATCH 16/16] fix(test): read only the restarted backend's log, and build the frontend layer on dms-frontend 0.11.8 --- package.json | 2 +- pnpm-lock.yaml | 10 +++++----- tests/automation.journeys.test.ts | 3 ++- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/package.json b/package.json index d886f18..c23636f 100644 --- a/package.json +++ b/package.json @@ -68,7 +68,7 @@ "devDependencies": { "@antelopejs/core": "~1.24.5", "@antelopejs/dms": "0.21.4-next.939.g641c7fa", - "@antelopejs/dms-frontend": ">=0.11.4 <0.12.0", + "@antelopejs/dms-frontend": ">=0.11.8 <0.12.0", "@antelopejs/interface-data-api": ">=0.8.0 <0.9.0", "@antelopejs/tooling-configs": ">=0.8.9 <0.9.0", "@playwright/test": "~1.65.0", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 6963dfd..4e4d241 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -40,8 +40,8 @@ importers: specifier: 0.21.4-next.939.g641c7fa version: 0.21.4-next.939.g641c7fa(@antelopejs/core@1.24.5)(@playwright/test@1.65.1)(typescript@6.0.3) '@antelopejs/dms-frontend': - specifier: '>=0.11.4 <0.12.0' - version: 0.11.6(@antelopejs/core@1.24.5) + specifier: '>=0.11.8 <0.12.0' + version: 0.11.8(@antelopejs/core@1.24.5) '@antelopejs/interface-data-api': specifier: '>=0.8.0 <0.9.0' version: 0.8.0(@antelopejs/interface-api-util@0.3.0(@antelopejs/interface-api@0.3.3(@antelopejs/interface-core@0.5.1))(@antelopejs/interface-core@0.5.1))(@antelopejs/interface-api@0.3.3(@antelopejs/interface-core@0.5.1))(@antelopejs/interface-core@0.5.1)(@antelopejs/interface-database-decorators@0.5.2(@antelopejs/interface-api@0.3.3(@antelopejs/interface-core@0.5.1))(@antelopejs/interface-core@0.5.1)(@antelopejs/interface-database@0.4.1(@antelopejs/interface-core@0.5.1)))(@antelopejs/interface-database@0.4.1(@antelopejs/interface-core@0.5.1)) @@ -111,8 +111,8 @@ packages: engines: {node: ^20.19.0 || >=22.12.0} hasBin: true - '@antelopejs/dms-frontend@0.11.6': - resolution: {integrity: sha512-1ZBkTp94wUElsVWUjEL4YRVzg3AJpiPOd7ezp73JbiQ1LHjPYb7yvI5MWL7YGxBAaaP60nFLMD08fm+2QKHzEw==} + '@antelopejs/dms-frontend@0.11.8': + resolution: {integrity: sha512-owMawDaP66LA0j7OlBTJ4gebBC0TFpn8xveEpsv9kUyy8ltkPn1DPoJHyJ+7QNxq5iO56m1V49+sW0j2M/LpvQ==} engines: {node: '>=20.19.0'} hasBin: true peerDependencies: @@ -3573,7 +3573,7 @@ snapshots: semver: 7.8.5 typescript: 5.9.3 - '@antelopejs/dms-frontend@0.11.6(@antelopejs/core@1.24.5)': + '@antelopejs/dms-frontend@0.11.8(@antelopejs/core@1.24.5)': dependencies: '@antelopejs/core': 1.24.5 '@intlify/message-compiler': 11.4.20 diff --git a/tests/automation.journeys.test.ts b/tests/automation.journeys.test.ts index 2c897f3..065938b 100644 --- a/tests/automation.journeys.test.ts +++ b/tests/automation.journeys.test.ts @@ -26,8 +26,9 @@ import { seedLogs, seedStatistics } from "./helpers/seed"; async function assertWithoutTheAutomationModuleTheRegistrationIsASilentNoOp( dms: Dms, ): Promise { + await dms.backend.stop(); const from = dms.backend.logSize(); - await dms.restart(); + await dms.backend.start(); await dms.traffic([{ path: "/fx/ok", count: 1 }]); const lines = dms.backend.logSince(from).split("\n"); const health = await dms.mon("/overview/health");