From 1f2a026d16d605b3ece06f5c0b5171b20c9bf20f Mon Sep 17 00:00:00 2001 From: Paulo Date: Mon, 9 Nov 2026 14:14:43 +0200 Subject: [PATCH] DRU-1112 -- Run Chat end to end with Playwright on every pull request The On Pull Request E2E workflow builds the Druks image from the pull request and installs it with scripts/install.sh on the docker provider. Only the model is fake: a scripted Anthropic API in the Playwright process serves api.anthropic.com with a certificate from a test CA. The box, the secrets proxy, the sandbox image, and the claude CLI are real. The lane runs on Linux and on macOS with Docker Desktop. The mock listens on every interface, and E2E_MOCK_ADDRESS names the address where a container reaches the host. Three specs cover two turns and a reload, a Stop in the middle of a stream, and a Druks restart in the middle of a turn. Each spec checks that the page shows each saved message once and in order, that each reply is what the mock streamed, that the second turn carries the first, and that every model request carries the real token. The lane found three bugs, and this change fixes them: - druks setup refused the [providers.anthropic] table that docs/configuration.md tells an operator to add, so the next install failed. Setup now keeps a [providers] table, like [apps] and [services]. - The bridge saves the session files as soon as a turn ends, but the claude CLI writes its transcript 60 to 120 ms later. So each saved archive missed the latest turn, and a fast turn failed with "The session files are missing". Chat now runs the CLI with CLAUDE_CODE_EAGER_FLUSH, and the CLI writes the transcript before it reports the result. - Only hosted installs ran the drukbox janitor. On the docker provider nothing deleted an expired sandbox, and a template build that a drukbox restart cut off stayed "building" with no end, so each later build waited on it. Every install shape now runs the janitor. --- .github/workflows/on-pull-request-e2e.yml | 73 +++++++++++ AGENTS.md | 8 ++ backend/druks/harnesses/claude.py | 7 +- backend/druks/setup_env.py | 3 +- backend/tests/test_chat.py | 5 +- backend/tests/test_setup_env.py | 15 ++- deploy/compose.yaml | 7 +- docs/deployment.md | 2 +- docs/development.md | 48 ++++++++ docs/full-local.md | 10 +- frontend/.gitignore | 4 + frontend/e2e/anthropic.ts | 143 ++++++++++++++++++++++ frontend/e2e/chat.spec.ts | 61 +++++++++ frontend/e2e/compose.override.yaml | 23 ++++ frontend/e2e/fixtures.ts | 135 ++++++++++++++++++++ frontend/e2e/install-stack.sh | 41 +++++++ frontend/e2e/sign-in.setup.ts | 9 ++ frontend/package-lock.json | 46 +++++++ frontend/package.json | 2 + frontend/playwright.config.ts | 26 ++++ frontend/tsconfig.node.json | 4 +- scripts/install.sh | 2 +- 22 files changed, 653 insertions(+), 21 deletions(-) create mode 100644 .github/workflows/on-pull-request-e2e.yml create mode 100644 frontend/e2e/anthropic.ts create mode 100644 frontend/e2e/chat.spec.ts create mode 100644 frontend/e2e/compose.override.yaml create mode 100644 frontend/e2e/fixtures.ts create mode 100755 frontend/e2e/install-stack.sh create mode 100644 frontend/e2e/sign-in.setup.ts create mode 100644 frontend/playwright.config.ts diff --git a/.github/workflows/on-pull-request-e2e.yml b/.github/workflows/on-pull-request-e2e.yml new file mode 100644 index 000000000..db1c984ec --- /dev/null +++ b/.github/workflows/on-pull-request-e2e.yml @@ -0,0 +1,73 @@ +name: On Pull Request E2E + +# Chat end to end on the stack that scripts/install.sh makes from this pull request. Only the +# model is fake: a scripted Anthropic mock in the Playwright process. A required check must +# always report, so the workflow has no path filter. +on: + pull_request: + branches: [main, "codex/**"] + +concurrency: + group: on-pull-request-e2e-${{ github.event.pull_request.number }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + chat: + runs-on: ubuntu-latest + timeout-minutes: 45 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + # The merge commit and its base, to see what the pull request changes. + fetch-depth: 2 + - uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4.4.1 + - uses: docker/build-push-action@c3c9e263c25d99ce0380d002d59b67737d91b0dc # v7.4.0 + with: + context: . + platforms: linux/amd64 + push: false + load: true + tags: ghcr.io/czpython/druks:e2e + cache-from: type=gha,scope=backend + - name: Install the stack + env: + DRUKS_REPO: ${{ github.event.pull_request.head.repo.full_name }} + DRUKS_REF: ${{ github.event.pull_request.head.sha }} + DRUKS_TAG: e2e + DRUKS_SKIP_PULL: "1" + run: | + setup=() + # The docker provider pulls a sandbox image first, so the image of this pull + # request takes a tag that no registry has. + if ! git diff --quiet HEAD^ -- deploy/sandbox; then + docker build --load --target sandbox --tag druks-sandbox:e2e deploy/sandbox + setup=(--set sandbox.image=druks-sandbox:e2e) + fi + bash frontend/e2e/install-stack.sh "${setup[@]}" + - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 + with: + node-version: "22" + cache: npm + cache-dependency-path: frontend/package-lock.json + - run: npm ci + working-directory: frontend + - run: npx playwright install --with-deps chromium + working-directory: frontend + # The mock serves api.anthropic.com on port 443. + - run: sudo sysctl -w net.ipv4.ip_unprivileged_port_start=443 + - run: npm run test:e2e + working-directory: frontend + - if: failure() + run: cd ~/druks && docker compose logs --no-color --timestamps > "$RUNNER_TEMP/compose.log" + - if: failure() + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: e2e-chat + path: | + frontend/playwright-report/ + frontend/test-results/ + ${{ runner.temp }}/compose.log diff --git a/AGENTS.md b/AGENTS.md index eacc4fdb8..67041eb04 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -147,6 +147,14 @@ npm --prefix frontend test npm --prefix frontend run build ``` +Run the Chat end-to-end lane against the stack that +`frontend/e2e/install-stack.sh` installs. `docs/development.md` describes a +local run: + +```bash +npm --prefix frontend run test:e2e +``` + Run the calls server's tests from `druks-calls/`: ```bash diff --git a/backend/druks/harnesses/claude.py b/backend/druks/harnesses/claude.py index fced5fe1e..aa3a3660f 100644 --- a/backend/druks/harnesses/claude.py +++ b/backend/druks/harnesses/claude.py @@ -73,7 +73,12 @@ def get_acp_session( "settingSources": [], "strictMcpConfig": True, } - env: dict[str, str | None] = {"CLAUDE_CODE_DISABLE_AUTO_MEMORY": "1"} + # The bridge saves the session files as soon as a turn ends. Claude Code buffers its + # transcript, so without an eager flush the files miss the turn. + env: dict[str, str | None] = { + "CLAUDE_CODE_DISABLE_AUTO_MEMORY": "1", + "CLAUDE_CODE_EAGER_FLUSH": "1", + } if identity and include_apps: # Claude Code loads a person's connectors only from CLAUDE_CODE_OAUTH_TOKEN, and # ANTHROPIC_API_KEY outranks it. The bridge fills the placeholder and removes a diff --git a/backend/druks/setup_env.py b/backend/druks/setup_env.py index 44630571d..073dbacea 100644 --- a/backend/druks/setup_env.py +++ b/backend/druks/setup_env.py @@ -374,8 +374,9 @@ def _canonical_config(raw: dict[str, Any]) -> dict[str, Any]: # Tables the runtime reads from druks.toml itself: ``[apps.]`` and # ``[services.]`` switch an app or a service off, or make a service managed. +# ``[providers.]`` selects how a person signs in to a provider. # Setup renders nothing from them and writes them back as they are. -_RUNTIME_TABLES = ("apps", "services") +_RUNTIME_TABLES = ("apps", "services", "providers") def _provider_tables(sandbox: dict[str, Any]) -> dict[str, Any]: diff --git a/backend/tests/test_chat.py b/backend/tests/test_chat.py index 15f956613..a1fb40ad2 100644 --- a/backend/tests/test_chat.py +++ b/backend/tests/test_chat.py @@ -787,7 +787,7 @@ async def follow_turn(session, conversation, message, bridge, naming): } } }, - "env": {"CLAUDE_CODE_DISABLE_AUTO_MEMORY": "1"}, + "env": {"CLAUDE_CODE_DISABLE_AUTO_MEMORY": "1", "CLAUDE_CODE_EAGER_FLUSH": "1"}, "files": {}, "mode": "bypassPermissions", "model": "claude-opus-4-7", @@ -815,7 +815,7 @@ async def follow_turn(session, conversation, message, bridge, naming): } } }, - "env": {"CLAUDE_CODE_DISABLE_AUTO_MEMORY": "1"}, + "env": {"CLAUDE_CODE_DISABLE_AUTO_MEMORY": "1", "CLAUDE_CODE_EAGER_FLUSH": "1"}, "files": {}, "mode": "bypassPermissions", "model": "claude-opus-4-7", @@ -955,6 +955,7 @@ def test_a_claude_subscription_moves_its_login_for_the_adapter(): ) assert session["env"] == { "CLAUDE_CODE_DISABLE_AUTO_MEMORY": "1", + "CLAUDE_CODE_EAGER_FLUSH": "1", "CLAUDE_CODE_OAUTH_TOKEN": "${ANTHROPIC_AUTH_TOKEN}", "CLAUDE_CODE_OAUTH_SCOPES": "user:inference user:mcp_servers", "ANTHROPIC_AUTH_TOKEN": None, diff --git a/backend/tests/test_setup_env.py b/backend/tests/test_setup_env.py index 2cbdc24eb..57bad2938 100644 --- a/backend/tests/test_setup_env.py +++ b/backend/tests/test_setup_env.py @@ -475,9 +475,9 @@ def test_nested_operator_content_is_refused(tmp_path): def test_app_and_service_tables_pass_through_setup(tmp_path): - """The runtime reads ``[apps.]`` and ``[services.]`` from druks.toml - itself. Setup renders nothing from them and keeps them, so a platform that - writes the file can still render .env with it.""" + """The runtime reads ``[apps.]``, ``[services.]``, and + ``[providers.]`` from druks.toml itself. Setup renders nothing from them and + keeps them, so a platform that writes the file can still render .env with it.""" env_path = tmp_path / ".env" _run(env_path, provider="docker") toml_path = tmp_path / "druks.toml" @@ -486,11 +486,18 @@ def test_app_and_service_tables_pass_through_setup(tmp_path): + "\n[apps.software_factory]\nenabled = false\n\n[services.jira]\nenabled = false\n" ) - _run(env_path, set_values=("urls.webhook_host=hooks.example.com",)) + _run( + env_path, + set_values=( + "urls.webhook_host=hooks.example.com", + "providers.anthropic.sign_in=setup_token", + ), + ) config = _read_toml(toml_path) assert config["apps"] == {"software_factory": {"enabled": False}} assert config["services"] == {"jira": {"enabled": False}} + assert config["providers"] == {"anthropic": {"sign_in": "setup_token"}} assert not any("JIRA" in key or "SOFTWARE_FACTORY" in key for key in read_env(env_path)) diff --git a/deploy/compose.yaml b/deploy/compose.yaml index 4224de417..8bad5189c 100644 --- a/deploy/compose.yaml +++ b/deploy/compose.yaml @@ -1,7 +1,7 @@ # The Druks stack: web, Postgres, Redis, the drukbox sandbox control plane, its # secrets exchange, and the secrets proxy. This one file applies to all install # shapes. install.sh writes COMPOSE_PROFILES to .env. The "hosted" profile -# starts the Caddy edge and the janitor. The "proxy" profile starts the secrets +# starts the Caddy edge. The "proxy" profile starts the secrets # proxy, on every provider but docker-sbx. The docker-sbx provider adds one # overlay file, because its drukbox has different mounts. All services use the # host network. Services connect to Postgres and Redis on 127.0.0.1. drukbox @@ -108,11 +108,10 @@ services: # The periodic janitor. drukbox has no scheduler, and this loop with sleep # does the work. A real timer belongs in the drukbox service or image, not - # in compose. Only hosted installs run it. A local box reaps on the next - # request. + # in compose. Every shape runs it. Nothing else deletes an expired sandbox + # or fails a template build that a drukbox restart cut off. drukbox-janitor: <<: *drukbox - profiles: ["hosted"] entrypoint: - sh - -c diff --git a/docs/deployment.md b/docs/deployment.md index 864b8e041..b26569dd5 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -19,7 +19,7 @@ Caddy. See [Full local](full-local.md). A **remote** installation uses each other `DRUKS_PROVIDER` value. It enables `COMPOSE_PROFILES=hosted,proxy`. The `hosted` profile includes a remote Drukbox -control plane, the periodic janitor, and stock Caddy. Caddy supplies the +control plane and stock Caddy. Caddy supplies the identity edge and proxy with a bind-mounted Caddyfile. The `proxy` profile includes the secrets proxy. diff --git a/docs/development.md b/docs/development.md index f4d1b9670..fcc439686 100644 --- a/docs/development.md +++ b/docs/development.md @@ -180,6 +180,54 @@ Mintlify builds `docs/` directly. Its GitHub App owns deployments and pull-request previews. The repository does not require a documentation-specific GitHub Actions workflow. +### Chat end to end + +The `On Pull Request E2E` workflow runs Chat end to end with Playwright on every +pull request. It builds the Druks image from the pull request and installs it +with `scripts/install.sh` on the docker provider. Only the model is fake: a +scripted Anthropic API in the Playwright process answers for +`api.anthropic.com`. The box, the secrets proxy, the sandbox image, and the +`claude` CLI are real. + +Run the lane on Linux or on macOS with Docker Desktop. The stack uses the host +network and the ports of an installed Druks, so stop any other Druks stack +first, and the development database of `deploy/compose.dev.yaml`. The mock +listens on port 443, and the containers reach it at the Docker bridge address, +`172.17.0.1`. + +1. Build the image of your checkout: + + ```bash + docker build -t ghcr.io/czpython/druks:e2e . + ``` + +2. Install the stack into `~/druks`. `DRUKS_REF` names a pushed commit whose + `deploy/` files match your checkout: + + ```bash + DRUKS_REF="$(git rev-parse origin/main)" DRUKS_TAG=e2e DRUKS_SKIP_PULL=1 bash frontend/e2e/install-stack.sh + ``` + +3. On Linux, let Node listen on port 443. Then run the suite: + + ```bash + sudo sysctl -w net.ipv4.ip_unprivileged_port_start=443 + npm --prefix frontend ci + npx --prefix frontend playwright install --with-deps chromium + npm --prefix frontend run test:e2e + ``` + +On macOS, enable host networking as +[Docker Desktop](container-runtimes/docker-desktop.md) describes. A container +reaches the mock at the host address of Docker Desktop, not at the bridge +address. Before step 2, run `export E2E_MOCK_ADDRESS=192.168.65.254`, and keep +it set for each `docker compose` command in the install directory. Skip the +`sysctl` command. + +Set `DRUKS_INSTALL_DIR` to install elsewhere, and set it for the suite too. The +suite needs no cleanup between runs. To start over, run +`docker compose down --volumes` in the install directory and remove it. + ## Working with sandboxes Backend tests mock most provider boundaries. For a real local sandbox, run diff --git a/docs/full-local.md b/docs/full-local.md index c369754c4..6e7360352 100644 --- a/docs/full-local.md +++ b/docs/full-local.md @@ -52,12 +52,12 @@ The local shape needs no authored values, so the first run goes all the way: - It creates `~/.config/druks/harnesses` for optional harness configuration. - It generates the database password and the stored-secret key. - It pulls images and applies migrations. -- It starts Druks, Postgres, Redis, Drukbox, the secrets exchange, and the - secrets proxy. Drukbox listens on `127.0.0.1:8780`. The exchange listens on - `127.0.0.1:8781`. The proxy listens on `172.17.0.1:8880`, where sandbox - containers reach it. +- It starts Druks, Postgres, Redis, Drukbox, its janitor, the secrets exchange, + and the secrets proxy. Drukbox listens on `127.0.0.1:8780`. The exchange + listens on `127.0.0.1:8781`. The proxy listens on `172.17.0.1:8880`, where + sandbox containers reach it. - It uses `COMPOSE_FILE=compose.yaml:compose.override.yaml` and - `COMPOSE_PROFILES=proxy`, without Caddy or the janitor. + `COMPOSE_PROFILES=proxy`, without Caddy. Drukbox controls sandboxes through the mounted `/var/run/docker.sock`. The installer records the group ID of the socket in `.env`. This value gives the diff --git a/frontend/.gitignore b/frontend/.gitignore index a547bf36d..3313b681f 100644 --- a/frontend/.gitignore +++ b/frontend/.gitignore @@ -22,3 +22,7 @@ dist-ssr *.njsproj *.sln *.sw? + +# Playwright +/test-results/ +/playwright-report/ diff --git a/frontend/e2e/anthropic.ts b/frontend/e2e/anthropic.ts new file mode 100644 index 000000000..10ef56f7f --- /dev/null +++ b/frontend/e2e/anthropic.ts @@ -0,0 +1,143 @@ +import type { IncomingMessage, ServerResponse } from 'node:http' +import { createServer } from 'node:https' + +/** The setup token the test account signs in with. The box holds only its placeholder. */ +export const TOKEN = 'sk-ant-oat01-druks-e2e' +/** The fixed answer to the request that names a new conversation. */ +export const TITLE = 'Druks end to end' + +/** Holds a scripted stream at its place until the test opens it. */ +export class Gate { + open = () => {} + readonly opened = new Promise((resolve) => { this.open = resolve }) +} + +/** The text deltas of one reply, with gates between them. */ +export type Script = (string | Gate)[] + +interface Message { + role: 'user' | 'assistant' + content: string | { type: string, text?: string }[] +} + +/** One Messages API request and the text deltas the mock streamed back. */ +export interface Completion { + /** The marker of the turn, or empty for a side request. */ + marker: string + authorization: string + messages: Message[] + streamed: string[] +} + +export interface Anthropic { + completions: Completion[] + /** Answer the turn whose message starts with the marker. */ + script: (marker: string, script: Script) => void + close: () => Promise +} + +/** A test's message starts with U-. */ +export const USER_MARKER = /^U\d+-[a-z0-9]+\b/ +// The start of NAMING_PROMPT in backend/druks/chat/constants.py. +const NAMING_PROMPT = 'Name this conversation' + +function textsOf(message: Message): string[] { + if (typeof message.content === 'string') return [message.content] + return message.content.map((block) => block.text ?? '') +} + +export function textOf(message: Message): string { + return textsOf(message).join('\n') +} + +/** The marker of the turn that a request with this last user message runs. History repeats + * older markers, so only the last user message counts. A side request, such as the title of a + * session, quotes the message inside its own text, so the marker must start a text block. */ +function markerOf(prompt: Message): string { + return textsOf(prompt).map((text) => text.match(USER_MARKER)?.[0]).find(Boolean) ?? '' +} + +/** An HTTPS Messages API for api.anthropic.com. It logs every request and answers an unknown + * path with 404. */ +export async function startAnthropic({ cert, key }: { cert: string, key: string }): Promise { + const completions: Completion[] = [] + const scripts = new Map() + + async function answer(request: IncomingMessage, response: ServerResponse) { + console.log(`anthropic: ${request.method} ${request.url}`) + const path = new URL(request.url ?? '/', 'https://api.anthropic.com').pathname + if (request.method !== 'POST' || path !== '/v1/messages') { + return sendError(response, 404, 'not_found_error', `The mock does not serve ${request.method} ${path}.`) + } + const chunks: Buffer[] = [] + for await (const chunk of request) chunks.push(chunk) + const body = JSON.parse(Buffer.concat(chunks).toString()) + if (!body.stream) return sendError(response, 400, 'invalid_request_error', 'The mock answers streaming requests only.') + const messages: Message[] = body.messages + const prompt = messages.findLast((message) => message.role === 'user')! + const marker = markerOf(prompt) + const script = marker ? scripts.get(marker) : [textOf(prompt).includes(NAMING_PROMPT) ? TITLE : 'OK'] + if (!script) return sendError(response, 400, 'invalid_request_error', `No script for ${marker}.`) + const completion: Completion = { marker, authorization: request.headers.authorization ?? '', messages, streamed: [] } + completions.push(completion) + await stream(response, body.model, script, completion) + } + + const server = createServer({ cert, key }, (request, response) => { + answer(request, response).catch((error) => { + console.error('anthropic:', error) + response.destroy() + }) + }) + await new Promise((resolve, reject) => { + server.once('error', reject) + // Every interface: a container reaches this process at the Docker bridge address on Linux, + // and through the host address of Docker Desktop on macOS. + server.listen(443, '0.0.0.0', resolve) + }) + return { + completions, + script: (marker, script) => { scripts.set(marker, script) }, + close: () => new Promise((resolve) => { + server.closeAllConnections() + server.close(() => resolve()) + }), + } +} + +function sendError(response: ServerResponse, status: number, type: string, message: string) { + response.writeHead(status, { 'content-type': 'application/json' }) + response.end(JSON.stringify({ type: 'error', error: { type, message } })) +} + +async function stream(response: ServerResponse, model: string, script: Script, completion: Completion) { + const closed = new Promise((resolve) => response.once('close', resolve)) + // A Stop hangs up mid-stream, and the rest of the script goes nowhere. + function send(type: string, data: object) { + if (response.destroyed) return false + response.write(`event: ${type}\ndata: ${JSON.stringify({ type, ...data })}\n\n`) + return true + } + response.writeHead(200, { 'content-type': 'text/event-stream', 'cache-control': 'no-cache' }) + send('message_start', { + message: { + id: `msg_e2e_${Math.random().toString(36).slice(2)}`, + type: 'message', + role: 'assistant', + model, + content: [], + stop_reason: null, + stop_sequence: null, + usage: { input_tokens: 1, output_tokens: 1, cache_creation_input_tokens: 0, cache_read_input_tokens: 0 }, + }, + }) + send('content_block_start', { index: 0, content_block: { type: 'text', text: '' } }) + for (const chunk of script) { + if (chunk instanceof Gate) await Promise.race([chunk.opened, closed]) + else if (send('content_block_delta', { index: 0, delta: { type: 'text_delta', text: chunk } })) completion.streamed.push(chunk) + } + send('content_block_stop', { index: 0 }) + send('message_delta', { delta: { stop_reason: 'end_turn', stop_sequence: null }, usage: { output_tokens: completion.streamed.length } }) + send('message_stop', {}) + response.end() +} diff --git a/frontend/e2e/chat.spec.ts b/frontend/e2e/chat.spec.ts new file mode 100644 index 000000000..29be5902c --- /dev/null +++ b/frontend/e2e/chat.spec.ts @@ -0,0 +1,61 @@ +import { + compose, expect, expectChatFacts, Gate, getConversation, listBoxes, markTurns, send, startConversation, test, TITLE, +} from './fixtures' + +test('two turns share one box, and a reload shows each message once', async ({ page, request, anthropic, violations }) => { + const [first, second] = markTurns(2) + anthropic.script(first.user, [`${first.reply} Hello`, ' from the first turn.']) + anthropic.script(second.user, [`${second.reply} Hello`, ' again.']) + const log = page.getByRole('log', { name: 'Messages' }) + + const id = await startConversation(page, `${first.user} Say hello.`) + await expect(log).toContainText(`${first.reply} Hello from the first turn.`) + await expect(page.getByRole('heading', { name: TITLE })).toBeVisible() + const boxes = listBoxes() + expect(boxes).toHaveLength(1) + + await send(page, `${second.user} Say hello again.`) + await expect(log).toContainText(`${second.reply} Hello again.`) + expect(listBoxes()).toEqual(boxes) + + await page.reload() + await expectChatFacts({ page, request, anthropic, violations }, id) +}) + +test('Stop keeps the streamed text, and the next turn runs', async ({ page, request, anthropic, violations }) => { + const [first, second] = markTurns(2) + anthropic.script(first.user, [`${first.reply} Counting`, new Gate()]) + anthropic.script(second.user, [`${second.reply} Still here.`]) + const log = page.getByRole('log', { name: 'Messages' }) + + const id = await startConversation(page, `${first.user} Count to a thousand.`) + await expect(log).toContainText(`${first.reply} Counting`) + await page.getByRole('button', { name: 'Stop' }).click() + await expect.poll(async () => { + const { messages } = await getConversation(request, id) + return messages.find((message) => message.body.startsWith(first.user))?.state + }).toBe('cancelled') + + await send(page, `${second.user} Are you there?`) + await expect(log).toContainText(`${second.reply} Still here.`) + await expectChatFacts({ page, request, anthropic, violations }, id) +}) + +test('a turn that runs through a Druks restart shows its reply once', async ({ page, request, anthropic, violations }) => { + const [first, second] = markTurns(2) + const restarted = new Gate() + anthropic.script(first.user, [`${first.reply} Before the restart,`, restarted, ' and after it.']) + anthropic.script(second.user, [`${second.reply} Back again.`]) + const log = page.getByRole('log', { name: 'Messages' }) + + const id = await startConversation(page, `${first.user} Keep talking.`) + await expect(log).toContainText(`${first.reply} Before the restart,`) + compose('restart', 'web') + await expect.poll(() => request.get('/health').then((response) => response.ok(), () => false)).toBe(true) + restarted.open() + await expect(log).toContainText(`${first.reply} Before the restart, and after it.`) + + await send(page, `${second.user} Still there?`) + await expect(log).toContainText(`${second.reply} Back again.`) + await expectChatFacts({ page, request, anthropic, violations }, id) +}) diff --git a/frontend/e2e/compose.override.yaml b/frontend/e2e/compose.override.yaml new file mode 100644 index 000000000..d0583ed57 --- /dev/null +++ b/frontend/e2e/compose.override.yaml @@ -0,0 +1,23 @@ +# The E2E lane copies this file into the install directory before scripts/install.sh runs, +# so Compose loads it in every command. api.anthropic.com resolves to the scripted mock in +# the Playwright process. The mock serves a certificate from the test CA in ./e2e. +# E2E_MOCK_ADDRESS is the address where a container reaches the host: the Docker bridge +# address on Linux, and 192.168.65.254 on Docker Desktop. +services: + drukbox-proxy: + extra_hosts: + - "api.anthropic.com:${E2E_MOCK_ADDRESS:-172.17.0.1}" + environment: + # The proxy refuses a private destination outside these networks. + SECRETS_PROXY_PRIVATE_NETWORKS: "${E2E_MOCK_ADDRESS:-172.17.0.1}/32" + volumes: + - ./e2e/ca.pem:/e2e/ca.pem:ro + # mitmdump loads this file at its start. It names the test CA as the trusted upstream CA. + - ./e2e/mitmproxy.yaml:/home/mitmproxy/.mitmproxy/config.yaml:ro + web: + extra_hosts: + - "api.anthropic.com:${E2E_MOCK_ADDRESS:-172.17.0.1}" + environment: + SSL_CERT_FILE: /e2e/ca.pem + volumes: + - ./e2e/ca.pem:/e2e/ca.pem:ro diff --git a/frontend/e2e/fixtures.ts b/frontend/e2e/fixtures.ts new file mode 100644 index 000000000..8935713bd --- /dev/null +++ b/frontend/e2e/fixtures.ts @@ -0,0 +1,135 @@ +import { execFileSync } from 'node:child_process' +import { readFileSync } from 'node:fs' +import { homedir } from 'node:os' +import { join } from 'node:path' + +import { test as base, expect, type APIRequestContext, type Page } from '@playwright/test' + +import type { Conversation } from '../src/chat/state' +import { startAnthropic, textOf, TOKEN, USER_MARKER, type Anthropic } from './anthropic' + +export { expect } +export { Gate, TITLE } from './anthropic' + +/** The compose project that scripts/install.sh made, with the test CA in its e2e folder. */ +export const installDir = process.env.DRUKS_INSTALL_DIR ?? join(homedir(), 'druks') + +declare global { + interface Window { + reportRenderViolation: (detail: string) => void + } +} + +// U marks a turn's message and A starts its reply, so a marker gives its place on the page. +const MARKER = /\b[UA]\d+-[a-z0-9]+\b/g + +export const test = base.extend<{ violations: Set }, { anthropic: Anthropic }>({ + // Playwright reads the fixtures that a fixture needs from this pattern, also when it is empty. + // eslint-disable-next-line no-empty-pattern + anthropic: [async ({}, provide) => { + const certificates = join(installDir, 'e2e') + const anthropic = await startAnthropic({ + cert: readFileSync(join(certificates, 'api.anthropic.com.pem'), 'utf8'), + key: readFileSync(join(certificates, 'api.anthropic.com.key'), 'utf8'), + }) + await provide(anthropic) + await anthropic.close() + }, { scope: 'worker' }], + violations: async ({ page }, provide) => { + const violations = new Set() + await page.exposeFunction('reportRenderViolation', (detail: string) => violations.add(detail)) + await page.addInitScript(watchMessages, MARKER.source) + await provide(violations) + }, +}) + +/** The markers of a test's turns. Each test owns its nonce, so tests share no state. */ +export function markTurns(count: number) { + const nonce = Math.random().toString(36).slice(2, 8) + return Array.from({ length: count }, (_, index) => ({ user: `U${index + 1}-${nonce}`, reply: `A${index + 1}-${nonce}` })) +} + +export async function send(page: Page, body: string) { + await page.getByRole('textbox', { name: 'Message', exact: true }).fill(body) + await page.getByRole('button', { name: 'Send', exact: true }).click() +} + +/** Open a new conversation with its first message, and return its id. */ +export async function startConversation(page: Page, body: string): Promise { + await page.goto('/chat/new') + await send(page, body) + await page.waitForURL(/\/chat\/[0-9a-f-]{36}$/) + return new URL(page.url()).pathname.split('/').at(-1)! +} + +export async function getConversation(request: APIRequestContext, id: string): Promise { + const response = await request.get(`/api/chat/conversations/${id}`) + expect(response.status()).toBe(200) + return response.json() +} + +export function compose(...args: string[]) { + execFileSync('docker', ['compose', ...args], { cwd: installDir, stdio: 'inherit' }) +} + +/** The running boxes that drukbox's docker provider made. */ +export function listBoxes(): string[] { + const output = execFileSync('docker', ['ps', '--quiet', '--filter', 'label=managed-by=drukbox'], { encoding: 'utf8' }) + return output.split('\n').filter(Boolean) +} + +/** Runs in the page. On every DOM change, it reports a marker that shows twice or out of order. */ +function watchMessages(pattern: string) { + const rank = (marker: string) => Number(marker.slice(1, marker.indexOf('-'))) * 2 + Number(marker[0] === 'A') + new MutationObserver(() => { + const log = document.querySelector('[role="log"][aria-label="Messages"]') + const markers: string[] = log?.innerText.match(new RegExp(pattern, 'g')) ?? [] + markers.forEach((marker, index) => { + if (markers.indexOf(marker) !== index) window.reportRenderViolation(`${marker} shows twice`) + else if (index > 0 && rank(markers[index - 1]) > rank(marker)) window.reportRenderViolation(`${marker} shows below ${markers[index - 1]}`) + }) + }).observe(document, { childList: true, subtree: true, characterData: true }) +} + +const normalize = (text: string) => text.replace(/\s+/g, ' ').trim() + +/** The page shows each saved message once and in order. Each reply is what the mock streamed for + * its turn. The second turn's request carries the first turn. Every request carries the real token. */ +export async function expectChatFacts( + { page, request, anthropic, violations }: { page: Page, request: APIRequestContext, anthropic: Anthropic, violations: Set }, + id: string, +) { + const log = page.getByRole('log', { name: 'Messages' }) + await expect.poll(async () => { + const { messages } = await getConversation(request, id) + const text = normalize(await log.innerText()) + let position = -1 + return messages.flatMap((message) => { + const body = normalize(message.body) + const count = text.split(body).length - 1 + if (count !== 1) return [`${message.role} "${body}" shows ${count} times`] + if (text.indexOf(body) < position) return [`${message.role} "${body}" shows above an older message`] + position = text.indexOf(body) + return [] + }) + }, { message: 'each saved message shows once, in order' }).toEqual([]) + expect([...violations], 'no marker showed twice or out of order').toEqual([]) + + const conversation = await getConversation(request, id) + const turns = conversation.messages.filter((message) => message.role === 'user').map((message) => { + const marker = message.body.match(USER_MARKER)![0] + const requests = anthropic.completions.filter((completion) => completion.marker === marker) + expect(requests, `one model request for ${marker}`).toHaveLength(1) + const reply = conversation.messages.find((saved) => saved.replyTo === message.id) + expect(reply?.body, `the reply to ${marker} is what the mock streamed`).toBe(requests[0].streamed.join('')) + return { message, reply: reply!, request: requests[0] } + }) + + const [first, second] = turns + const history = second.request.messages.slice(0, -1).map(textOf).join('\n') + expect(history, 'the second turn carries the first message').toContain(first.message.body) + expect(history, 'the second turn carries the first reply').toContain(first.reply.body) + + const authorizations = new Set(anthropic.completions.map((completion) => completion.authorization)) + expect([...authorizations], 'every model request carries the real token').toEqual([`Bearer ${TOKEN}`]) +} diff --git a/frontend/e2e/install-stack.sh b/frontend/e2e/install-stack.sh new file mode 100755 index 000000000..c33bdbaef --- /dev/null +++ b/frontend/e2e/install-stack.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env bash +# Install the stack that the E2E lane drives: scripts/install.sh on the docker provider, +# with a test CA and the E2E Compose override, then the identity and sign-in settings. +# +# It takes the env knobs of scripts/install.sh. Its arguments go to `druks setup`, for +# example --set sandbox.image=druks-sandbox:e2e. +set -euo pipefail + +e2e="$(cd "$(dirname "$0")" && pwd)" +install_dir="${DRUKS_INSTALL_DIR:-$HOME/druks}" + +# The mock serves api.anthropic.com with a certificate from this CA. Each extension is +# explicit, because OpenSSL and the LibreSSL of macOS add different ones by default. +mkdir -p "$install_dir/e2e" +cd "$install_dir/e2e" +openssl req -x509 -newkey rsa:2048 -nodes -days 2 -subj "/CN=Druks E2E CA" \ + -addext "basicConstraints=critical,CA:TRUE" -addext "keyUsage=critical,keyCertSign" \ + -addext "subjectKeyIdentifier=hash" -keyout ca.key -out ca.pem +openssl req -newkey rsa:2048 -nodes -subj "/CN=api.anthropic.com" \ + -keyout api.anthropic.com.key -out api.anthropic.com.csr +openssl x509 -req -in api.anthropic.com.csr -days 2 -CA ca.pem -CAkey ca.key -CAcreateserial \ + -extfile <(printf 'subjectAltName=DNS:api.anthropic.com\nbasicConstraints=CA:FALSE\nauthorityKeyIdentifier=keyid\n') \ + -out api.anthropic.com.pem +echo "ssl_verify_upstream_trusted_ca: /e2e/ca.pem" > mitmproxy.yaml + +# The installer keeps an override that exists. +cp "$e2e/compose.override.yaml" "$install_dir/compose.override.yaml" +DRUKS_PROVIDER=docker bash "$e2e/../../scripts/install.sh" + +cd "$install_dir" +docker run --rm --user "$(id -u):$(id -g)" -v "$install_dir:/bootstrap" \ + "ghcr.io/czpython/druks:$(sed -n 's/^DRUKS_TAG=//p' .env)" \ + druks setup /bootstrap/.env --home "$HOME" \ + --set identity.mode=header --set identity.header=X-Forwarded-Email \ + --set providers.anthropic.sign_in=setup_token "$@" +docker compose restart web +# drukbox pulls the sandbox image inside the request that creates a template, and a first +# pull can take longer than that request may. +image="$(sed -n 's/^image = "\(.*\)"$/\1/p' druks.toml)" +docker image inspect "$image" >/dev/null 2>&1 || docker pull "$image" +docker compose exec -T web druks sandboxes build diff --git a/frontend/e2e/sign-in.setup.ts b/frontend/e2e/sign-in.setup.ts new file mode 100644 index 000000000..2af6efa3b --- /dev/null +++ b/frontend/e2e/sign-in.setup.ts @@ -0,0 +1,9 @@ +import { expect, test as setup } from '@playwright/test' + +import { TOKEN } from './anthropic' + +// Druks checks a setup token's prefix only, so the fake token needs no network call. +setup('the test account signs in to Anthropic', async ({ request }) => { + const response = await request.post('/api/providers/anthropic/sign-in/token', { data: { token: TOKEN } }) + expect(response.status(), await response.text()).toBe(200) +}) diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 9350f38c6..b8c483d36 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -27,6 +27,7 @@ "devDependencies": { "@agentclientprotocol/sdk": "2.1.0", "@eslint/js": "^10.0.1", + "@playwright/test": "^1.65.0", "@testing-library/dom": "^10.4.2", "@testing-library/react": "^16.3.3", "@types/node": "^24.19.2", @@ -879,6 +880,22 @@ "url": "https://github.com/sponsors/oxc-project" } }, + "node_modules/@playwright/test": { + "version": "1.65.0", + "resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.65.0.tgz", + "integrity": "sha512-iGURLGuKDLakSEl7Pgfv8h7f+XLT+V861Wfhz0zlEl5nyFuKUopdkFBPKqTllTUp06aOwPuH8iuUEb3yp3MpbQ==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "playwright": "1.65.0" + }, + "bin": { + "playwright": "cli.js" + }, + "engines": { + "node": ">=20" + } + }, "node_modules/@redocly/ajv": { "version": "8.11.2", "resolved": "https://registry.npmjs.org/@redocly/ajv/-/ajv-8.11.2.tgz", @@ -5051,6 +5068,35 @@ "url": "https://github.com/sponsors/jonschlinkert" } }, + "node_modules/playwright": { + "version": "1.65.0", + "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.65.0.tgz", + "integrity": "sha512-F3AMWcR44qMBt4kq+6Kb6mX7bjGXshktKFOPULd7reED6ZJWevjbNRqysP7q+AATE9kdPEZBibex1BYsvpVoeA==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "playwright-core": "1.65.0" + }, + "bin": { + "playwright": "cli.js" + }, + "engines": { + "node": ">=20" + } + }, + "node_modules/playwright-core": { + "version": "1.65.0", + "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.65.0.tgz", + "integrity": "sha512-vNSKgH6YqtN7cdT/L334zo0a+YwkrpicyrHJlWWPIm/Q+Nn2SWInil01oT0mbu96U4GgoRtCYQD9CzJQgju+lA==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "playwright-core": "cli.js" + }, + "engines": { + "node": ">=20" + } + }, "node_modules/pluralize": { "version": "8.0.0", "resolved": "https://registry.npmjs.org/pluralize/-/pluralize-8.0.0.tgz", diff --git a/frontend/package.json b/frontend/package.json index e6d7e9841..f21f3d731 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -10,6 +10,7 @@ "preview": "vite preview", "test": "vitest run", "test:watch": "vitest", + "test:e2e": "playwright test", "types:openapi": "openapi-typescript http://127.0.0.1:8000/openapi.json -o src/api/openapi.ts" }, "dependencies": { @@ -32,6 +33,7 @@ "devDependencies": { "@agentclientprotocol/sdk": "2.1.0", "@eslint/js": "^10.0.1", + "@playwright/test": "^1.65.0", "@testing-library/dom": "^10.4.2", "@testing-library/react": "^16.3.3", "@types/node": "^24.19.2", diff --git a/frontend/playwright.config.ts b/frontend/playwright.config.ts new file mode 100644 index 000000000..982d9f133 --- /dev/null +++ b/frontend/playwright.config.ts @@ -0,0 +1,26 @@ +import { defineConfig } from '@playwright/test' + +// The suite drives a stack that scripts/install.sh made, with only the model faked. See +// docs/development.md. +export default defineConfig({ + testDir: 'e2e', + forbidOnly: true, + retries: 0, + workers: 1, + timeout: 180_000, + // A first turn boots the account's box. + expect: { timeout: 60_000 }, + reporter: [['list'], ['html', { open: 'never' }]], + use: { + // The endpoint of a local install: the chat socket accepts this origin only. + baseURL: 'http://127.0.0.1:8001', + // The edge identity that druks.toml names in identity.header. + extraHTTPHeaders: { 'X-Forwarded-Email': 'e2e@druks.test' }, + trace: 'retain-on-failure', + screenshot: 'only-on-failure', + }, + projects: [ + { name: 'sign-in', testMatch: /\.setup\.ts$/ }, + { name: 'chat', dependencies: ['sign-in'] }, + ], +}) diff --git a/frontend/tsconfig.node.json b/frontend/tsconfig.node.json index d3c52ea64..eec70ad77 100644 --- a/frontend/tsconfig.node.json +++ b/frontend/tsconfig.node.json @@ -2,7 +2,7 @@ "compilerOptions": { "tsBuildInfoFile": "./node_modules/.tmp/tsconfig.node.tsbuildinfo", "target": "es2023", - "lib": ["ES2023"], + "lib": ["ES2023", "DOM"], "module": "esnext", "types": ["node"], "skipLibCheck": true, @@ -20,5 +20,5 @@ "erasableSyntaxOnly": true, "noFallthroughCasesInSwitch": true }, - "include": ["vite.config.ts"] + "include": ["vite.config.ts", "playwright.config.ts", "e2e"] } diff --git a/scripts/install.sh b/scripts/install.sh index f22d3f128..a2e2e1853 100755 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -152,7 +152,7 @@ main() { # Shape selection, written to .env. After that, `docker compose` in this # directory starts the services of the shape. The `hosted` profile adds the - # Caddy edge and the janitor. The `proxy` profile adds the secrets proxy on + # Caddy edge. The `proxy` profile adds the secrets proxy on # every provider but docker-sbx, where sbx swaps the placeholders itself. # A `docker` box runs without Caddy, with the dashboard on :8001. docker-sbx # also layers the overlay, which connects drukbox to the host sandboxd, and