diff --git a/.github/workflows/on-pull-request-e2e.yml b/.github/workflows/on-pull-request-e2e.yml new file mode 100644 index 00000000..db1c984e --- /dev/null +++ b/.github/workflows/on-pull-request-e2e.yml @@ -0,0 +1,73 @@ +name: On Pull Request E2E + +# Chat end to end on the stack that scripts/install.sh makes from this pull request. Only the +# model is fake: a scripted Anthropic mock in the Playwright process. A required check must +# always report, so the workflow has no path filter. +on: + pull_request: + branches: [main, "codex/**"] + +concurrency: + group: on-pull-request-e2e-${{ github.event.pull_request.number }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + chat: + runs-on: ubuntu-latest + timeout-minutes: 45 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + # The merge commit and its base, to see what the pull request changes. + fetch-depth: 2 + - uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4.4.1 + - uses: docker/build-push-action@c3c9e263c25d99ce0380d002d59b67737d91b0dc # v7.4.0 + with: + context: . + platforms: linux/amd64 + push: false + load: true + tags: ghcr.io/czpython/druks:e2e + cache-from: type=gha,scope=backend + - name: Install the stack + env: + DRUKS_REPO: ${{ github.event.pull_request.head.repo.full_name }} + DRUKS_REF: ${{ github.event.pull_request.head.sha }} + DRUKS_TAG: e2e + DRUKS_SKIP_PULL: "1" + run: | + setup=() + # The docker provider pulls a sandbox image first, so the image of this pull + # request takes a tag that no registry has. + if ! git diff --quiet HEAD^ -- deploy/sandbox; then + docker build --load --target sandbox --tag druks-sandbox:e2e deploy/sandbox + setup=(--set sandbox.image=druks-sandbox:e2e) + fi + bash frontend/e2e/install-stack.sh "${setup[@]}" + - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 + with: + node-version: "22" + cache: npm + cache-dependency-path: frontend/package-lock.json + - run: npm ci + working-directory: frontend + - run: npx playwright install --with-deps chromium + working-directory: frontend + # The mock serves api.anthropic.com on port 443. + - run: sudo sysctl -w net.ipv4.ip_unprivileged_port_start=443 + - run: npm run test:e2e + working-directory: frontend + - if: failure() + run: cd ~/druks && docker compose logs --no-color --timestamps > "$RUNNER_TEMP/compose.log" + - if: failure() + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: e2e-chat + path: | + frontend/playwright-report/ + frontend/test-results/ + ${{ runner.temp }}/compose.log diff --git a/AGENTS.md b/AGENTS.md index eacc4fdb..67041eb0 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -147,6 +147,14 @@ npm --prefix frontend test npm --prefix frontend run build ``` +Run the Chat end-to-end lane against the stack that +`frontend/e2e/install-stack.sh` installs. `docs/development.md` describes a +local run: + +```bash +npm --prefix frontend run test:e2e +``` + Run the calls server's tests from `druks-calls/`: ```bash diff --git a/backend/druks/harnesses/claude.py b/backend/druks/harnesses/claude.py index fced5fe1..aa3a3660 100644 --- a/backend/druks/harnesses/claude.py +++ b/backend/druks/harnesses/claude.py @@ -73,7 +73,12 @@ def get_acp_session( "settingSources": [], "strictMcpConfig": True, } - env: dict[str, str | None] = {"CLAUDE_CODE_DISABLE_AUTO_MEMORY": "1"} + # The bridge saves the session files as soon as a turn ends. Claude Code buffers its + # transcript, so without an eager flush the files miss the turn. + env: dict[str, str | None] = { + "CLAUDE_CODE_DISABLE_AUTO_MEMORY": "1", + "CLAUDE_CODE_EAGER_FLUSH": "1", + } if identity and include_apps: # Claude Code loads a person's connectors only from CLAUDE_CODE_OAUTH_TOKEN, and # ANTHROPIC_API_KEY outranks it. The bridge fills the placeholder and removes a diff --git a/backend/druks/setup_env.py b/backend/druks/setup_env.py index 44630571..073dbace 100644 --- a/backend/druks/setup_env.py +++ b/backend/druks/setup_env.py @@ -374,8 +374,9 @@ def _canonical_config(raw: dict[str, Any]) -> dict[str, Any]: # Tables the runtime reads from druks.toml itself: ``[apps.]`` and # ``[services.]`` switch an app or a service off, or make a service managed. +# ``[providers.]`` selects how a person signs in to a provider. # Setup renders nothing from them and writes them back as they are. -_RUNTIME_TABLES = ("apps", "services") +_RUNTIME_TABLES = ("apps", "services", "providers") def _provider_tables(sandbox: dict[str, Any]) -> dict[str, Any]: diff --git a/backend/tests/test_chat.py b/backend/tests/test_chat.py index 15f95661..a1fb40ad 100644 --- a/backend/tests/test_chat.py +++ b/backend/tests/test_chat.py @@ -787,7 +787,7 @@ async def follow_turn(session, conversation, message, bridge, naming): } } }, - "env": {"CLAUDE_CODE_DISABLE_AUTO_MEMORY": "1"}, + "env": {"CLAUDE_CODE_DISABLE_AUTO_MEMORY": "1", "CLAUDE_CODE_EAGER_FLUSH": "1"}, "files": {}, "mode": "bypassPermissions", "model": "claude-opus-4-7", @@ -815,7 +815,7 @@ async def follow_turn(session, conversation, message, bridge, naming): } } }, - "env": {"CLAUDE_CODE_DISABLE_AUTO_MEMORY": "1"}, + "env": {"CLAUDE_CODE_DISABLE_AUTO_MEMORY": "1", "CLAUDE_CODE_EAGER_FLUSH": "1"}, "files": {}, "mode": "bypassPermissions", "model": "claude-opus-4-7", @@ -955,6 +955,7 @@ def test_a_claude_subscription_moves_its_login_for_the_adapter(): ) assert session["env"] == { "CLAUDE_CODE_DISABLE_AUTO_MEMORY": "1", + "CLAUDE_CODE_EAGER_FLUSH": "1", "CLAUDE_CODE_OAUTH_TOKEN": "${ANTHROPIC_AUTH_TOKEN}", "CLAUDE_CODE_OAUTH_SCOPES": "user:inference user:mcp_servers", "ANTHROPIC_AUTH_TOKEN": None, diff --git a/backend/tests/test_setup_env.py b/backend/tests/test_setup_env.py index 2cbdc24e..57bad293 100644 --- a/backend/tests/test_setup_env.py +++ b/backend/tests/test_setup_env.py @@ -475,9 +475,9 @@ def test_nested_operator_content_is_refused(tmp_path): def test_app_and_service_tables_pass_through_setup(tmp_path): - """The runtime reads ``[apps.]`` and ``[services.]`` from druks.toml - itself. Setup renders nothing from them and keeps them, so a platform that - writes the file can still render .env with it.""" + """The runtime reads ``[apps.]``, ``[services.]``, and + ``[providers.]`` from druks.toml itself. Setup renders nothing from them and + keeps them, so a platform that writes the file can still render .env with it.""" env_path = tmp_path / ".env" _run(env_path, provider="docker") toml_path = tmp_path / "druks.toml" @@ -486,11 +486,18 @@ def test_app_and_service_tables_pass_through_setup(tmp_path): + "\n[apps.software_factory]\nenabled = false\n\n[services.jira]\nenabled = false\n" ) - _run(env_path, set_values=("urls.webhook_host=hooks.example.com",)) + _run( + env_path, + set_values=( + "urls.webhook_host=hooks.example.com", + "providers.anthropic.sign_in=setup_token", + ), + ) config = _read_toml(toml_path) assert config["apps"] == {"software_factory": {"enabled": False}} assert config["services"] == {"jira": {"enabled": False}} + assert config["providers"] == {"anthropic": {"sign_in": "setup_token"}} assert not any("JIRA" in key or "SOFTWARE_FACTORY" in key for key in read_env(env_path)) diff --git a/deploy/compose.yaml b/deploy/compose.yaml index 4224de41..8bad5189 100644 --- a/deploy/compose.yaml +++ b/deploy/compose.yaml @@ -1,7 +1,7 @@ # The Druks stack: web, Postgres, Redis, the drukbox sandbox control plane, its # secrets exchange, and the secrets proxy. This one file applies to all install # shapes. install.sh writes COMPOSE_PROFILES to .env. The "hosted" profile -# starts the Caddy edge and the janitor. The "proxy" profile starts the secrets +# starts the Caddy edge. The "proxy" profile starts the secrets # proxy, on every provider but docker-sbx. The docker-sbx provider adds one # overlay file, because its drukbox has different mounts. All services use the # host network. Services connect to Postgres and Redis on 127.0.0.1. drukbox @@ -108,11 +108,10 @@ services: # The periodic janitor. drukbox has no scheduler, and this loop with sleep # does the work. A real timer belongs in the drukbox service or image, not - # in compose. Only hosted installs run it. A local box reaps on the next - # request. + # in compose. Every shape runs it. Nothing else deletes an expired sandbox + # or fails a template build that a drukbox restart cut off. drukbox-janitor: <<: *drukbox - profiles: ["hosted"] entrypoint: - sh - -c diff --git a/docs/deployment.md b/docs/deployment.md index 864b8e04..b26569dd 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -19,7 +19,7 @@ Caddy. See [Full local](full-local.md). A **remote** installation uses each other `DRUKS_PROVIDER` value. It enables `COMPOSE_PROFILES=hosted,proxy`. The `hosted` profile includes a remote Drukbox -control plane, the periodic janitor, and stock Caddy. Caddy supplies the +control plane and stock Caddy. Caddy supplies the identity edge and proxy with a bind-mounted Caddyfile. The `proxy` profile includes the secrets proxy. diff --git a/docs/development.md b/docs/development.md index f4d1b967..fcc43968 100644 --- a/docs/development.md +++ b/docs/development.md @@ -180,6 +180,54 @@ Mintlify builds `docs/` directly. Its GitHub App owns deployments and pull-request previews. The repository does not require a documentation-specific GitHub Actions workflow. +### Chat end to end + +The `On Pull Request E2E` workflow runs Chat end to end with Playwright on every +pull request. It builds the Druks image from the pull request and installs it +with `scripts/install.sh` on the docker provider. Only the model is fake: a +scripted Anthropic API in the Playwright process answers for +`api.anthropic.com`. The box, the secrets proxy, the sandbox image, and the +`claude` CLI are real. + +Run the lane on Linux or on macOS with Docker Desktop. The stack uses the host +network and the ports of an installed Druks, so stop any other Druks stack +first, and the development database of `deploy/compose.dev.yaml`. The mock +listens on port 443, and the containers reach it at the Docker bridge address, +`172.17.0.1`. + +1. Build the image of your checkout: + + ```bash + docker build -t ghcr.io/czpython/druks:e2e . + ``` + +2. Install the stack into `~/druks`. `DRUKS_REF` names a pushed commit whose + `deploy/` files match your checkout: + + ```bash + DRUKS_REF="$(git rev-parse origin/main)" DRUKS_TAG=e2e DRUKS_SKIP_PULL=1 bash frontend/e2e/install-stack.sh + ``` + +3. On Linux, let Node listen on port 443. Then run the suite: + + ```bash + sudo sysctl -w net.ipv4.ip_unprivileged_port_start=443 + npm --prefix frontend ci + npx --prefix frontend playwright install --with-deps chromium + npm --prefix frontend run test:e2e + ``` + +On macOS, enable host networking as +[Docker Desktop](container-runtimes/docker-desktop.md) describes. A container +reaches the mock at the host address of Docker Desktop, not at the bridge +address. Before step 2, run `export E2E_MOCK_ADDRESS=192.168.65.254`, and keep +it set for each `docker compose` command in the install directory. Skip the +`sysctl` command. + +Set `DRUKS_INSTALL_DIR` to install elsewhere, and set it for the suite too. The +suite needs no cleanup between runs. To start over, run +`docker compose down --volumes` in the install directory and remove it. + ## Working with sandboxes Backend tests mock most provider boundaries. For a real local sandbox, run diff --git a/docs/full-local.md b/docs/full-local.md index c369754c..6e736035 100644 --- a/docs/full-local.md +++ b/docs/full-local.md @@ -52,12 +52,12 @@ The local shape needs no authored values, so the first run goes all the way: - It creates `~/.config/druks/harnesses` for optional harness configuration. - It generates the database password and the stored-secret key. - It pulls images and applies migrations. -- It starts Druks, Postgres, Redis, Drukbox, the secrets exchange, and the - secrets proxy. Drukbox listens on `127.0.0.1:8780`. The exchange listens on - `127.0.0.1:8781`. The proxy listens on `172.17.0.1:8880`, where sandbox - containers reach it. +- It starts Druks, Postgres, Redis, Drukbox, its janitor, the secrets exchange, + and the secrets proxy. Drukbox listens on `127.0.0.1:8780`. The exchange + listens on `127.0.0.1:8781`. The proxy listens on `172.17.0.1:8880`, where + sandbox containers reach it. - It uses `COMPOSE_FILE=compose.yaml:compose.override.yaml` and - `COMPOSE_PROFILES=proxy`, without Caddy or the janitor. + `COMPOSE_PROFILES=proxy`, without Caddy. Drukbox controls sandboxes through the mounted `/var/run/docker.sock`. The installer records the group ID of the socket in `.env`. This value gives the diff --git a/frontend/.gitignore b/frontend/.gitignore index a547bf36..3313b681 100644 --- a/frontend/.gitignore +++ b/frontend/.gitignore @@ -22,3 +22,7 @@ dist-ssr *.njsproj *.sln *.sw? + +# Playwright +/test-results/ +/playwright-report/ diff --git a/frontend/e2e/anthropic.ts b/frontend/e2e/anthropic.ts new file mode 100644 index 00000000..10ef56f7 --- /dev/null +++ b/frontend/e2e/anthropic.ts @@ -0,0 +1,143 @@ +import type { IncomingMessage, ServerResponse } from 'node:http' +import { createServer } from 'node:https' + +/** The setup token the test account signs in with. The box holds only its placeholder. */ +export const TOKEN = 'sk-ant-oat01-druks-e2e' +/** The fixed answer to the request that names a new conversation. */ +export const TITLE = 'Druks end to end' + +/** Holds a scripted stream at its place until the test opens it. */ +export class Gate { + open = () => {} + readonly opened = new Promise((resolve) => { this.open = resolve }) +} + +/** The text deltas of one reply, with gates between them. */ +export type Script = (string | Gate)[] + +interface Message { + role: 'user' | 'assistant' + content: string | { type: string, text?: string }[] +} + +/** One Messages API request and the text deltas the mock streamed back. */ +export interface Completion { + /** The marker of the turn, or empty for a side request. */ + marker: string + authorization: string + messages: Message[] + streamed: string[] +} + +export interface Anthropic { + completions: Completion[] + /** Answer the turn whose message starts with the marker. */ + script: (marker: string, script: Script) => void + close: () => Promise +} + +/** A test's message starts with U-. */ +export const USER_MARKER = /^U\d+-[a-z0-9]+\b/ +// The start of NAMING_PROMPT in backend/druks/chat/constants.py. +const NAMING_PROMPT = 'Name this conversation' + +function textsOf(message: Message): string[] { + if (typeof message.content === 'string') return [message.content] + return message.content.map((block) => block.text ?? '') +} + +export function textOf(message: Message): string { + return textsOf(message).join('\n') +} + +/** The marker of the turn that a request with this last user message runs. History repeats + * older markers, so only the last user message counts. A side request, such as the title of a + * session, quotes the message inside its own text, so the marker must start a text block. */ +function markerOf(prompt: Message): string { + return textsOf(prompt).map((text) => text.match(USER_MARKER)?.[0]).find(Boolean) ?? '' +} + +/** An HTTPS Messages API for api.anthropic.com. It logs every request and answers an unknown + * path with 404. */ +export async function startAnthropic({ cert, key }: { cert: string, key: string }): Promise { + const completions: Completion[] = [] + const scripts = new Map() + + async function answer(request: IncomingMessage, response: ServerResponse) { + console.log(`anthropic: ${request.method} ${request.url}`) + const path = new URL(request.url ?? '/', 'https://api.anthropic.com').pathname + if (request.method !== 'POST' || path !== '/v1/messages') { + return sendError(response, 404, 'not_found_error', `The mock does not serve ${request.method} ${path}.`) + } + const chunks: Buffer[] = [] + for await (const chunk of request) chunks.push(chunk) + const body = JSON.parse(Buffer.concat(chunks).toString()) + if (!body.stream) return sendError(response, 400, 'invalid_request_error', 'The mock answers streaming requests only.') + const messages: Message[] = body.messages + const prompt = messages.findLast((message) => message.role === 'user')! + const marker = markerOf(prompt) + const script = marker ? scripts.get(marker) : [textOf(prompt).includes(NAMING_PROMPT) ? TITLE : 'OK'] + if (!script) return sendError(response, 400, 'invalid_request_error', `No script for ${marker}.`) + const completion: Completion = { marker, authorization: request.headers.authorization ?? '', messages, streamed: [] } + completions.push(completion) + await stream(response, body.model, script, completion) + } + + const server = createServer({ cert, key }, (request, response) => { + answer(request, response).catch((error) => { + console.error('anthropic:', error) + response.destroy() + }) + }) + await new Promise((resolve, reject) => { + server.once('error', reject) + // Every interface: a container reaches this process at the Docker bridge address on Linux, + // and through the host address of Docker Desktop on macOS. + server.listen(443, '0.0.0.0', resolve) + }) + return { + completions, + script: (marker, script) => { scripts.set(marker, script) }, + close: () => new Promise((resolve) => { + server.closeAllConnections() + server.close(() => resolve()) + }), + } +} + +function sendError(response: ServerResponse, status: number, type: string, message: string) { + response.writeHead(status, { 'content-type': 'application/json' }) + response.end(JSON.stringify({ type: 'error', error: { type, message } })) +} + +async function stream(response: ServerResponse, model: string, script: Script, completion: Completion) { + const closed = new Promise((resolve) => response.once('close', resolve)) + // A Stop hangs up mid-stream, and the rest of the script goes nowhere. + function send(type: string, data: object) { + if (response.destroyed) return false + response.write(`event: ${type}\ndata: ${JSON.stringify({ type, ...data })}\n\n`) + return true + } + response.writeHead(200, { 'content-type': 'text/event-stream', 'cache-control': 'no-cache' }) + send('message_start', { + message: { + id: `msg_e2e_${Math.random().toString(36).slice(2)}`, + type: 'message', + role: 'assistant', + model, + content: [], + stop_reason: null, + stop_sequence: null, + usage: { input_tokens: 1, output_tokens: 1, cache_creation_input_tokens: 0, cache_read_input_tokens: 0 }, + }, + }) + send('content_block_start', { index: 0, content_block: { type: 'text', text: '' } }) + for (const chunk of script) { + if (chunk instanceof Gate) await Promise.race([chunk.opened, closed]) + else if (send('content_block_delta', { index: 0, delta: { type: 'text_delta', text: chunk } })) completion.streamed.push(chunk) + } + send('content_block_stop', { index: 0 }) + send('message_delta', { delta: { stop_reason: 'end_turn', stop_sequence: null }, usage: { output_tokens: completion.streamed.length } }) + send('message_stop', {}) + response.end() +} diff --git a/frontend/e2e/chat.spec.ts b/frontend/e2e/chat.spec.ts new file mode 100644 index 00000000..29be5902 --- /dev/null +++ b/frontend/e2e/chat.spec.ts @@ -0,0 +1,61 @@ +import { + compose, expect, expectChatFacts, Gate, getConversation, listBoxes, markTurns, send, startConversation, test, TITLE, +} from './fixtures' + +test('two turns share one box, and a reload shows each message once', async ({ page, request, anthropic, violations }) => { + const [first, second] = markTurns(2) + anthropic.script(first.user, [`${first.reply} Hello`, ' from the first turn.']) + anthropic.script(second.user, [`${second.reply} Hello`, ' again.']) + const log = page.getByRole('log', { name: 'Messages' }) + + const id = await startConversation(page, `${first.user} Say hello.`) + await expect(log).toContainText(`${first.reply} Hello from the first turn.`) + await expect(page.getByRole('heading', { name: TITLE })).toBeVisible() + const boxes = listBoxes() + expect(boxes).toHaveLength(1) + + await send(page, `${second.user} Say hello again.`) + await expect(log).toContainText(`${second.reply} Hello again.`) + expect(listBoxes()).toEqual(boxes) + + await page.reload() + await expectChatFacts({ page, request, anthropic, violations }, id) +}) + +test('Stop keeps the streamed text, and the next turn runs', async ({ page, request, anthropic, violations }) => { + const [first, second] = markTurns(2) + anthropic.script(first.user, [`${first.reply} Counting`, new Gate()]) + anthropic.script(second.user, [`${second.reply} Still here.`]) + const log = page.getByRole('log', { name: 'Messages' }) + + const id = await startConversation(page, `${first.user} Count to a thousand.`) + await expect(log).toContainText(`${first.reply} Counting`) + await page.getByRole('button', { name: 'Stop' }).click() + await expect.poll(async () => { + const { messages } = await getConversation(request, id) + return messages.find((message) => message.body.startsWith(first.user))?.state + }).toBe('cancelled') + + await send(page, `${second.user} Are you there?`) + await expect(log).toContainText(`${second.reply} Still here.`) + await expectChatFacts({ page, request, anthropic, violations }, id) +}) + +test('a turn that runs through a Druks restart shows its reply once', async ({ page, request, anthropic, violations }) => { + const [first, second] = markTurns(2) + const restarted = new Gate() + anthropic.script(first.user, [`${first.reply} Before the restart,`, restarted, ' and after it.']) + anthropic.script(second.user, [`${second.reply} Back again.`]) + const log = page.getByRole('log', { name: 'Messages' }) + + const id = await startConversation(page, `${first.user} Keep talking.`) + await expect(log).toContainText(`${first.reply} Before the restart,`) + compose('restart', 'web') + await expect.poll(() => request.get('/health').then((response) => response.ok(), () => false)).toBe(true) + restarted.open() + await expect(log).toContainText(`${first.reply} Before the restart, and after it.`) + + await send(page, `${second.user} Still there?`) + await expect(log).toContainText(`${second.reply} Back again.`) + await expectChatFacts({ page, request, anthropic, violations }, id) +}) diff --git a/frontend/e2e/compose.override.yaml b/frontend/e2e/compose.override.yaml new file mode 100644 index 00000000..d0583ed5 --- /dev/null +++ b/frontend/e2e/compose.override.yaml @@ -0,0 +1,23 @@ +# The E2E lane copies this file into the install directory before scripts/install.sh runs, +# so Compose loads it in every command. api.anthropic.com resolves to the scripted mock in +# the Playwright process. The mock serves a certificate from the test CA in ./e2e. +# E2E_MOCK_ADDRESS is the address where a container reaches the host: the Docker bridge +# address on Linux, and 192.168.65.254 on Docker Desktop. +services: + drukbox-proxy: + extra_hosts: + - "api.anthropic.com:${E2E_MOCK_ADDRESS:-172.17.0.1}" + environment: + # The proxy refuses a private destination outside these networks. + SECRETS_PROXY_PRIVATE_NETWORKS: "${E2E_MOCK_ADDRESS:-172.17.0.1}/32" + volumes: + - ./e2e/ca.pem:/e2e/ca.pem:ro + # mitmdump loads this file at its start. It names the test CA as the trusted upstream CA. + - ./e2e/mitmproxy.yaml:/home/mitmproxy/.mitmproxy/config.yaml:ro + web: + extra_hosts: + - "api.anthropic.com:${E2E_MOCK_ADDRESS:-172.17.0.1}" + environment: + SSL_CERT_FILE: /e2e/ca.pem + volumes: + - ./e2e/ca.pem:/e2e/ca.pem:ro diff --git a/frontend/e2e/fixtures.ts b/frontend/e2e/fixtures.ts new file mode 100644 index 00000000..8935713b --- /dev/null +++ b/frontend/e2e/fixtures.ts @@ -0,0 +1,135 @@ +import { execFileSync } from 'node:child_process' +import { readFileSync } from 'node:fs' +import { homedir } from 'node:os' +import { join } from 'node:path' + +import { test as base, expect, type APIRequestContext, type Page } from '@playwright/test' + +import type { Conversation } from '../src/chat/state' +import { startAnthropic, textOf, TOKEN, USER_MARKER, type Anthropic } from './anthropic' + +export { expect } +export { Gate, TITLE } from './anthropic' + +/** The compose project that scripts/install.sh made, with the test CA in its e2e folder. */ +export const installDir = process.env.DRUKS_INSTALL_DIR ?? join(homedir(), 'druks') + +declare global { + interface Window { + reportRenderViolation: (detail: string) => void + } +} + +// U marks a turn's message and A starts its reply, so a marker gives its place on the page. +const MARKER = /\b[UA]\d+-[a-z0-9]+\b/g + +export const test = base.extend<{ violations: Set }, { anthropic: Anthropic }>({ + // Playwright reads the fixtures that a fixture needs from this pattern, also when it is empty. + // eslint-disable-next-line no-empty-pattern + anthropic: [async ({}, provide) => { + const certificates = join(installDir, 'e2e') + const anthropic = await startAnthropic({ + cert: readFileSync(join(certificates, 'api.anthropic.com.pem'), 'utf8'), + key: readFileSync(join(certificates, 'api.anthropic.com.key'), 'utf8'), + }) + await provide(anthropic) + await anthropic.close() + }, { scope: 'worker' }], + violations: async ({ page }, provide) => { + const violations = new Set() + await page.exposeFunction('reportRenderViolation', (detail: string) => violations.add(detail)) + await page.addInitScript(watchMessages, MARKER.source) + await provide(violations) + }, +}) + +/** The markers of a test's turns. Each test owns its nonce, so tests share no state. */ +export function markTurns(count: number) { + const nonce = Math.random().toString(36).slice(2, 8) + return Array.from({ length: count }, (_, index) => ({ user: `U${index + 1}-${nonce}`, reply: `A${index + 1}-${nonce}` })) +} + +export async function send(page: Page, body: string) { + await page.getByRole('textbox', { name: 'Message', exact: true }).fill(body) + await page.getByRole('button', { name: 'Send', exact: true }).click() +} + +/** Open a new conversation with its first message, and return its id. */ +export async function startConversation(page: Page, body: string): Promise { + await page.goto('/chat/new') + await send(page, body) + await page.waitForURL(/\/chat\/[0-9a-f-]{36}$/) + return new URL(page.url()).pathname.split('/').at(-1)! +} + +export async function getConversation(request: APIRequestContext, id: string): Promise { + const response = await request.get(`/api/chat/conversations/${id}`) + expect(response.status()).toBe(200) + return response.json() +} + +export function compose(...args: string[]) { + execFileSync('docker', ['compose', ...args], { cwd: installDir, stdio: 'inherit' }) +} + +/** The running boxes that drukbox's docker provider made. */ +export function listBoxes(): string[] { + const output = execFileSync('docker', ['ps', '--quiet', '--filter', 'label=managed-by=drukbox'], { encoding: 'utf8' }) + return output.split('\n').filter(Boolean) +} + +/** Runs in the page. On every DOM change, it reports a marker that shows twice or out of order. */ +function watchMessages(pattern: string) { + const rank = (marker: string) => Number(marker.slice(1, marker.indexOf('-'))) * 2 + Number(marker[0] === 'A') + new MutationObserver(() => { + const log = document.querySelector('[role="log"][aria-label="Messages"]') + const markers: string[] = log?.innerText.match(new RegExp(pattern, 'g')) ?? [] + markers.forEach((marker, index) => { + if (markers.indexOf(marker) !== index) window.reportRenderViolation(`${marker} shows twice`) + else if (index > 0 && rank(markers[index - 1]) > rank(marker)) window.reportRenderViolation(`${marker} shows below ${markers[index - 1]}`) + }) + }).observe(document, { childList: true, subtree: true, characterData: true }) +} + +const normalize = (text: string) => text.replace(/\s+/g, ' ').trim() + +/** The page shows each saved message once and in order. Each reply is what the mock streamed for + * its turn. The second turn's request carries the first turn. Every request carries the real token. */ +export async function expectChatFacts( + { page, request, anthropic, violations }: { page: Page, request: APIRequestContext, anthropic: Anthropic, violations: Set }, + id: string, +) { + const log = page.getByRole('log', { name: 'Messages' }) + await expect.poll(async () => { + const { messages } = await getConversation(request, id) + const text = normalize(await log.innerText()) + let position = -1 + return messages.flatMap((message) => { + const body = normalize(message.body) + const count = text.split(body).length - 1 + if (count !== 1) return [`${message.role} "${body}" shows ${count} times`] + if (text.indexOf(body) < position) return [`${message.role} "${body}" shows above an older message`] + position = text.indexOf(body) + return [] + }) + }, { message: 'each saved message shows once, in order' }).toEqual([]) + expect([...violations], 'no marker showed twice or out of order').toEqual([]) + + const conversation = await getConversation(request, id) + const turns = conversation.messages.filter((message) => message.role === 'user').map((message) => { + const marker = message.body.match(USER_MARKER)![0] + const requests = anthropic.completions.filter((completion) => completion.marker === marker) + expect(requests, `one model request for ${marker}`).toHaveLength(1) + const reply = conversation.messages.find((saved) => saved.replyTo === message.id) + expect(reply?.body, `the reply to ${marker} is what the mock streamed`).toBe(requests[0].streamed.join('')) + return { message, reply: reply!, request: requests[0] } + }) + + const [first, second] = turns + const history = second.request.messages.slice(0, -1).map(textOf).join('\n') + expect(history, 'the second turn carries the first message').toContain(first.message.body) + expect(history, 'the second turn carries the first reply').toContain(first.reply.body) + + const authorizations = new Set(anthropic.completions.map((completion) => completion.authorization)) + expect([...authorizations], 'every model request carries the real token').toEqual([`Bearer ${TOKEN}`]) +} diff --git a/frontend/e2e/install-stack.sh b/frontend/e2e/install-stack.sh new file mode 100755 index 00000000..c33bdbae --- /dev/null +++ b/frontend/e2e/install-stack.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env bash +# Install the stack that the E2E lane drives: scripts/install.sh on the docker provider, +# with a test CA and the E2E Compose override, then the identity and sign-in settings. +# +# It takes the env knobs of scripts/install.sh. Its arguments go to `druks setup`, for +# example --set sandbox.image=druks-sandbox:e2e. +set -euo pipefail + +e2e="$(cd "$(dirname "$0")" && pwd)" +install_dir="${DRUKS_INSTALL_DIR:-$HOME/druks}" + +# The mock serves api.anthropic.com with a certificate from this CA. Each extension is +# explicit, because OpenSSL and the LibreSSL of macOS add different ones by default. +mkdir -p "$install_dir/e2e" +cd "$install_dir/e2e" +openssl req -x509 -newkey rsa:2048 -nodes -days 2 -subj "/CN=Druks E2E CA" \ + -addext "basicConstraints=critical,CA:TRUE" -addext "keyUsage=critical,keyCertSign" \ + -addext "subjectKeyIdentifier=hash" -keyout ca.key -out ca.pem +openssl req -newkey rsa:2048 -nodes -subj "/CN=api.anthropic.com" \ + -keyout api.anthropic.com.key -out api.anthropic.com.csr +openssl x509 -req -in api.anthropic.com.csr -days 2 -CA ca.pem -CAkey ca.key -CAcreateserial \ + -extfile <(printf 'subjectAltName=DNS:api.anthropic.com\nbasicConstraints=CA:FALSE\nauthorityKeyIdentifier=keyid\n') \ + -out api.anthropic.com.pem +echo "ssl_verify_upstream_trusted_ca: /e2e/ca.pem" > mitmproxy.yaml + +# The installer keeps an override that exists. +cp "$e2e/compose.override.yaml" "$install_dir/compose.override.yaml" +DRUKS_PROVIDER=docker bash "$e2e/../../scripts/install.sh" + +cd "$install_dir" +docker run --rm --user "$(id -u):$(id -g)" -v "$install_dir:/bootstrap" \ + "ghcr.io/czpython/druks:$(sed -n 's/^DRUKS_TAG=//p' .env)" \ + druks setup /bootstrap/.env --home "$HOME" \ + --set identity.mode=header --set identity.header=X-Forwarded-Email \ + --set providers.anthropic.sign_in=setup_token "$@" +docker compose restart web +# drukbox pulls the sandbox image inside the request that creates a template, and a first +# pull can take longer than that request may. +image="$(sed -n 's/^image = "\(.*\)"$/\1/p' druks.toml)" +docker image inspect "$image" >/dev/null 2>&1 || docker pull "$image" +docker compose exec -T web druks sandboxes build diff --git a/frontend/e2e/sign-in.setup.ts b/frontend/e2e/sign-in.setup.ts new file mode 100644 index 00000000..2af6efa3 --- /dev/null +++ b/frontend/e2e/sign-in.setup.ts @@ -0,0 +1,9 @@ +import { expect, test as setup } from '@playwright/test' + +import { TOKEN } from './anthropic' + +// Druks checks a setup token's prefix only, so the fake token needs no network call. +setup('the test account signs in to Anthropic', async ({ request }) => { + const response = await request.post('/api/providers/anthropic/sign-in/token', { data: { token: TOKEN } }) + expect(response.status(), await response.text()).toBe(200) +}) diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 9350f38c..b8c483d3 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -27,6 +27,7 @@ "devDependencies": { "@agentclientprotocol/sdk": "2.1.0", "@eslint/js": "^10.0.1", + "@playwright/test": "^1.65.0", "@testing-library/dom": "^10.4.2", "@testing-library/react": "^16.3.3", "@types/node": "^24.19.2", @@ -879,6 +880,22 @@ "url": "https://github.com/sponsors/oxc-project" } }, + "node_modules/@playwright/test": { + "version": "1.65.0", + "resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.65.0.tgz", + "integrity": "sha512-iGURLGuKDLakSEl7Pgfv8h7f+XLT+V861Wfhz0zlEl5nyFuKUopdkFBPKqTllTUp06aOwPuH8iuUEb3yp3MpbQ==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "playwright": "1.65.0" + }, + "bin": { + "playwright": "cli.js" + }, + "engines": { + "node": ">=20" + } + }, "node_modules/@redocly/ajv": { "version": "8.11.2", "resolved": "https://registry.npmjs.org/@redocly/ajv/-/ajv-8.11.2.tgz", @@ -5051,6 +5068,35 @@ "url": "https://github.com/sponsors/jonschlinkert" } }, + "node_modules/playwright": { + "version": "1.65.0", + "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.65.0.tgz", + "integrity": "sha512-F3AMWcR44qMBt4kq+6Kb6mX7bjGXshktKFOPULd7reED6ZJWevjbNRqysP7q+AATE9kdPEZBibex1BYsvpVoeA==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "playwright-core": "1.65.0" + }, + "bin": { + "playwright": "cli.js" + }, + "engines": { + "node": ">=20" + } + }, + "node_modules/playwright-core": { + "version": "1.65.0", + "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.65.0.tgz", + "integrity": "sha512-vNSKgH6YqtN7cdT/L334zo0a+YwkrpicyrHJlWWPIm/Q+Nn2SWInil01oT0mbu96U4GgoRtCYQD9CzJQgju+lA==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "playwright-core": "cli.js" + }, + "engines": { + "node": ">=20" + } + }, "node_modules/pluralize": { "version": "8.0.0", "resolved": "https://registry.npmjs.org/pluralize/-/pluralize-8.0.0.tgz", diff --git a/frontend/package.json b/frontend/package.json index e6d7e984..f21f3d73 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -10,6 +10,7 @@ "preview": "vite preview", "test": "vitest run", "test:watch": "vitest", + "test:e2e": "playwright test", "types:openapi": "openapi-typescript http://127.0.0.1:8000/openapi.json -o src/api/openapi.ts" }, "dependencies": { @@ -32,6 +33,7 @@ "devDependencies": { "@agentclientprotocol/sdk": "2.1.0", "@eslint/js": "^10.0.1", + "@playwright/test": "^1.65.0", "@testing-library/dom": "^10.4.2", "@testing-library/react": "^16.3.3", "@types/node": "^24.19.2", diff --git a/frontend/playwright.config.ts b/frontend/playwright.config.ts new file mode 100644 index 00000000..982d9f13 --- /dev/null +++ b/frontend/playwright.config.ts @@ -0,0 +1,26 @@ +import { defineConfig } from '@playwright/test' + +// The suite drives a stack that scripts/install.sh made, with only the model faked. See +// docs/development.md. +export default defineConfig({ + testDir: 'e2e', + forbidOnly: true, + retries: 0, + workers: 1, + timeout: 180_000, + // A first turn boots the account's box. + expect: { timeout: 60_000 }, + reporter: [['list'], ['html', { open: 'never' }]], + use: { + // The endpoint of a local install: the chat socket accepts this origin only. + baseURL: 'http://127.0.0.1:8001', + // The edge identity that druks.toml names in identity.header. + extraHTTPHeaders: { 'X-Forwarded-Email': 'e2e@druks.test' }, + trace: 'retain-on-failure', + screenshot: 'only-on-failure', + }, + projects: [ + { name: 'sign-in', testMatch: /\.setup\.ts$/ }, + { name: 'chat', dependencies: ['sign-in'] }, + ], +}) diff --git a/frontend/tsconfig.node.json b/frontend/tsconfig.node.json index d3c52ea6..eec70ad7 100644 --- a/frontend/tsconfig.node.json +++ b/frontend/tsconfig.node.json @@ -2,7 +2,7 @@ "compilerOptions": { "tsBuildInfoFile": "./node_modules/.tmp/tsconfig.node.tsbuildinfo", "target": "es2023", - "lib": ["ES2023"], + "lib": ["ES2023", "DOM"], "module": "esnext", "types": ["node"], "skipLibCheck": true, @@ -20,5 +20,5 @@ "erasableSyntaxOnly": true, "noFallthroughCasesInSwitch": true }, - "include": ["vite.config.ts"] + "include": ["vite.config.ts", "playwright.config.ts", "e2e"] } diff --git a/scripts/install.sh b/scripts/install.sh index f22d3f12..a2e2e185 100755 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -152,7 +152,7 @@ main() { # Shape selection, written to .env. After that, `docker compose` in this # directory starts the services of the shape. The `hosted` profile adds the - # Caddy edge and the janitor. The `proxy` profile adds the secrets proxy on + # Caddy edge. The `proxy` profile adds the secrets proxy on # every provider but docker-sbx, where sbx swaps the placeholders itself. # A `docker` box runs without Caddy, with the dashboard on :8001. docker-sbx # also layers the overlay, which connects drukbox to the host sandboxd, and