From 88a43106e66cb890b6f15c0acc0c0065f15a4a41 Mon Sep 17 00:00:00 2001 From: Wanjun Gu Date: Fri, 16 Oct 2026 19:12:28 -0700 Subject: [PATCH 01/32] docs(crew): design contract for open workspaces and automatic setup --- .../open-workspaces-and-automatic-setup.md | 172 ++++++++++++++++++ 1 file changed, 172 insertions(+) create mode 100644 docs/crew/design/open-workspaces-and-automatic-setup.md diff --git a/docs/crew/design/open-workspaces-and-automatic-setup.md b/docs/crew/design/open-workspaces-and-automatic-setup.md new file mode 100644 index 000000000..08457e587 --- /dev/null +++ b/docs/crew/design/open-workspaces-and-automatic-setup.md @@ -0,0 +1,172 @@ +# Open workspaces and automatic setup + +> **What this is.** The design contract for six Crew changes made together in October 2026: automatic server setup, open (shared) workspaces, deletion, `@agent` from the composer, a default model for Crew tasks, and agents that run their work on the server with a per-workspace context document. +> **Status:** Current. Implemented on the `claude/crew-auto-setup` branch; where this page and the code disagree, the code is what shipped. +> **Audience:** Developers and reviewers of Crew. Users read the [Crew user manual](../README.md). + +## Why + +A lab that tried Crew on `leo.ucsf.edu` (Ubuntu 22.04, Linux 5.15, no job scheduler) hit six walls: + +1. **No setup.** Nothing installed `~/.local/bin/biorouter-crew`; the app showed copy-paste commands with a placeholder path, and "Start it for me" could not sign in to a password-only account. +2. **Members could not use agents with files.** Every agent folder had to be owned by the member, inside a confined sandbox that requires Landlock ABI 3 (Linux 6.2+). On leo every command failed for everyone, and no channel file could reach the server. +3. **Owner-only management.** Only owners renamed, archived or added people, and every addition ended every agent grant in the workspace. +4. **No deletion** of channels, teams or workspaces. +5. **One way to start an agent:** the **Ask my agent** button, with the chat's default model only. +6. **Agents knew nothing about the server** they were meant to compute on. + +The owner rulings that shape the answers: *user warns, agent never*; *privacy is safety, not security*; *local agent forgery is not a threat*; keep the user experience simple; keep model-tier refusals. + +## Decisions + +| ID | Decision | +|---|---| +| OW-1 | A workspace has a **governance** setting, `open` or `managed`. Journals written before this change read as `managed` (today's rules). A workspace created by this release's host flow is set to `open` right after bootstrap. Only the host changes it. | +| OW-2 | In an **open** workspace every active member may rename, archive and delete teams and channels, add and remove people in teams and channels, join any team or channel, and invite and approve joiners. Only the host removes someone from the workspace, changes privacy or governance, renames or deletes the workspace. | +| OW-3 | In an open workspace a new member is added to every team and every unarchived channel when they are admitted; a new team gets every active member; a new channel gets every member of its team. Additions in an open workspace do **not** bump the policy epoch, so they no longer end agent grants. Removals, archive, delete and policy changes still do. | +| OW-4 | `channel.delete`, `team.delete` and `workspace.delete` exist. Channel and team deletion remove live state (the channel or team, its attachments' state entries and files, references, invitations and read positions) and revoke runs that name the channel. Message records stay in the journal but become unreadable to everyone, because their channel no longer exists. They do not bump the epoch. A team's `#general` is deleted only with its team. | +| OW-5 | `workspace.delete` is host-only. It commits a tombstone (devices, grants, joins cleared, every run revoked), answers every later request except `hello` with `workspace_deleted`, waits 30 seconds so connected members learn of it, then erases only the files the broker created and exits. `biorouter-crew delete` does the same for a stopped broker. | +| OW-6 | A connection's agent access has three levels: **off** (no work folder), **folder** (today's confined mode) and **full** (*run as me*). Full runs commands as the member's own account through their own bridge: login shell, network, child processes, any path the account can reach, jobs that may outlive the task. The member chooses it for their own account; new connections made by the setup flow default to full with one warning. Public models never get any level (all four existing layers still refuse them). | +| OW-7 | In full mode a channel attachment can be copied to the server (`remote.fetch_blob`) and a server file can be posted to the channel (`remote.share_file`), up to 1 GiB, through a background transfer in the bridge. | +| OW-8 | Each workspace has a **context document** (Markdown, at most 64 KiB) stored by the broker, with a revision, editor and time. The host flow generates it from a fixed server probe right after bootstrap. Any member edits it in an open workspace; only the host in a managed one. Every member's Crew agent receives it as reference information, framed as authorizing nothing. | +| OW-9 | **Setup is a daemon service** used by the GUI and the CLI: probe, sign in if needed, transfer the broker over the SSH session's stdin, verify its SHA-256, install it atomically at `~/.local/bin/biorouter-crew`, then start the workspace (host) or connect (member). The broker comes from the desktop build itself (`crew-broker/linux-x86_64/` next to `biorouterd`, with a manifest); a development override is `BIOROUTER_CREW_BROKER_DIR`. | +| OW-10 | `@agent`, `/agent` and `/task` at the start of a Crew message start a task in that channel with the person's model for Crew tasks. The composer shows the same facts the pane shows before Enter; Enter is the consent. A person whose username is `agent` keeps `@agent` as their mention; `/agent` still works. | +| OW-11 | The model for Crew tasks is the config key `CREW_TASK_MODEL` = `{provider, model}`. The renderer and the CLI resolve it (falling back to `BIOROUTER_PROVIDER`/`BIOROUTER_MODEL`) and send it explicitly; the daemon never reads it, so it is a preference, not a capability key. Every start is still admitted by the daemon and the broker. | +| OW-12 | Crew task limits rise to 60 model turns, 200 tool calls and 55 minutes, because full-access tasks do real work. | + +## Broker wire contract (server, `crates/biorouter-crew`) + +All additions are compatible with a 1.92.1 broker reading the same journal: new state fields use `serde(default, skip_serializing_if)`, deletions are map `Remove` patches, and new behavior is a new method or capability. Capabilities are appended to `capabilities()` after `presence_v1`, in this order: `open_governance_v1`, `delete_v1`, `workspace_context_v1`, `remote_access_v1`, `wide_runs_v1`. + +### State + +- `Workspace.governance`: `"managed"` (default, omitted) or `"open"`. +- `Workspace.deleted_at`: unix seconds, omitted until deleted. +- `State.context` (or `Workspace.context`): `{text, revision, updated_by, updated_at}`, omitted until first set. +- `Run.remote_access`: `"folder"` (default, omitted) or `"full"`. + +### Methods + +| Method | Who | Params | Result | +|---|---|---|---| +| `workspace.set_governance` | host, person | `{governance: "open"\|"managed"}` | `{governance}` | +| `team.join` | open: any active member | `{team_id}` | `{team_id, joined: true}`; also joins its `#general` | +| `channel.join` | open: any member of the channel's team | `{channel_id}` | `{channel_id, joined: true}` | +| `channel.delete` | managed: owner, or host stewarding an orphan; open: any channel member; always a person | `{channel_id, expected_name?}` | `{deleted: true, channel_id, team_id, removed_attachments, removed_references}` | +| `team.delete` | managed: creator, or host when the creator left; open: any team member; person | `{team_id, expected_name?}` | `{deleted: true, team_id, channels}` | +| `workspace.delete` | host, person | `{workspace_id, expected_name}` | `{deleted: true, workspace_id, erase_after_seconds: 30}` | +| `workspace.context.set` | open: any member; managed: host; person | `{text, expected_revision}` | `{revision, updated_at}` | +| `workspace.context.get` | any member; a worker of a private run, or of a public run when the workspace is Public | `{}` | `{context: {...} \| null}` | + +Refusals keep the broker's `code: sentence` form. New codes: `workspace_deleted`, `revision_conflict`, `name_changed`. `#general` alone: `invalid_params: #general belongs to its team. Delete the team instead.` + +`workspace.snapshot` adds `workspace.governance`, `context` (text, revision, updated_by, updated_at) and, in an open workspace, `joinable_teams` and `joinable_channels` (`{id, team_id?, name, display_name, handle, classification, member_count}`), inside the existing snapshot byte budget. + +`run.create` accepts `remote_access: "full"` only with `remote_root`, `remote_execution: true` and a private provider; it echoes `remote_access` in the run. With `wide_runs_v1` a run may read up to 100 source channels (was 20). `run.remote_scope` adds `access` and `owner_id`. + +No new method is in the worker allowlist except `workspace.context.get`. Every mutation above also checks `actor.run.is_none()`. + +### Bridge-local methods (member's own binary) + +The bridge answers these itself and never forwards them: + +- `bridge.hello {}` → `{bridge_version, capabilities: ["remote_full_v1", "remote_transfer_v1", "bridge_probe_v1"], landlock_abi, confined_execution_supported, user, uid, home, shell}`. An old bridge forwards it and the broker refuses, which the daemon reads as "old bridge". +- `bridge.probe {}` → the server facts below. + +Full-access `remote.*` methods, used only when `run.remote_scope` answers `access: "full"` (a missing `access` means folder mode, so an old broker fails closed): + +| Method | Params | Notes | +|---|---|---| +| `remote.shell` | `{command, cwd?, timeout_seconds?, wait_seconds?, detach?, idempotency_key}` | Runs ` -lc ` as the member, stdin `/dev/null`, output to `~/.local/state/biorouter-crew/jobs//`. Waits up to `wait_seconds` (default 10, at most 20) and returns the result or `running` with the job id. `detach: true` survives the bridge. | +| `remote.execute` | `{argv, ...same}` | As `remote.shell` with an argv. | +| `remote.job_status` | `{job_id, tail_bytes?}` | Any run of the same owner may read it. | +| `remote.cancel` | `{job_id}` | Kills the job's process group. | +| `remote.list`, `remote.stat`, `remote.read`, `remote.write`, `remote.hash` | absolute, `~/` or work-folder-relative `path`; `read {offset, length ≤ 393216}`; `write {text \| data_hex, mode: truncate\|append\|create_new, create_parents}` | Symlinks followed, no ownership rule. | +| `remote.fetch_blob` | `{blob_id, path, idempotency_key}` | Background copy of a channel attachment, SHA-256 checked; returns a job id. | +| `remote.share_file` | `{path, name?, media_type?, idempotency_key}` | Background upload of a server file (≤ 1 GiB); the job result carries the `blob_id` the daemon then posts. | + +Folder mode is unchanged, except that a kernel without Landlock ABI 3 is refused up front with a sentence that names full access. + +### Server probe and context draft + +`biorouter-crew probe --output-format json` and `bridge.probe` run fixed, bounded checks (no network, 5 s each) and return: + +```json +{"schema": 1, "host": "leo.ucsf.edu", "os": "Ubuntu 22.04.5 LTS", "kernel": "5.15.0-141-generic", + "arch": "x86_64", "glibc": "2.35", "cpus": 16, "mem_gib": 187, "gpus": [], + "scheduler": {"kind": null, "commands": []}, "modules": false, + "conda": ["~/anaconda3"], "containers": {"docker": true, "docker_group": true, "apptainer": false, "podman": false}, + "languages": {"python3": "3.x", "R": "4.x"}, + "storage": [{"path": "/pool1/home", "fs": "zfs", "size_gib": 20000, "avail_gib": 19900}], + "landlock_abi": 1, "user": "wanjun", "home": "/pool1/home/wanjun"} +``` + +It never reads environment variables, process lists, other people's homes, SSH configuration or history files. `biorouter-crew doctor --output-format json` adds hosting blockers and execution limits. The daemon renders the facts as the first context document (`crew::context::render_workspace_context`). + +## Desktop daemon contract (`crates/biorouter`, `crates/biorouter-server`) + +### Setup routes (all require proof of a person) + +| Route | Body → answer | +|---|---| +| `GET /crew/setup/artifact` | → `{version, target, sha256, size, source: "bundled"\|"dev"}` | +| `POST /crew/setup` | `{target: {kind: "connection", connection_id} \| {kind: "pending", preparation_id, ssh_target, port?, identity_file?, proxy_jump?}, intent: "check"\|"install"\|"host"\|"member", workspace_name?, replace_existing?}` → `SetupStatus` | +| `GET /crew/setup/{job_id}` | → `SetupStatus {job_id, intent, state: running\|needs_sign_in\|finished\|failed\|cancelled, stage, stages[], server, installed, artifact, outcome, error}` | +| `DELETE /crew/setup/{job_id}` | cancel | +| `POST /crew/setup/{job_id}/authentication` | `{request_id, controller_id, cols, rows}` → an authentication session; the terminal is the existing `GET /crew/authentication/{id}/terminal` | +| `POST /crew/connections/{id}/context/draft` | → `{draft, facts}` from `bridge.probe` over that connection | + +`SetupStatus.outcome` carries `installed_version`, `reused`, `host` (the same start output as `/crew/host/start`), `host_command`, `connection_id`, `work_folder` (the canonical `~/crew-work/` the job created) and `context_draft`. A member job sets that folder and full access on the connection when it has no work folder yet and the broker and bridge support it. A setup sign-in rides the same control path the connection will use, so one password covers install, start, create and connect. A sign-in whose bridge is missing keeps its SSH master so setup can use it. + +### Connection, runs and agents + +- `Connection.remote_access`: `"folder"` (default) or `"full"`; also on `SaveConnection` and the invitation's advanced fields. Full requires a work folder and `remote_execution`. +- The connection view adds `server_access: {full_access_available, confined_execution_available, bridge_version, landlock_abi, user, home}` from `bridge.hello`, or `null` before the first connect. +- `run.create` sends `remote_access` only when the broker advertises `remote_access_v1` and the bridge `remote_full_v1`, and checks the echo. +- `crew__request` gains `remote.shell`, `remote.stat`, `remote.fetch_blob`, `remote.share_file`, `blob.list` (when advertised) and `workspace.context.get`. The tool instructions say what full mode can do, that heavy work runs on the server, and to read the workspace context. +- Every Crew task and connected chat gets the workspace context inside the escaped hidden context, framed as reference information that authorizes nothing. +- `POST /crew/connections/{id}/workspace/delete {workspace_id, expected_name}` sends `workspace.delete` and then removes the connection from this computer. A `workspace_deleted` refusal marks a member's connection so the interface offers **Remove from this computer**. + +## Interfaces + +### Desktop + +- **Host dialog:** "Set up and start Crew" runs a host setup job, shows plain stages (Checking server, Signing in, Installing Crew in your account, Starting workspace, Ready), then Create bootstraps, sets the institution, sets governance to open and saves the context draft. +- **Not set up / Join:** "Set up Crew for my account" runs a member setup job and reconnects; Join starts it by itself when the bridge is missing. +- **Composer:** `@agent`, `/agent`, `/task` with a suggestion list (agent row and people), an inline consent line, and the send button labelled "Start agent task". +- **Settings > Models:** a "Crew tasks" row ("Same as new chats" or a model); the task pane starts on it. +- **Workspace settings:** governance switch (host), a "Context" tab with an editor and **Draft from server**, and **Delete workspace…** (host, type the name). Members get **Remove from this computer**. +- **Sidebar and channel:** **Delete channel…** and **Delete team…** wherever the person may; **Browse channels** lists joinable teams and channels in an open workspace. +- **Connection settings:** "What can your agent do on {server}?" Off / In a folder / As me on the server. + +### CLI + +| Command | Does | +|---|---| +| `biorouter crew setup --server LOGIN` / `--connection NAME` [`--replace-existing`] | install or update the broker in your account | +| `biorouter crew doctor --server LOGIN` | probe only | +| `biorouter crew host --name NAME --server LOGIN [--institution ID] [--mode private\|public]` | prepare, install, start, save, sign in, bootstrap, privacy, open governance, context | +| `biorouter crew channels delete CHANNEL [--yes]`, `channels join CHANNEL` | | +| `biorouter crew teams delete TEAM [--yes]`, `teams join TEAM` | | +| `biorouter crew workspace delete --confirm NAME` | host | +| `biorouter crew workspace governance open\|managed` | host | +| `biorouter crew workspace context show\|set FILE\|edit\|draft` | | +| `biorouter crew connections set-access off\|folder\|full [--folder PATH]` | | +| `biorouter crew tasks start CHANNEL --text T` | `--provider`/`--model` optional (resolved from `CREW_TASK_MODEL`) | +| `biorouter crew ask CHANNEL TEXT…` | `tasks start` with the defaults and posting allowed | +| `biorouter crew defaults show\|set --provider P --model M\|clear` | the model for Crew tasks | + +## Risks accepted + +- **Full access is the member's whole account.** A misled agent can run anything the account can, including replacing its own bridge (attacker M2 in the [naming design](naming-design.md)). Device keys stay on the desktop, so it still cannot sign as the person. The connection warning says so, and every `remote.shell` is announced in the channel by program name and length, never the command text. +- **Prompt injection.** Other members' messages, files and the context document reach every member's agent. The agent's instructions treat them as information. Model-tier refusals, the secret guard on arguments and the secret-output scan on results still apply. +- **Deleted channels' text stays in the journal** until the workspace is deleted. The interface says "removed from the workspace". +- **Open governance trusts every member** with everything but removal from the workspace. It is the host's choice and the host can switch back. + +## Related documentation + +- [Broker protocol](protocol-contract.md): the wire contract this page extends. +- [Naming design](naming-design.md): identity rules, attacker M2. +- [Linux portability](linux-portability.md): the broker build and its glibc floor. +- [Agents and chat access](../agents-and-chat-access.md): what users see of agents. +- [Privacy tiers](../../security/privacy-tiers.md): the model-tier rules Crew keeps. From bcc68e6a413da6eee0c07f0be6a97c41751a6e3f Mon Sep 17 00:00:00 2001 From: Wanjun Gu Date: Fri, 16 Oct 2026 19:15:22 -0700 Subject: [PATCH 02/32] crew: add the RemoteAccess level type (design OW-6) --- crates/biorouter/src/crew/mod.rs | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/crates/biorouter/src/crew/mod.rs b/crates/biorouter/src/crew/mod.rs index 0b3042ac5..8e0c8d671 100644 --- a/crates/biorouter/src/crew/mod.rs +++ b/crates/biorouter/src/crew/mod.rs @@ -68,6 +68,21 @@ pub enum ClusterMode { #[default] Private, } +/// What a connection's agents may do on the server (design OW-6). `Folder` is the confined work +/// folder Crew always had; `Full` runs commands as the person's own account through their own +/// bridge. "Off" is the absence of a work folder, not a level. +#[derive(Clone, Copy, Debug, Default, Deserialize, Serialize, PartialEq, Eq, utoipa::ToSchema)] +#[serde(rename_all = "snake_case")] +pub enum RemoteAccess { + #[default] + Folder, + Full, +} +impl RemoteAccess { + pub fn is_folder(&self) -> bool { + *self == Self::Folder + } +} /// A connection to save or edit (`POST /crew/connections`, `PATCH /crew/connections/{id}`). #[derive(Clone, Debug, Deserialize, Serialize, utoipa::ToSchema)] #[serde(deny_unknown_fields)] From f787a7ab21ddb6f14ca9fa2b37ee90a62ec4d13a Mon Sep 17 00:00:00 2001 From: Wanjun Gu Date: Fri, 16 Oct 2026 19:39:11 -0700 Subject: [PATCH 03/32] crew(ui): model for Crew tasks, pane seed and composer run-start routing (OW-10, OW-11) --- .../src/components/ConfigContext.test.tsx | 36 ++- ui/desktop/src/components/ConfigContext.tsx | 21 +- .../components/crew/pane/AgentTaskPane.tsx | 227 ++++++++++++------ ui/desktop/src/components/crew/pane/copy.ts | 25 ++ .../src/components/crew/pane/presentation.ts | 163 +++++++++++++ .../crew/pane/useConfiguredModels.ts | 99 ++++++-- .../src/components/crew/state/crewActions.ts | 1 + .../src/components/crew/state/crewRunStart.ts | 67 +++++- .../crew/state/crewTaskModel.test.ts | 103 ++++++++ .../components/crew/state/crewTaskModel.ts | 82 +++++++ ui/desktop/src/components/crew/state/types.ts | 56 ++++- .../crew/state/useCrewController.ts | 2 + 12 files changed, 768 insertions(+), 114 deletions(-) create mode 100644 ui/desktop/src/components/crew/state/crewTaskModel.test.ts create mode 100644 ui/desktop/src/components/crew/state/crewTaskModel.ts diff --git a/ui/desktop/src/components/ConfigContext.test.tsx b/ui/desktop/src/components/ConfigContext.test.tsx index b5ce757f7..705dbf334 100644 --- a/ui/desktop/src/components/ConfigContext.test.tsx +++ b/ui/desktop/src/components/ConfigContext.test.tsx @@ -451,6 +451,19 @@ describe('ConfigContext announces writes of the app-wide model selection (F3)', + + @@ -479,15 +492,20 @@ describe('ConfigContext announces writes of the app-wide model selection (F3)', ); - it.each([['Write provider'], ['Write model'], ['Remove model']])( - '%s announces, once the write has resolved', - async (button) => { - renderWriteProbe(); - fireEvent.click(screen.getByRole('button', { name: button })); - await waitFor(() => expect(screen.getByTestId('write-result')).toHaveTextContent('ok')); - expect(nudges).toBe(1); - } - ); + // OW-11: Crew's composer and Ask my agent state the model a task starts on, which falls back to + // the pair, so a write of `CREW_TASK_MODEL` reaches every window the same way. + it.each([ + ['Write provider'], + ['Write model'], + ['Remove model'], + ['Write Crew task model'], + ['Remove Crew task model'], + ])('%s announces, once the write has resolved', async (button) => { + renderWriteProbe(); + fireEvent.click(screen.getByRole('button', { name: button })); + await waitFor(() => expect(screen.getByTestId('write-result')).toHaveTextContent('ok')); + expect(nudges).toBe(1); + }); it('says nothing about a key a new chat does not bind', async () => { renderWriteProbe(); diff --git a/ui/desktop/src/components/ConfigContext.tsx b/ui/desktop/src/components/ConfigContext.tsx index 8eca94452..8b06050fb 100644 --- a/ui/desktop/src/components/ConfigContext.tsx +++ b/ui/desktop/src/components/ConfigContext.tsx @@ -96,18 +96,25 @@ export class MalformedConfigError extends Error { const ConfigContext = createContext(undefined); /** - * F3 — the two keys `/agent/start` binds a new chat to. + * F3 — the two keys `/agent/start` binds a new chat to, and the model Crew + * tasks start on. * - * A write of either changes what every window's composer must state, so it is - * announced to all of them (`utils/sessionBindingSync`), each of which re-reads - * the pair. `ModelAndProviderContext.changeModel` announces its own writes; the - * ones caught HERE are those that never pass through it — the local and - * coding-agent onboarding cards, Lead/Worker settings, Settings' reset — which - * until now left even their own window's chip naming the previous model. + * A write of either of the first two changes what every window's composer must + * state, so it is announced to all of them (`utils/sessionBindingSync`), each of + * which re-reads the pair. `ModelAndProviderContext.changeModel` announces its + * own writes; the ones caught HERE are those that never pass through it — the + * local and coding-agent onboarding cards, Lead/Worker settings, Settings' reset + * — which until now left even their own window's chip naming the previous model. + * + * `CREW_TASK_MODEL` (design OW-11) rides the same announcement: Crew's composer + * and Ask my agent state the model a task starts on, which falls back to the + * pair above, and a chat composer that re-reads the pair for it finds nothing + * changed. It is a preference, not a capability key: the daemon never reads it. */ const APP_MODEL_SELECTION_KEYS: ReadonlySet = new Set([ 'BIOROUTER_PROVIDER', 'BIOROUTER_MODEL', + 'CREW_TASK_MODEL', ]); export const ConfigProvider: React.FC = ({ children }) => { diff --git a/ui/desktop/src/components/crew/pane/AgentTaskPane.tsx b/ui/desktop/src/components/crew/pane/AgentTaskPane.tsx index b4462c5eb..09052d76c 100644 --- a/ui/desktop/src/components/crew/pane/AgentTaskPane.tsx +++ b/ui/desktop/src/components/crew/pane/AgentTaskPane.tsx @@ -16,6 +16,7 @@ import type { CrewController } from '../state/types'; import { agentCopy, LONG_TASK_CHARS, LONG_TASK_LINES, unknownOutcomeCopy } from './copy'; import { CrewModelPicker, ModelTierMarks } from './CrewModelPicker'; import { newestTaskIn } from './newestTask'; +import { crewTaskModelUsable } from '../state/crewTaskModel'; import { fileBaseName, isAffiliationRefusal, @@ -25,13 +26,13 @@ import { modelDisplay, modelMismatch, modelRefusalText, - protectedRunContext, - publicModelRefusal, - runInstitution, + remoteAccessText, sameNamedFiles, sharedWhen, + taskStartCheck, unsharedFileNames, usePanePresentation, + wideRunChannels, workspaceInstitutionLabel, type SharedFile, } from './presentation'; @@ -199,9 +200,11 @@ export interface AgentTaskPaneProps { * reads this channel and posts in it. * * - The destination is always shown; it is the consent. - * - **Task** has its own state, seeded once from the composer draft when the pane opens (L9). On - * success the Task clears, the composer draft clears only if it still equals that seed, and the - * pane closes. The daemon posts the task itself in the channel ("Task: …"), so the field says so, + * - **Task** has its own state, seeded once from the composer draft when the pane opens (L9), or, + * when the composer opened it for `@agent` (OW-10), from the words after the trigger. On success + * the Task clears, the composer draft clears only if it still equals that seed (the whole + * `@agent …` draft), and the pane closes. The daemon posts the task itself in the channel + * ("Task: …"), so the field says so, * and says it again for a task long enough to be pasted data (T-24). It is at least six lines, * grows with what is written and has no resize grip (`.crew-agent-task`, Q2-30). * - A task that names a file (`counts.csv`) no message in the channel shares gets a warning before @@ -211,8 +214,10 @@ export interface AgentTaskPaneProps { * describing Start: "2 files named … are shared in #…. Your agent will use the newest one, shared * at 1:55 AM." — the daemon tells the agent to do exactly that and to name the file it used * (Q3-02). The Task field does not spell-check: file names and sample IDs are not words (Q3-34). - * - **Model**: the app's default model as a summary with Change when it resolves to a configured - * provider, else the picker; "No models are set up." with Open Settings when nothing is + * - **Model**: the model for Crew tasks (`CREW_TASK_MODEL`, OW-11), else the app's default model, + * as a summary with Change when it resolves to a configured provider, else the picker (opened at + * once when the composer's `@agent` model was blocked); **Use for future tasks** under any other + * model makes it the one tasks start on; "No models are set up." with Open Settings when nothing is * configured (Crew bypasses provider onboarding). A model is named as the composer's model chip * names it and wraps rather than ellipsizing, followed by one worded chip, "🔒 Private · UCSF" * (`ModelTierMarks`, Q2-67, one piece since Q4-28), then Change as a separate link. A private @@ -221,7 +226,8 @@ export interface AgentTaskPaneProps { * - **Advanced** holds Also read (unmounted while closed, which keeps the unknown-outcome gate's * checkbox the only one in the document, C9) and the remote folder line; closed, it says what it * holds: "Reads only #general" or "Also reads #methods". With neither to offer it is not shown - * at all. + * at all. In an open workspace whose broker reads wide (`wideRunChannels`), Also read lists every + * channel the person is in, ticked, and the summary says "Reads N channels". * - Its errors are `pane:agent`'s. `DetailsPane` dismisses one when the pane leaves this mode, so * a refusal does not reappear in the connection bar for a drawer that is gone (T-48). * - **Start my agent and allow posting here** is rendered unconditionally with a stable key, and @@ -241,7 +247,11 @@ export interface AgentTaskPaneProps { export function AgentTaskPane({ onShowTask, className }: AgentTaskPaneProps) { const { crew, snapshot, channel, team, verified, workspace } = usePanePresentation(); const navigate = useNavigate(); - const [seed] = useState(() => crew.draft.body); + // The composer opens the pane for `@agent` with the task's words without the trigger and the + // draft they came from (OW-10); otherwise the Task is the draft itself. A seed that arrives while + // the pane is already open replaces the Task (below). + const intentSeed = crew.ui.pane?.mode === 'agent' ? crew.ui.pane.seed : undefined; + const [seed, setSeed] = useState(() => intentSeed?.draft ?? crew.draft.body); // A task whose start failed here, after the person had moved on, comes back as the Task // (MSG2-N10): only while the view verifies it under the scope it was started under. Read once as // the pane opens, and forgotten once shown, as a kept draft is. @@ -254,14 +264,41 @@ export function AgentTaskPane({ onShowTask, className }: AgentTaskPaneProps) { ) : null ); - const [task, setTask] = useState(kept ?? seed); + const [task, setTask] = useState(kept ?? intentSeed?.task ?? seed); useEffect(() => { if (kept !== null) forgetFailedTask(crew.connectionId, crew.channelId); }, []); // eslint-disable-line react-hooks/exhaustive-deps -- once, for the task this pane opened with const models = useConfiguredModels(); const [choice, setChoice] = useState(null); - const [picking, setPicking] = useState(false); + // `pickModel`: the composer's model was blocked here, or there was none, so the pane opens on the + // picker rather than on a summary of a model that cannot start this task. + const [picking, setPicking] = useState(intentSeed?.pickModel === true); const [pickerOpen, setPickerOpen] = useState(false); + // The picker opens a frame after the pane does: `DetailsPane` moves focus into the pane as it + // opens (a parent's effect, which runs after this one), and focus leaving an open popover closes + // it, so opened at once it would close again before anyone saw it. + const [pickerRequested, setPickerRequested] = useState(intentSeed?.pickModel === true); + useEffect(() => { + if (!pickerRequested) return; + const frame = requestAnimationFrame(() => { + setPickerOpen(true); + setPickerRequested(false); + }); + return () => cancelAnimationFrame(frame); + }, [pickerRequested]); + const lastSeed = useRef(intentSeed); + useEffect(() => { + if (!intentSeed || intentSeed === lastSeed.current) return; + lastSeed.current = intentSeed; + setSeed(intentSeed.draft); + setTask(intentSeed.task); + if (intentSeed.pickModel) { + setPicking(true); + setPickerRequested(true); + } + }, [intentSeed]); + const [savedDefault, setSavedDefault] = useState(null); + const [saveFailure, setSaveFailure] = useState(null); const [modelInvalid, setModelInvalid] = useState(false); const [advancedOpen, setAdvancedOpen] = useState(false); // From the click until the pane closes, Start reads "Starting…" (Q2-67). A failure or an @@ -293,69 +330,85 @@ export function AgentTaskPane({ onShowTask, className }: AgentTaskPaneProps) { const mentioned = useMemo(() => mentionedFileNames(task), [task]); const sharedFiles = useSharedFiles(crew, channel?.id, mentioned.length > 0); - const selected = choice ?? models.defaults; - const summary = !picking && choice === null && models.defaults !== null; + // The model a Crew task starts on (OW-11), else the app's default: the summary's model. + const fallback = models.taskDefault ?? models.defaults; + const selected = choice ?? fallback; + const summary = !picking && choice === null && fallback !== null; const noModels = models.providers !== null && models.providers.length === 0; const selectedProvider = models.providers?.find((item) => item.name === selected?.provider); const unknown = crew.unknownRunDestination; const pending = crew.isPending('run.start'); + // An open workspace whose broker reads wide: the task reads every channel the person is in, and + // Also read lists them ticked, to untick (`wideRunChannels`). Elsewhere, only what is ticked. + const wide = useMemo( + () => wideRunChannels({ snapshot, channel, capabilities: crew.capabilities }), + [snapshot, channel, crew.capabilities] + ); + const [unticked, setUnticked] = useState>(() => new Set()); + const contextChannels = useMemo( + () => (wide ? wide.filter((id) => !unticked.has(id)) : crew.contextChannels), + [wide, unticked, crew.contextChannels] + ); const otherChannels = useMemo( - () => (snapshot?.channels ?? []).filter((item) => item.id !== channel?.id && !item.archived), - [snapshot, channel] + () => + wide + ? wide + .map((id) => snapshot?.channels.find((item) => item.id === id)) + .filter((item): item is NonNullable => item !== undefined) + : (snapshot?.channels ?? []).filter((item) => item.id !== channel?.id && !item.archived), + [wide, snapshot, channel] ); const channelLabels = useMemo( () => channelNamesAcrossTeams(snapshot?.channels ?? [], snapshot?.teams ?? []), [snapshot] ); const known = useMemo(() => knownInstitutions(models.providers), [models.providers]); - const privateOnly = protectedRunContext({ - connection: crew.connection, - snapshot, - channel, - contextChannels: crew.contextChannels, - }); if (!channel) return null; const here = channelName(channel); - const institution = runInstitution({ - connection: crew.connection, - snapshot, - channel, - contextChannels: crew.contextChannels, - known, - }); - const notApproved = (provider: ProviderDetails) => - modelMismatch(provider, institution) && institution - ? agentCopy.notApproved(institution.label) - : null; const shown = selected ? modelDisplay(selected, selectedProvider) : null; const shownName = shown ? agentCopy.modelChoice(shown.model, shown.provider) : ''; - const mismatch = modelMismatch(selectedProvider, institution); - const mismatchText = - mismatch && institution && shown - ? mismatch.affiliation - ? agentCopy.institutionMismatch( - shown.model, - mismatch.affiliation, - workspace, - institution.label - ) - : agentCopy.institutionUnstated(shown.model, workspace, institution.label) - : null; - // A public model where the daemon refuses one (AG-F4): said before Start, which it disables, as a - // model the institution has not approved is. - const publicText = publicModelRefusal({ + const { privateOnly, institution, mismatchText, publicText, blockText } = taskStartCheck({ provider: selectedProvider, + providers: models.providers, connection: crew.connection, snapshot, channel, - contextChannels: crew.contextChannels, + contextChannels, workspace, channelLabel: (item) => channelLabels.get(item.id) ?? channelName(item), + model: shown?.model ?? null, }); - const blockText = mismatchText ?? publicText; + const notApproved = (provider: ProviderDetails) => + modelMismatch(provider, institution) && institution + ? agentCopy.notApproved(institution.label) + : null; + // "Use for future tasks" (OW-11): offered under a model that is not already the one Crew tasks + // start on, where the configuration can be written and the model could run a Crew task. + const offerAsDefault = + models.saveTaskDefault !== undefined && + selected !== null && + crewTaskModelUsable(selectedProvider) && + (models.taskDefault?.provider !== selected.provider || + models.taskDefault?.model !== selected.model); + const saveForFutureTasks = async () => { + if (!selected || !models.saveTaskDefault) return; + const name = shownName; + setSaveFailure(null); + try { + await models.saveTaskDefault(selected); + setSavedDefault(name); + } catch (failure) { + setSavedDefault(null); + setSaveFailure( + agentCopy.useForFutureTasksFailed( + failure instanceof Error && failure.message ? failure.message : String(failure) + ) + ); + } + }; // The daemon's refusal in the pane's words: the same sentence the pane says before Start when it // can, else what it does know. const errorText = @@ -372,15 +425,9 @@ export function AgentTaskPane({ onShowTask, className }: AgentTaskPaneProps) { const lines = task.split('\n').length; const longTask = lines > LONG_TASK_LINES || task.length > LONG_TASK_CHARS; const hasAdvanced = - otherChannels.length > 0 || - crew.contextChannels.length > 0 || - Boolean(crew.connection?.remote_root); - const folderText = crew.connection?.remote_root - ? crew.connection.remote_execution - ? agentCopy.folderExec(crew.connection.remote_root) - : agentCopy.folderRead(crew.connection.remote_root) - : null; - const alsoReads = crew.contextChannels + otherChannels.length > 0 || contextChannels.length > 0 || Boolean(crew.connection?.remote_root); + const folderText = remoteAccessText(crew.connection); + const alsoReads = contextChannels .filter((id) => id !== channel.id) .map((id) => { const item = snapshot?.channels.find((candidate) => candidate.id === id); @@ -420,7 +467,7 @@ export function AgentTaskPane({ onShowTask, className }: AgentTaskPaneProps) { prompt: task, provider: selected.provider, model: selected.model, - contextChannels: crew.contextChannels, + contextChannels, deliberateRestart, }); if (started) { @@ -569,7 +616,7 @@ export function AgentTaskPane({ onShowTask, className }: AgentTaskPaneProps) { className="h-auto p-0 text-label" aria-label={agentCopy.modelChangeName} onClick={() => { - setChoice(models.defaults); + setChoice(fallback); setPicking(true); setPickerOpen(true); }} @@ -616,6 +663,8 @@ export function AgentTaskPane({ onShowTask, className }: AgentTaskPaneProps) { setChoice(next); setPicking(true); setModelInvalid(false); + setSavedDefault(null); + setSaveFailure(null); // A refusal of the model just replaced is no longer about the choice on screen. if ( error?.source === 'pane:agent' && @@ -632,13 +681,41 @@ export function AgentTaskPane({ onShowTask, className }: AgentTaskPaneProps) { )} )} + {/* A link, not a checkbox (C9). The saved model is what the next summary shows. */} + {offerAsDefault && !noModels && ( +
+ +
+ )} + {saveFailure ? ( +

+ {saveFailure} +

+ ) : savedDefault && !offerAsDefault ? ( +

+ {agentCopy.usedForFutureTasks(savedDefault)} +

+ ) : null} {hasAdvanced && ( 0 + ? [agentCopy.readsChannels(alsoReads.length + 1), folderText] + .filter(Boolean) + .join(' · ') + : agentCopy.advancedSummary(here, alsoReads, folderText) + } >
{otherChannels.length > 0 && ( @@ -652,14 +729,24 @@ export function AgentTaskPane({ onShowTask, className }: AgentTaskPaneProps) { className="flex min-w-0 items-center gap-2 text-label text-text-default" > + checked={contextChannels.includes(item.id)} + onChange={(event) => { + const on = event.target.checked; + if (wide) { + setUnticked((current) => { + const next = new Set(current); + if (on) next.delete(item.id); + else next.add(item.id); + return next; + }); + return; + } crew.setContextChannels( - event.target.checked + on ? [...crew.contextChannels, item.id] : crew.contextChannels.filter((id) => id !== item.id) - ) - } + ); + }} /> {channelLabels.get(item.id) ?? channelName(item)} @@ -675,7 +762,11 @@ export function AgentTaskPane({ onShowTask, className }: AgentTaskPaneProps) {
-

{agentCopy.scope(here)}

+

+ {alsoReads.length > 0 && wide + ? agentCopy.scopeWide(here, alsoReads.length + 1) + : agentCopy.scope(here)} +

{/* Before Start, which it disables: the pane says what the daemon would refuse (T-47), for a model the institution has not approved and a public model alike (AG-F4). */} {blockText && ( diff --git a/ui/desktop/src/components/crew/pane/copy.ts b/ui/desktop/src/components/crew/pane/copy.ts index 998bd023f..e305e59d7 100644 --- a/ui/desktop/src/components/crew/pane/copy.ts +++ b/ui/desktop/src/components/crew/pane/copy.ts @@ -157,8 +157,20 @@ export const agentCopy = { return [reads, folder].filter(Boolean).join(' · '); }, alsoRead: 'Also read', + /** + * Advanced's closed summary in an open workspace whose broker reads wide (`wideRunChannels`): + * the task reads every channel the person is in. `count` includes the channel it posts in. + */ + readsChannels: (count: number) => `Reads ${count} channels`, folderExec: (path: string) => `Can run commands in ${path}`, folderRead: (path: string) => `Can read ${path}`, + /** + * Full access (design OW-6): commands run as the person's own account on the server, through + * their own bridge, with the work folder as where they start. `server` is what the person calls + * the server; `path` the work folder. + */ + fullAccess: (server: string, path: string) => + `Can run commands as you on ${server || 'the server'}, starting in ${path}`, /** * A public model where the daemon refuses one (AG-F4), said before Start, which it disables, and * in place of the daemon's "Private cluster blocks public models" and the other public-model @@ -227,6 +239,19 @@ export const agentCopy = { * success as well as failure and cancel (T-25). Keep the two together. */ scope: (channel: string) => `Your agent can read ${channel} and post there, for this task only.`, + /** The same, for a task that reads other channels too. `count` includes `channel`. */ + scopeWide: (channel: string, count: number) => + `Your agent can read ${count} channels and post in ${channel}, for this task only.`, + /** + * Under the model when it is not the one Crew tasks start on (OW-11): a link that makes it so. + * Not a checkbox (C9: the unknown-outcome gate's is the only one). + */ + useForFutureTasks: 'Use for future tasks', + /** Said once the model for Crew tasks is saved; `model` as the summary names it. */ + usedForFutureTasks: (model: string) => `Future tasks start on ${model}.`, + /** The model for Crew tasks could not be saved. */ + useForFutureTasksFailed: (message: string) => + `Couldn’t save the model for Crew tasks. ${message}`, /** Pinned. */ start: 'Start my agent and allow posting here', /** Start's label, beside a spinner, from the click until the pane closes (Q2-67). */ diff --git a/ui/desktop/src/components/crew/pane/presentation.ts b/ui/desktop/src/components/crew/pane/presentation.ts index 78f8effaf..daf377776 100644 --- a/ui/desktop/src/components/crew/pane/presentation.ts +++ b/ui/desktop/src/components/crew/pane/presentation.ts @@ -10,6 +10,7 @@ import { import type { Channel, CrewConnection, Snapshot, Team } from '../crewApi'; import { connectionNames, + connectionServer, identityCopy, institutionLabel, isMachineIdShaped, @@ -553,6 +554,168 @@ export function publicModelRefusal({ return restricted ? agentCopy.publicRestricted(channelLabel(restricted)) : null; } +/** What a task started here would be refused for before it reaches the daemon, as far as Crew sees. */ +export interface TaskStartCheck { + /** The task's context is protected, so only a private model can run it (`protectedRunContext`). */ + privateOnly: boolean; + /** The institution a private model must be approved for, when the daemon will ask. */ + institution: RunInstitution | null; + /** The model is not approved for that institution, in the panes' words. */ + mismatchText: string | null; + /** A public model where the daemon refuses one, in the panes' words. */ + publicText: string | null; + /** Either of the two: what Start would be disabled for. */ + blockText: string | null; +} + +/** + * The checks Ask my agent makes before Start, as one pure function: the pane runs it on the model + * it shows, and the composer runs it on the model `@agent` would start, so the two can never + * disagree about a model (OW-10). A conservative mirror, like each check it is made of: it can miss + * a refusal the daemon makes (a restricted message, an earlier grant), never invent one, and it + * decides only whether to start at once or to open the pane. The daemon and the broker decide. + * + * `model` is the model's display name for the sentences (`modelDisplay`), or `null` while no model + * is chosen, when there is no institution sentence to say. + */ +export function taskStartCheck({ + provider, + providers, + connection, + snapshot, + channel, + contextChannels, + workspace, + channelLabel, + model, +}: { + provider: ProviderDetails | undefined; + providers: readonly ProviderDetails[] | null; + connection: CrewConnection | null; + snapshot: Snapshot | null; + channel: Channel | null; + contextChannels: readonly string[]; + workspace: string; + channelLabel(channel: Channel): string; + model: string | null; +}): TaskStartCheck { + const privateOnly = protectedRunContext({ connection, snapshot, channel, contextChannels }); + const institution = runInstitution({ + connection, + snapshot, + channel, + contextChannels, + known: knownInstitutions(providers), + }); + const mismatch = modelMismatch(provider, institution); + const mismatchText = + mismatch && institution && model !== null + ? mismatch.affiliation + ? agentCopy.institutionMismatch(model, mismatch.affiliation, workspace, institution.label) + : agentCopy.institutionUnstated(model, workspace, institution.label) + : null; + // A public model where the daemon refuses one (AG-F4): said before Start, which it disables, as a + // model the institution has not approved is. + const publicText = publicModelRefusal({ + provider, + connection, + snapshot, + channel, + contextChannels, + workspace, + channelLabel, + }); + return { + privateOnly, + institution, + mismatchText, + publicText, + blockText: mismatchText ?? publicText, + }; +} + +/** The broker reads up to 100 source channels for a run (design OW-3, `wide_runs_v1`). */ +export const WIDE_RUNS_CAPABILITY = 'wide_runs_v1'; +/** How many channels a wide run reads besides the one it posts in. */ +export const WIDE_RUN_EXTRA_CHANNELS = 99; + +/** + * Whether the workspace is open (design OW-1): every member manages its teams and channels. Read + * from a broker that says so; one that says nothing (before `open_governance_v1`) is managed. + */ +export function workspaceIsOpen(snapshot: Snapshot | null | undefined): boolean { + const workspace = snapshot?.workspace as { governance?: unknown } | undefined; + return workspace?.governance === 'open'; +} + +/** + * The channels a task started in `channel` reads by default besides it, or `null` where a task + * reads only what the person adds (every managed workspace, and an open one whose broker cannot + * take more than 20 sources). In an open workspace whose broker advertises `wide_runs_v1`, a task + * reads every channel the person is in and that is not archived, in the snapshot's order, up to + * {@link WIDE_RUN_EXTRA_CHANNELS} of them: an open workspace trusts its members with one another's + * channels, and the person could read each of them anyway. The pane lets them untick any. + * + * Presentation only. The broker checks the person's membership of every source when the run is + * created, and the daemon still refuses a public model for any Restricted one among them. + */ +export function wideRunChannels({ + snapshot, + channel, + capabilities, +}: { + snapshot: Snapshot | null; + channel: Channel | null; + capabilities: readonly string[] | null | undefined; +}): string[] | null { + if (!snapshot || !channel || !workspaceIsOpen(snapshot)) return null; + if (!Array.isArray(capabilities) || !capabilities.includes(WIDE_RUNS_CAPABILITY)) return null; + const me = snapshot.actor.id; + return snapshot.channels + .filter( + (item) => + item.id !== channel.id && + !item.archived && + Array.isArray(item.members) && + item.members.includes(me) + ) + .slice(0, WIDE_RUN_EXTRA_CHANNELS) + .map((item) => item.id); +} + +/** + * What a connection lets the person's agent do on the server (design OW-6), from a daemon that + * says so: `full` only when the connection says `remote_access: "full"`, `folder` for any other + * connection with a work folder, and `null` without one. An older daemon sends no `remote_access`, + * which reads as the confined folder it always was. + */ +export function connectionRemoteAccess( + connection: CrewConnection | null | undefined +): 'full' | 'folder' | null { + if (!connection?.remote_root) return null; + return (connection as { remote_access?: unknown }).remote_access === 'full' ? 'full' : 'folder'; +} + +/** + * The line Ask my agent and the composer's consent say about the server, or `null` without a work + * folder: "Can run commands as you on leo, starting in ~/crew-work/lab" (full access), "Can run + * commands in …" or "Can read …" (the confined folder). The server is named as the sidebar names + * it (`serverLabel`): the person's own alias for it, else the host. Only a private model gets any + * of it (the daemon sends no folder for a public one, and refuses a public model wherever there is + * a folder). + */ +export function remoteAccessText(connection: CrewConnection | null | undefined): string | null { + const root = connection?.remote_root; + const access = connectionRemoteAccess(connection); + if (!connection || !root || !access) return null; + if (access === 'full') + return agentCopy.fullAccess( + sanitizeDisplayText(connection.server_label) || connectionServer(connection), + root + ); + return connection.remote_execution ? agentCopy.folderExec(root) : agentCopy.folderRead(root); +} + /** * A model refusal in the panes' words (AG-F4, SF-F4): the institution refusal as the sentence the * pane already says before Start (`mismatch`), else the daemon's own sentence when a coded daemon diff --git a/ui/desktop/src/components/crew/pane/useConfiguredModels.ts b/ui/desktop/src/components/crew/pane/useConfiguredModels.ts index ba741cc1a..491279402 100644 --- a/ui/desktop/src/components/crew/pane/useConfiguredModels.ts +++ b/ui/desktop/src/components/crew/pane/useConfiguredModels.ts @@ -1,20 +1,36 @@ -import { useEffect, useState } from 'react'; +import { useCallback, useEffect, useState } from 'react'; import type { ProviderDetails } from '../../../api'; +import { subscribeAppModelSelectionChanges } from '../../../utils/sessionBindingSync'; import { useConfig } from '../../ConfigContext'; +import { + CREW_TASK_MODEL_KEY, + resolveTaskModel, + type TaskModelChoice, + type TaskModelSource, +} from '../state/crewTaskModel'; /** A provider and one of its models, as the run request names them. */ -export interface ModelChoice { - provider: string; - model: string; -} +export type ModelChoice = TaskModelChoice; export interface ConfiguredModels { /** Configured providers, in the daemon's order; `null` until the list has loaded. */ providers: ProviderDetails[] | null; /** `BIOROUTER_PROVIDER` / `BIOROUTER_MODEL`, only when both resolve to a configured provider. */ defaults: ModelChoice | null; + /** + * The model a Crew task starts on (OW-11): `CREW_TASK_MODEL`, else the app's default, each only + * when it names a configured provider that can run a Crew task (never Claude Code or Codex). + */ + taskDefault: ModelChoice | null; + /** Where `taskDefault` came from, or `null` with it. */ + taskDefaultSource: TaskModelSource | null; /** The list could not be loaded; the message is the daemon's. */ failure: string | null; + /** + * Make `choice` the model for Crew tasks (`CREW_TASK_MODEL`). Absent where the configuration + * cannot be written (outside a `ConfigProvider`). Rejects when the daemon refused the write. + */ + saveTaskDefault?(choice: ModelChoice): Promise; } /** @@ -37,32 +53,57 @@ export function providerLabel(provider: ProviderDetails | undefined, fallback = return typeof name === 'string' && name.trim() ? name : (provider?.name ?? fallback); } +const EMPTY: ConfiguredModels = { + providers: null, + defaults: null, + taskDefault: null, + taskDefaultSource: null, + failure: null, +}; + /** - * The configured providers and the app's default model, read when Ask my agent opens. + * The configured providers, the app's default model and the model for Crew tasks, read when Ask + * my agent opens (or, for the composer, the first time `@agent` needs them: `enabled`). * * Crew bypasses provider onboarding, so a person can arrive here with nothing configured: then * `providers` is an empty list, which the pane turns into "No models are set up." A failed read is * reported separately and never claims that nothing is set up. + * + * Read again whenever a window announces a write of the app's model selection or of + * `CREW_TASK_MODEL` (`ConfigContext` announces both), so a change made in Settings reaches a pane + * or a composer that is already open. */ -export function useConfiguredModels(options: { optional?: boolean } = {}): ConfiguredModels { +export function useConfiguredModels( + options: { optional?: boolean; enabled?: boolean } = {} +): ConfiguredModels { const config = useOptionalConfig(options.optional === true); const getProviders = config?.getProviders; const read = config?.read; - const [state, setState] = useState({ - providers: null, - defaults: null, - failure: null, - }); + const upsert = config?.upsert; + const enabled = options.enabled !== false; + const [state, setState] = useState(EMPTY); + // Once asked for, the answer is kept and re-read on every announcement, whether or not the + // caller still wants it: a composer that types `@agent` again finds the model it showed. + const [wanted, setWanted] = useState(enabled); + useEffect(() => { + if (enabled) setWanted(true); + }, [enabled]); + const [version, setVersion] = useState(0); + useEffect(() => { + if (!wanted) return; + return subscribeAppModelSelectionChanges(() => setVersion((current) => current + 1)); + }, [wanted]); useEffect(() => { - if (!getProviders || !read) return; + if (!wanted || !getProviders || !read) return; let active = true; void Promise.all([ getProviders(false), read('BIOROUTER_PROVIDER', false), read('BIOROUTER_MODEL', false), + read(CREW_TASK_MODEL_KEY, false), ]) - .then(([items, provider, model]) => { + .then(([items, provider, model, crew]) => { if (!active) return; const configured = (Array.isArray(items) ? items : []).filter( (item): item is ProviderDetails => @@ -75,20 +116,40 @@ export function useConfiguredModels(options: { optional?: boolean } = {}): Confi configured.some((item) => item.name === provider) ? { provider, model: model.trim() } : null; - setState({ providers: configured, defaults, failure: null }); + const task = resolveTaskModel({ + crew, + appProvider: provider, + appModel: model, + configured, + }); + setState({ + providers: configured, + defaults, + taskDefault: task?.choice ?? null, + taskDefaultSource: task?.source ?? null, + failure: null, + }); }) .catch((failure: unknown) => { if (!active) return; setState({ - providers: null, - defaults: null, + ...EMPTY, failure: failure instanceof Error && failure.message ? failure.message : String(failure), }); }); return () => { active = false; }; - }, [getProviders, read]); + }, [wanted, getProviders, read, version]); + + const saveTaskDefault = useCallback( + async (choice: ModelChoice) => { + if (!upsert) return; + // `upsert` announces the write once it resolved, which reads the models again everywhere. + await upsert(CREW_TASK_MODEL_KEY, { provider: choice.provider, model: choice.model }, false); + }, + [upsert] + ); - return state; + return upsert ? { ...state, saveTaskDefault } : state; } diff --git a/ui/desktop/src/components/crew/state/crewActions.ts b/ui/desktop/src/components/crew/state/crewActions.ts index fecd56c9a..916957fb2 100644 --- a/ui/desktop/src/components/crew/state/crewActions.ts +++ b/ui/desktop/src/components/crew/state/crewActions.ts @@ -113,6 +113,7 @@ export function useCrewActions(): CrewActions { ...(code !== undefined ? { code } : {}), ...(details?.destination !== undefined ? { destination: details.destination } : {}), ...(details?.transport ? { transport: true } : {}), + ...(details?.action ? { action: details.action } : {}), }), [] ); diff --git a/ui/desktop/src/components/crew/state/crewRunStart.ts b/ui/desktop/src/components/crew/state/crewRunStart.ts index 022c8eda4..4b1dfa9e0 100644 --- a/ui/desktop/src/components/crew/state/crewRunStart.ts +++ b/ui/desktop/src/components/crew/state/crewRunStart.ts @@ -15,7 +15,7 @@ import { crewHttp, CrewHttpError, type Channel, type ObservedRun, type Snapshot import { channelName, teamName } from '../identity/objectNames'; import { crewActionCopy } from './copy'; import { postDestination } from './crewSend'; -import { resetBetweenTests } from './draftStash'; +import { forgetStashedDraft, resetBetweenTests, stashedDraft } from './draftStash'; import { draftScope, draftScopeChanged, @@ -27,6 +27,7 @@ import { import type { ActionKey, ActOptions, + AskAgentInput, ErrorDetails, ErrorSource, ObservedPrivacy, @@ -156,6 +157,11 @@ export interface CrewRunStartContext { */ generation: MutableRefObject; setBody: Dispatch>; + /** + * Take what was submitted out of the composer's body, leaving words typed since (QA M8), as a + * send does. `askAgent` calls it after a start from the composer; absent, the body stays. + */ + clearSentBody?(sent: string): void; /** * Observe again without clearing what is on screen: the verified view stays until the new * observation's first frame replaces it, or its end clears it. @@ -177,6 +183,7 @@ export interface CrewRunStartContext { export interface CrewRunStart { startOwnedRun(input: StartOwnedRunInput): Promise; + askAgent(input: AskAgentInput): Promise; unknownRunDestination: UnknownRunDestination | null; inspectedPriorRun: boolean; setInspectedPriorRun: Dispatch>; @@ -225,6 +232,7 @@ export function useCrewRunStart(context: CrewRunStartContext): CrewRunStart { runs, generation, setBody, + clearSentBody, restartObservation, resetSurfaces, act, @@ -385,11 +393,15 @@ export function useCrewRunStart(context: CrewRunStartContext): CrewRunStart { }; /** - * A failed start, told where the person is (MSG2-N10): in the pane's slot while they are still - * on the channel the task was for, else in the connection bar, naming that channel. A failure of - * the link is marked so, so the connection verifying again takes it away instead of leaving it - * red under "Connected". Unless the task may have started (the gate above handles that), its - * words are kept for that channel's Ask my agent, which offers them again. + * A failed start, told where the person is (MSG2-N10): in the slot of the surface that started + * it while they are still on the channel the task was for — Ask my agent's, or the composer's for + * `@agent` (OW-10), marked `task-start` so it leads "Couldn't start your agent." and names the + * draft it answers — else in the connection bar, naming that channel. A failure of the link is + * marked so, so the connection verifying again takes it away instead of leaving it red under + * "Connected". Unless the task may have started (the gate above handles that), a pane task's + * words are kept for that channel's Ask my agent, which offers them again. A composer task's + * words are still the draft (put aside with the channel when the person moves on), so they are + * not kept twice. */ const tellStartFailure = ( failure: unknown, @@ -401,12 +413,22 @@ export function useCrewRunStart(context: CrewRunStartContext): CrewRunStart { ) => { const outcomeUnknown = failure instanceof CrewHttpError && failure.code === 'crew_start_outcome_unknown'; - if (!outcomeUnknown && frame) keepFailedTask(forConnection, forChannel, input.prompt, frame); + const composer = input.surface === 'composer'; + if (!outcomeUnknown && frame && !composer) + keepFailedTask(forConnection, forChannel, input.prompt, frame); const message = failureMessage(failure, crewActionCopy.actionFallback); const code = failureCode(failure); const details = isTransportFailure(failure) ? { transport: true } : undefined; const now = view.current; if (now.connectionId === forConnection && now.channelId === forChannel) { + if (composer) { + reportError(message, 'composer', code, { + ...details, + destination: postDestination(forConnection, forChannel), + action: 'task-start', + }); + return; + } reportError(message, 'pane:agent', code, details); return; } @@ -425,8 +447,10 @@ export function useCrewRunStart(context: CrewRunStartContext): CrewRunStart { const forChannel = channelId; const channelLabel = channelName(channel); const frame = viewScopeFrame(connectionId, snapshot, observedPrivacy); + // One pending key for both surfaces, so the pane and the composer cannot start two at once. + const source = input.surface === 'composer' ? 'composer' : 'pane:agent'; return ( - (await act('pane:agent', 'run.start', async () => { + (await act(source, 'run.start', async () => { try { return await submitOwnedRun(input); } catch (failure) { @@ -437,8 +461,35 @@ export function useCrewRunStart(context: CrewRunStartContext): CrewRunStart { ); }; + /** + * `@agent` from the composer (OW-10): the pane's request exactly — the same verified epochs, + * `posting_grant`, idempotency and unknown-outcome lock — with its failures told in the + * composer. On success what was submitted leaves the draft, and words typed since stay; a draft + * put aside for the channel meanwhile (the person moved on while it started) is forgotten while it + * is still those words, so coming back cannot offer to start the same task again. + */ + const askAgent = async ({ prompt, draftBody, model, contextChannels }: AskAgentInput) => { + const forConnection = connectionId; + const forChannel = channelId; + const started = await startOwnedRun({ + prompt, + provider: model.provider, + model: model.model, + contextChannels, + surface: 'composer', + }); + if (!started) return false; + const now = view.current; + if (now.connectionId === forConnection && now.channelId === forChannel) + clearSentBody?.(draftBody); + if (stashedDraft(forConnection, forChannel)?.body === draftBody) + forgetStashedDraft(forConnection, forChannel); + return true; + }; + return { startOwnedRun, + askAgent, unknownRunDestination, inspectedPriorRun, setInspectedPriorRun, diff --git a/ui/desktop/src/components/crew/state/crewTaskModel.test.ts b/ui/desktop/src/components/crew/state/crewTaskModel.test.ts new file mode 100644 index 000000000..f8190d045 --- /dev/null +++ b/ui/desktop/src/components/crew/state/crewTaskModel.test.ts @@ -0,0 +1,103 @@ +import { describe, expect, it } from 'vitest'; +import type { ProviderDetails } from '../../../api'; +import { + CREW_TASK_MODEL_KEY, + crewTaskModelUsable, + parseTaskModel, + resolveTaskModel, +} from './crewTaskModel'; + +const provider = (name: string, configured = true) => + ({ name, is_configured: configured, metadata: { display_name: name } }) as ProviderDetails; + +const versa = provider('versa_azure'); +const ollama = provider('ollama'); +const claudeCode = provider('claude_code'); +const codex = provider('codex'); +const unconfigured = provider('openai', false); + +describe('the model Crew tasks start on (OW-11)', () => { + it('is the config key the CLI reads too', () => { + expect(CREW_TASK_MODEL_KEY).toBe('CREW_TASK_MODEL'); + }); + + it('reads a stored choice only when both fields are non-blank strings', () => { + expect(parseTaskModel({ provider: 'ollama', model: ' qwen3.8:27b ' })).toEqual({ + provider: 'ollama', + model: 'qwen3.8:27b', + }); + for (const value of [ + null, + undefined, + '', + 'ollama', + [], + { provider: 'ollama' }, + { model: 'qwen3.8:27b' }, + { provider: ' ', model: 'qwen3.8:27b' }, + { provider: 'ollama', model: 7 }, + ]) + expect(parseTaskModel(value)).toBeNull(); + }); + + it('never offers Claude Code or Codex, which the daemon refuses for every Crew task', () => { + expect(crewTaskModelUsable(versa)).toBe(true); + expect(crewTaskModelUsable(claudeCode)).toBe(false); + expect(crewTaskModelUsable(codex)).toBe(false); + expect(crewTaskModelUsable(unconfigured)).toBe(false); + expect(crewTaskModelUsable(undefined)).toBe(false); + }); + + it('resolves the Crew setting first, then the app’s default, and says which', () => { + const configured = [versa, ollama]; + expect( + resolveTaskModel({ + crew: { provider: 'ollama', model: 'qwen3.8:27b' }, + appProvider: 'versa_azure', + appModel: 'gpt-5.5', + configured, + }) + ).toEqual({ choice: { provider: 'ollama', model: 'qwen3.8:27b' }, source: 'crew' }); + expect( + resolveTaskModel({ crew: null, appProvider: 'versa_azure', appModel: 'gpt-5.5', configured }) + ).toEqual({ choice: { provider: 'versa_azure', model: 'gpt-5.5' }, source: 'app' }); + }); + + it('falls back to the app’s default when the Crew setting names an unconfigured or unusable provider', () => { + for (const crew of [ + { provider: 'openai', model: 'gpt-6-sol' }, + { provider: 'claude_code', model: 'opus' }, + { provider: 'versa_azure' }, + ]) + expect( + resolveTaskModel({ + crew, + appProvider: 'versa_azure', + appModel: 'gpt-5.5', + configured: [versa, claudeCode, unconfigured], + })?.source + ).toBe('app'); + }); + + it('resolves nothing when neither names a model a Crew task could start on', () => { + expect( + resolveTaskModel({ + crew: '', + appProvider: 'claude_code', + appModel: 'opus', + configured: [claudeCode, versa], + }) + ).toBeNull(); + expect( + resolveTaskModel({ + crew: null, + appProvider: 'versa_azure', + appModel: ' ', + configured: [versa], + }) + ).toBeNull(); + expect( + resolveTaskModel({ crew: null, appProvider: null, appModel: null, configured: [] }) + ).toBeNull(); + }); +}); diff --git a/ui/desktop/src/components/crew/state/crewTaskModel.ts b/ui/desktop/src/components/crew/state/crewTaskModel.ts new file mode 100644 index 000000000..3766cfb67 --- /dev/null +++ b/ui/desktop/src/components/crew/state/crewTaskModel.ts @@ -0,0 +1,82 @@ +import type { ProviderDetails } from '../../../api'; +import { CODING_AGENT_ORDER } from '../../onboarding/codingAgentStatus'; + +/** + * The model Crew tasks start on (design OW-11): the config key `CREW_TASK_MODEL`, an object + * `{provider, model}` in `config.yaml`, written by Settings › Models › Crew tasks and by the pane's + * "Use for future tasks", and read by the CLI the same way (`/config/read` on the shared daemon). + * + * A preference, not a capability key: the daemon never reads it. Every task names its provider and + * model in a request a person proves, and the daemon and the broker admit each one, so a default + * that cannot run somewhere is refused there exactly as a model chosen in the pane would be. + * + * The CLI reads and writes the same key through the same daemon (`biorouter crew defaults`), with + * the same fallback, so the terminal and the app start a task on one model. + */ +export const CREW_TASK_MODEL_KEY = 'CREW_TASK_MODEL'; + +/** A provider and one of its models, as the run request names them. */ +export interface TaskModelChoice { + provider: string; + model: string; +} + +/** Where the model a Crew task starts on came from: the Crew setting, or the app's default. */ +export type TaskModelSource = 'crew' | 'app'; + +export interface ResolvedTaskModel { + choice: TaskModelChoice; + source: TaskModelSource; +} + +const text = (value: unknown): string => (typeof value === 'string' ? value.trim() : ''); + +/** + * `CREW_TASK_MODEL` as stored, or `null` unless both fields are non-blank strings: a half-written + * value is "unset", never half a choice. + */ +export function parseTaskModel(value: unknown): TaskModelChoice | null { + if (!value || typeof value !== 'object' || Array.isArray(value)) return null; + const provider = text((value as { provider?: unknown }).provider); + const model = text((value as { model?: unknown }).model); + return provider && model ? { provider, model } : null; +} + +/** + * Whether a configured provider can run a Crew task at all. Claude Code and Codex never can: the + * daemon refuses a tool-bridge provider for every Crew task (`uses_tool_bridge`), so offering one + * as the default would make every `@agent` fail. + */ +export function crewTaskModelUsable(provider: ProviderDetails | undefined): boolean { + return ( + provider !== undefined && + provider.is_configured === true && + !(CODING_AGENT_ORDER as readonly string[]).includes(provider.name) + ); +} + +/** + * The model a Crew task starts on: `CREW_TASK_MODEL` when it names a usable configured provider, + * else the app's `BIOROUTER_PROVIDER` / `BIOROUTER_MODEL` under the same rule, else `null`. The + * source says which, so Settings can say "Same as new chats". + */ +export function resolveTaskModel({ + crew, + appProvider, + appModel, + configured, +}: { + crew: unknown; + appProvider: unknown; + appModel: unknown; + configured: readonly ProviderDetails[]; +}): ResolvedTaskModel | null { + const usable = (choice: TaskModelChoice | null) => + choice !== null && + crewTaskModelUsable(configured.find((item) => item.name === choice.provider)); + const own = parseTaskModel(crew); + if (own && usable(own)) return { choice: own, source: 'crew' }; + const app = parseTaskModel({ provider: appProvider, model: appModel }); + if (app && usable(app)) return { choice: app, source: 'app' }; + return null; +} diff --git a/ui/desktop/src/components/crew/state/types.ts b/ui/desktop/src/components/crew/state/types.ts index 43a20334b..f5d22e88e 100644 --- a/ui/desktop/src/components/crew/state/types.ts +++ b/ui/desktop/src/components/crew/state/types.ts @@ -144,10 +144,22 @@ export type DialogIntent = | { kind: 'confirm'; confirm: ConfirmIntent }; export type DialogKind = DialogIntent['kind']; +/** + * What Ask my agent opens with when the composer opens it for `@agent` (OW-10): the task's words + * without the trigger, the whole draft they came from (cleared on success only while it is still + * that draft), and whether to open the model picker at once (the model was blocked or missing). + * Memory only: `rememberPaneIntent` keeps no agent intent, so task words never reach storage. + */ +export interface AgentPaneSeed { + task: string; + draft: string; + pickModel?: boolean; +} + /** The one non-modal details pane has one mode at a time. */ export type PaneIntent = | { mode: 'details'; tab?: DetailsTab } - | { mode: 'agent' } + | { mode: 'agent'; seed?: AgentPaneSeed } | { mode: 'chat-access'; sessionId?: string }; export type PaneMode = PaneIntent['mode']; @@ -185,6 +197,12 @@ export interface CrewActionError { * connection verifies again, so it is dismissed then instead of staying under "Connected". */ transport?: boolean; + /** + * What the person was doing when it failed, where the surface words it differently: `task-start` + * is an agent task started from the composer with `@agent` (OW-10), which the composer leads with + * "Couldn't start your agent." rather than "Couldn't send." + */ + action?: 'task-start'; } /** @@ -201,10 +219,14 @@ export interface LostDraft { body: string; } -/** What an error may say beyond its words: the composer it belongs to, and whether it is the link's. */ +/** + * What an error may say beyond its words: the composer it belongs to, whether it is the link's, and + * what the person was doing (`CrewActionError.action`). + */ export interface ErrorDetails { destination?: string; transport?: boolean; + action?: 'task-start'; } /** @@ -259,6 +281,24 @@ export interface StartOwnedRunInput { * `clearBodyIfEquals(seed)` after a start. */ clearBody?: boolean; + /** + * Which surface started it, and so where a failure is told: `pane` (the default) in Ask my + * agent's slot, `composer` in the composer's note with its own lead (`@agent`, OW-10). Never sent + * to the daemon. + */ + surface?: 'pane' | 'composer'; +} + +/** + * `@agent` from the composer (OW-10): the task's words without the trigger, the draft they came + * from, the model the consent line showed (started exactly as shown, never re-read at Enter) and + * the channels it reads besides this one. + */ +export interface AskAgentInput { + prompt: string; + draftBody: string; + model: { provider: string; model: string }; + contextChannels: string[]; } /** @@ -484,8 +524,18 @@ export interface CrewController { dismissLostDraft?(id: number): void; // Owned runs (module-scoped unknown-outcome lock, C10) - /** Records its error under `pane:agent`; resolves true only when the start was accepted. */ + /** + * Records its error under `pane:agent`, or the composer's for `surface: 'composer'`; resolves + * true only when the start was accepted. + */ startOwnedRun(input: StartOwnedRunInput): Promise; + /** + * Start the person's agent from the composer (`@agent`, `/agent`, `/task`; OW-10): the same + * request as Ask my agent's, with its errors in the composer. On success the submitted words + * leave the draft and words typed since stay. Absent on a stand-in controller, where the composer + * opens Ask my agent instead. + */ + askAgent?(input: AskAgentInput): Promise; /** Where a start whose outcome is unknown went, while that lock holds; else null. */ unknownRunDestination: UnknownRunDestination | null; inspectedPriorRun: boolean; diff --git a/ui/desktop/src/components/crew/state/useCrewController.ts b/ui/desktop/src/components/crew/state/useCrewController.ts index e2cc60711..ef33e0979 100644 --- a/ui/desktop/src/components/crew/state/useCrewController.ts +++ b/ui/desktop/src/components/crew/state/useCrewController.ts @@ -878,6 +878,7 @@ export function useCrewController(options: CrewControllerOptions = {}): CrewCont runs, generation, setBody: draft.setBody, + clearSentBody: draft.clearSentBody, restartObservation, resetSurfaces, act, @@ -1086,6 +1087,7 @@ export function useCrewController(options: CrewControllerOptions = {}): CrewCont dismissLostDraft, startOwnedRun: runStart.startOwnedRun, + askAgent: runStart.askAgent, unknownRunDestination: runStart.unknownRunDestination, inspectedPriorRun: runStart.inspectedPriorRun, setInspectedPriorRun: runStart.setInspectedPriorRun, From 7c6fd59fbb8f6f08e05afa36a87be082ca1ef5cc Mon Sep 17 00:00:00 2001 From: Wanjun Gu Date: Fri, 16 Oct 2026 19:54:54 -0700 Subject: [PATCH 04/32] crew: runs carry remote_access and remote_scope reports it (OW-6) run.create accepts remote_access "full" only with a remote root, remote execution and a private provider, stores it on the run (serialized only when full, so older journals replay byte for byte) and echoes it. run.remote_scope adds access, owner_id and channel_id, and the broker advertises remote_access_v1 after presence_v1. --- crates/biorouter-crew/src/broker.rs | 24 ++- crates/biorouter-crew/src/lib.rs | 63 ++++++ .../tests/remote_access_contract.rs | 191 ++++++++++++++++++ 3 files changed, 275 insertions(+), 3 deletions(-) create mode 100644 crates/biorouter-crew/tests/remote_access_contract.rs diff --git a/crates/biorouter-crew/src/broker.rs b/crates/biorouter-crew/src/broker.rs index 3b23c3ec2..8be2f5fde 100644 --- a/crates/biorouter-crew/src/broker.rs +++ b/crates/biorouter-crew/src/broker.rs @@ -1944,8 +1944,8 @@ impl Broker { Ok(hello) } /// What `hello` advertises, in the order it advertises it (the v2 signature covers the - /// order). - fn capabilities() -> Vec<&'static str> { + /// order). `biorouter-crew --version --output-format json` reports the same list. + pub fn capabilities() -> Vec<&'static str> { #[allow(unused_mut)] let mut capabilities = vec![ "human_chat", @@ -1956,6 +1956,7 @@ impl Broker { "unique_names_v1", "direct_add_v1", "presence_v1", + "remote_access_v1", ]; #[cfg(feature = "join-by-name")] capabilities.push("join_by_name_v1"); @@ -3068,8 +3069,11 @@ impl Broker { .remote_root .as_ref() .ok_or_else(|| anyhow!("forbidden: run has no remote path approval"))?; + // `access` decides which helper the owner's bridge runs; a bridge that finds none (an + // older broker) stays in folder mode. `owner_id` keys full-access jobs, which any run of + // the same owner may read, and `channel_id` is where a shared server file is uploaded. Ok( - json!({"root":root,"allow_execute":run.remote_execution,"public_provider":run.public_provider,"run_id":run.id,"policy_epoch":run.policy_epoch,"expires_at":run.expires_at}), + json!({"root":root,"allow_execute":run.remote_execution,"public_provider":run.public_provider,"run_id":run.id,"policy_epoch":run.policy_epoch,"expires_at":run.expires_at,"access":run.remote_access.as_str(),"owner_id":run.owner_id,"channel_id":run.channel_id}), ) } fn read_context_manifest(&self, actor: &Actor, _req: &Request) -> Result { @@ -4223,6 +4227,19 @@ impl Broker { !remote_execution || remote_root.is_some(), "invalid_params: execution requires remote root" ); + let remote_access = match p.get("remote_access") { + None | Some(Value::Null) => RemoteAccess::Folder, + Some(value) => serde_json::from_value(value.clone()).map_err(|_| { + anyhow!("invalid_params: remote_access must be \"folder\" or \"full\".") + })?, + }; + // Full access runs as the owner's whole account, so it needs everything a confined + // command needs: a work folder, the person's execution grant and a private model. + ensure!( + remote_access == RemoteAccess::Folder + || (remote_root.is_some() && remote_execution && !consent.public_provider), + "invalid_params: Full remote access needs a remote root, remote execution and a private provider." + ); let protected_context = Self::is_protected_context(s, &sources, &consent.personal_mode, remote_root.as_deref()); ensure!( @@ -4244,6 +4261,7 @@ impl Broker { connection_institution_id: consent.connection_institution_id, remote_root, remote_execution, + remote_access, id: id(), owner_id: who.clone(), channel_id: channel.into(), diff --git a/crates/biorouter-crew/src/lib.rs b/crates/biorouter-crew/src/lib.rs index fc1fe6b36..ca66e42c1 100644 --- a/crates/biorouter-crew/src/lib.rs +++ b/crates/biorouter-crew/src/lib.rs @@ -200,6 +200,34 @@ pub struct Run { pub remote_root: Option, #[serde(default)] pub remote_execution: bool, + /// What the run's agent may do on the server: the confined work folder, or full access as + /// the owner's own account (design OW-6). Serialized only when full, so a journal written + /// before this field existed replays and re-serializes byte for byte, and an older broker + /// ignores it. + #[serde(default, skip_serializing_if = "RemoteAccess::is_folder")] + pub remote_access: RemoteAccess, +} +/// A run's level of agent access on the server (design OW-6). "Off" is a run with no +/// `remote_root`, not a level. `Folder` is the confined work folder Crew always had; `Full` runs +/// commands and file operations as the owner's own account through the owner's own bridge. +#[derive(Clone, Copy, Debug, Default, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum RemoteAccess { + #[default] + Folder, + Full, +} +impl RemoteAccess { + pub fn is_folder(&self) -> bool { + *self == Self::Folder + } + /// The wire spelling, as `run.remote_scope` reports it. + pub fn as_str(&self) -> &'static str { + match self { + Self::Folder => "folder", + Self::Full => "full", + } + } } #[derive(Clone, Debug, Serialize, Deserialize)] pub struct Blob { @@ -375,3 +403,38 @@ pub use broker::{JournalCall, Quotas}; #[cfg(unix)] pub mod remote; + +#[cfg(test)] +mod tests { + use super::*; + + fn run_json() -> Value { + serde_json::json!({ + "id": "run", "owner_id": "owner", "channel_id": "channel", "source_channels": ["channel"], + "provider_policy_id": "private", "public_provider": false, "personal_mode": "private", + "policy_epoch": 1, "expires_at": 10, "revoked": false, "remote_root": "/home/bob/work", + "remote_execution": true + }) + } + + /// A run journaled before `remote_access` existed reads as a folder run, and a folder run + /// serializes without the field, so old journals replay and re-serialize byte for byte. + #[test] + fn a_run_without_remote_access_is_a_folder_run_and_stays_unchanged() { + let run: Run = serde_json::from_value(run_json()).unwrap(); + assert_eq!(run.remote_access, RemoteAccess::Folder); + let written = serde_json::to_value(&run).unwrap(); + assert!(written.get("remote_access").is_none(), "{written}"); + } + + #[test] + fn a_full_access_run_round_trips() { + let mut value = run_json(); + value["remote_access"] = serde_json::json!("full"); + let run: Run = serde_json::from_value(value).unwrap(); + assert_eq!(run.remote_access, RemoteAccess::Full); + assert_eq!(serde_json::to_value(&run).unwrap()["remote_access"], "full"); + assert_eq!(RemoteAccess::Full.as_str(), "full"); + assert_eq!(RemoteAccess::Folder.as_str(), "folder"); + } +} diff --git a/crates/biorouter-crew/tests/remote_access_contract.rs b/crates/biorouter-crew/tests/remote_access_contract.rs new file mode 100644 index 000000000..c6a0c6e7f --- /dev/null +++ b/crates/biorouter-crew/tests/remote_access_contract.rs @@ -0,0 +1,191 @@ +//! Full remote access in the broker (design OW-6): `run.create` takes `remote_access: "full"` +//! only with a work folder, the person's execution grant and a private provider, echoes it in +//! the run, and `run.remote_scope` reports the level and the run's owner to the bridge. +#![cfg(unix)] + +mod support; + +use biorouter_crew::{Broker, Request}; +use serde_json::{json, Value}; +use support::{ok, refused, TempRoot, Workspace}; + +/// `run.create` params for a private run in `channel`, with the policy fields the broker checks. +fn run_params(ws: &Workspace, channel: &str, extra: Value) -> Value { + let mut params = json!({ + "channel_id": channel, + "source_channels": [channel], + "provider_policy_id": "private-test", + "personal_mode": "private", + "public_provider": false, + "expires_in": 600, + "expected_workspace_policy_epoch": ws.broker.workspace().policy_epoch, + "workspace_institution_id": ws.broker.workspace().institution_id, + "connection_institution_id": ws.broker.workspace().institution_id, + "provider_affiliation": {"kind": "local"}, + "expected_protected_context": true, + }); + for (key, value) in extra.as_object().unwrap() { + params[key] = value.clone(); + } + params +} + +fn worker(ws: &mut Workspace, credential: &str, method: &str) -> biorouter_crew::Response { + let request = Request { + version: 1, + id: format!("worker-{method}"), + method: method.into(), + params: json!({}), + auth: None, + credential: Some(credential.into()), + }; + ws.broker + .handle(ws.host.uid, &mut ws.host.connection, request) +} + +#[test] +fn a_full_access_run_is_echoed_and_its_scope_names_the_level_and_owner() { + let mut ws = Workspace::new("full-access"); + let folder = TempRoot::new("full-access-work"); + let (_, channel) = ws.host_team("lab"); + let params = run_params( + &ws, + &channel, + json!({"remote_root": folder.path(), "remote_execution": true, "remote_access": "full"}), + ); + let created = ws.host_ok("run.create", params); + assert_eq!(created["run"]["remote_access"], "full"); + let credential = created["credential"].as_str().unwrap().to_owned(); + let scope = ok(worker(&mut ws, &credential, "run.remote_scope")); + assert_eq!(scope["access"], "full"); + assert_eq!(scope["owner_id"], ws.host.principal_id.as_str()); + assert_eq!(scope["channel_id"], channel.as_str()); + assert_eq!(scope["root"], folder.path().to_str().unwrap()); + assert_eq!(scope["allow_execute"], true); +} + +#[test] +fn a_run_without_remote_access_stays_a_folder_run() { + let mut ws = Workspace::new("folder-access"); + let folder = TempRoot::new("folder-access-work"); + let (_, channel) = ws.host_team("lab"); + for extra in [ + json!({"remote_root": folder.path(), "remote_execution": true}), + json!({"remote_root": folder.path(), "remote_access": "folder"}), + json!({"remote_root": folder.path(), "remote_access": null}), + ] { + let created = ws.host_ok("run.create", run_params(&ws, &channel, extra.clone())); + // A folder run serializes as it always did: no `remote_access` field. + assert!( + created["run"].get("remote_access").is_none(), + "{extra}: {created}" + ); + let credential = created["credential"].as_str().unwrap().to_owned(); + let scope = ok(worker(&mut ws, &credential, "run.remote_scope")); + assert_eq!(scope["access"], "folder", "{extra}"); + } +} + +/// Full access needs everything a confined command needs, and an unknown level is refused. +#[test] +fn full_access_needs_a_folder_execution_and_a_private_provider() { + let mut ws = Workspace::new("full-access-refusals"); + let folder = TempRoot::new("full-access-refusals-work"); + let (_, channel) = ws.host_team("lab"); + for (extra, code) in [ + (json!({"remote_access": "full"}), "invalid_params"), + ( + json!({"remote_root": folder.path(), "remote_access": "full"}), + "invalid_params", + ), + ( + json!({"remote_root": folder.path(), "remote_execution": false, "remote_access": "full"}), + "invalid_params", + ), + ( + json!({"remote_root": folder.path(), "remote_execution": true, "remote_access": "root"}), + "invalid_params", + ), + ( + json!({"remote_root": folder.path(), "remote_execution": true, "remote_access": true}), + "invalid_params", + ), + ] { + let params = run_params(&ws, &channel, extra.clone()); + let (got, message) = refused(ws.host_call("run.create", params)); + assert_eq!(got, code, "{extra}: {message}"); + } + // A public provider never gets a remote root at all, so never full access either. + let mut public = run_params( + &ws, + &channel, + json!({"remote_root": folder.path(), "remote_execution": true, "remote_access": "full"}), + ); + public["public_provider"] = json!(true); + let (code, _) = refused(ws.host_call("run.create", public)); + assert_eq!(code, "privacy_denied"); +} + +/// A worker credential cannot mint a run, full or not: `run.create` stays a person's request. +#[test] +fn a_worker_cannot_create_a_full_access_run() { + let mut ws = Workspace::new("full-access-worker"); + let folder = TempRoot::new("full-access-worker-work"); + let (_, channel) = ws.host_team("lab"); + let created = ws.host_ok( + "run.create", + run_params( + &ws, + &channel, + json!({"remote_root": folder.path(), "remote_execution": true, "remote_access": "full"}), + ), + ); + let credential = created["credential"].as_str().unwrap().to_owned(); + let request = Request { + version: 1, + id: "worker-mint".into(), + method: "run.create".into(), + params: run_params( + &ws, + &channel, + json!({"remote_root": folder.path(), "remote_execution": true, "remote_access": "full", "idempotency_key": "mint"}), + ), + auth: None, + credential: Some(credential), + }; + let response = ws + .broker + .handle(ws.host.uid, &mut ws.host.connection, request); + assert_eq!(response.error.unwrap().code, "forbidden"); +} + +/// The level replays from the journal like every other run field. +#[test] +fn a_full_access_run_survives_a_restart() { + let mut ws = Workspace::new("full-access-replay"); + let folder = TempRoot::new("full-access-replay-work"); + let (_, channel) = ws.host_team("lab"); + let created = ws.host_ok( + "run.create", + run_params( + &ws, + &channel, + json!({"remote_root": folder.path(), "remote_execution": true, "remote_access": "full"}), + ), + ); + let credential = created["credential"].as_str().unwrap().to_owned(); + let mut ws = ws.reopen(); + let scope = ok(worker(&mut ws, &credential, "run.remote_scope")); + assert_eq!(scope["access"], "full"); +} + +/// `remote_access_v1` is advertised after `presence_v1`, where the design appends capabilities. +#[test] +fn the_broker_advertises_remote_access() { + let capabilities = Broker::capabilities(); + let presence = capabilities + .iter() + .position(|c| *c == "presence_v1") + .unwrap(); + assert!(capabilities[presence + 1..].contains(&"remote_access_v1")); +} From 137d2854018f4b1af527768cbd204297b696dcb2 Mon Sep 17 00:00:00 2001 From: Wanjun Gu Date: Fri, 16 Oct 2026 19:55:08 -0700 Subject: [PATCH 05/32] crew: full access in the member's bridge, bridge.hello/probe and doctor (OW-6, OW-7) The bridge answers bridge.hello and bridge.probe itself and never forwards a bridge.* frame. A scope with access "full" goes to the new remote_full module: remote.shell and remote.execute run as the member through the login shell under a __crew-run supervisor (owner-keyed job records in ~/.local/state/biorouter-crew/jobs, detach, timeout, cancel by process group), file operations take absolute, ~/ or work-folder paths, and remote.fetch_blob and remote.share_file move attachments on a second broker connection in the background. Folder mode is unchanged except that a kernel below Landlock ABI 3 is refused up front. New verbs: probe, doctor and --version --output-format json. An answer too large for a frame is now refused under its ID instead of ending the bridge. --- crates/biorouter-crew/src/broker.rs | 165 +- crates/biorouter-crew/src/lib.rs | 4 + crates/biorouter-crew/src/main.rs | 47 +- crates/biorouter-crew/src/probe.rs | 1048 +++++++++ crates/biorouter-crew/src/remote.rs | 102 +- crates/biorouter-crew/src/remote_full.rs | 1920 +++++++++++++++++ .../tests/full_access_bridge.rs | 299 +++ .../tests/full_access_contract.rs | 709 ++++++ 8 files changed, 4280 insertions(+), 14 deletions(-) create mode 100644 crates/biorouter-crew/src/probe.rs create mode 100644 crates/biorouter-crew/src/remote_full.rs create mode 100644 crates/biorouter-crew/tests/full_access_bridge.rs create mode 100644 crates/biorouter-crew/tests/full_access_contract.rs diff --git a/crates/biorouter-crew/src/broker.rs b/crates/biorouter-crew/src/broker.rs index 8be2f5fde..526ed8bb2 100644 --- a/crates/biorouter-crew/src/broker.rs +++ b/crates/biorouter-crew/src/broker.rs @@ -5559,13 +5559,46 @@ pub fn bridge(socket: &Path, owner: u32, workspace: &str) -> Result<()> { ); let root = runtime_root_of(socket)?; let (stream, reader, _) = open_workspace(root, socket, owner, workspace)?; + // A full-access transfer opens its own connection the same way, so the relay never waits + // on it. + let reconnect: crate::remote_full::Connect = { + let (root, socket, workspace) = ( + root.to_path_buf(), + socket.to_path_buf(), + workspace.to_owned(), + ); + Arc::new(move || { + open_workspace(&root, &socket, owner, &workspace) + .map(|(stream, reader, _)| (stream, reader)) + }) + }; // Standard input is read through a buffer the relay can see into: it waits on the input // descriptor only when that buffer is empty. use std::os::fd::AsFd; let mut input = BufReader::new(File::from(std::io::stdin().as_fd().try_clone_to_owned()?)); let stdout = std::io::stdout(); let mut output = stdout.lock(); - relay(&mut input, &mut output, stream, reader) + relay(&mut input, &mut output, stream, reader, &reconnect) +} +/// Write `response` to the daemon as one frame. An answer too large for a frame is replaced by +/// a refusal under the same ID rather than ending the bridge. +fn write_response(output: &mut impl Write, response: Response) -> Result<()> { + let mut bytes = serde_json::to_vec(&response)?; + if bytes.len() >= MAX_FRAME { + bytes = serde_json::to_vec(&Response { + id: response.id, + result: None, + error: Some(ProtocolError { + code: "response_too_large".into(), + message: "response_too_large: The answer was larger than one frame. Ask for less." + .into(), + }), + })?; + } + bytes.push(b'\n'); + output.write_all(&bytes)?; + output.flush()?; + Ok(()) } /// What a bridge answers a request it could not hand to the broker: nothing of it reached the /// workspace, so nothing changed and it is safe to send again once reconnected. The member's @@ -5583,11 +5616,16 @@ const BROKER_GONE: &str = /// (R-5). Waiting on the daemon alone, it noticed a broker that died only when the next /// request failed, and the member read "Connected" for up to two and a half minutes. A request /// that could not be written to the broker is answered [`NOT_DELIVERED`] before it exits. +/// +/// `bridge.*` requests are the bridge's own (design OW-6, OW-8) and are answered here, never +/// forwarded, an unknown one with a local refusal. `reconnect` opens the second broker +/// connection a full-access transfer runs on. fn relay( input: &mut BufReader, output: &mut impl Write, mut stream: UnixStream, mut reader: BufReader, + reconnect: &crate::remote_full::Connect, ) -> Result<()> { loop { if input.buffer().is_empty() { @@ -5606,7 +5644,21 @@ fn relay( return Ok(()); }; let request: Request = serde_json::from_slice(&frame)?; - if request.method.starts_with("remote.") { + if request.method.starts_with("bridge.") { + let response = match crate::probe::bridge_method(&request.method) { + Ok(value) => Response { + id: request.id, + result: Some(value), + error: None, + }, + Err(error) => Response { + id: request.id, + result: None, + error: Some(protocol_error(&error)), + }, + }; + write_response(output, response)?; + } else if request.method.starts_with("remote.") { let result = (|| -> Result { ensure!( request.auth.is_none() && request.credential.is_some(), @@ -5633,7 +5685,12 @@ fn relay( let scope = scope_response .result .ok_or_else(|| anyhow!("invalid_response"))?; - let result = crate::remote::handle(&request.method, &request.params, &scope)?; + let link = crate::remote_full::BrokerLink { + connect: reconnect.clone(), + credential: request.credential.clone().unwrap_or_default(), + }; + let result = + crate::remote::handle(&request.method, &request.params, &scope, &link)?; stream.write_all(&scope_bytes)?; stream.flush()?; let recheck: Response = serde_json::from_slice( @@ -5660,11 +5717,7 @@ fn relay( }), }, }; - let mut bytes = serde_json::to_vec(&response)?; - ensure!(bytes.len() < MAX_FRAME, "response_too_large"); - bytes.push(b'\n'); - output.write_all(&bytes)?; - output.flush()?; + write_response(output, response)?; } else { if stream .write_all(&frame) @@ -6666,12 +6719,106 @@ mod bridge_tests { std::thread::spawn(move || { let reader = BufReader::new(bridge_side.try_clone().unwrap()); let mut output = Vec::new(); - let result = relay(&mut input, &mut output, bridge_side, reader); + let reconnect: crate::remote_full::Connect = + Arc::new(|| bail!("no second broker connection in this test")); + let result = relay(&mut input, &mut output, bridge_side, reader, &reconnect); let _ = sender.send((result, output)); }); receiver } + /// A broker that records every frame it receives and answers each `{ok: true}`. + fn recording_broker(broker_side: UnixStream) -> Arc>> { + let seen = Arc::new(Mutex::new(Vec::new())); + let record = seen.clone(); + std::thread::spawn(move || { + let mut reader = BufReader::new(broker_side.try_clone().unwrap()); + let mut writer = broker_side; + while let Ok(Some(request)) = read_frame(&mut reader) { + let request: Value = serde_json::from_slice(&request).unwrap(); + let answer = json!({"id": request["id"], "result": {"ok": true}}); + record.lock().unwrap().push(request); + writer.write_all(format!("{answer}\n").as_bytes()).unwrap(); + } + }); + seen + } + + fn answers(output: &[u8]) -> Vec { + output + .split(|b| *b == b'\n') + .filter(|line| !line.is_empty()) + .map(|line| serde_json::from_slice::(line).unwrap()) + .collect() + } + + /// `bridge.hello` and `bridge.probe` are answered by the bridge itself, and any other + /// `bridge.*` method is refused there: none of them ever reaches the broker, while an + /// ordinary request after them still does. + #[test] + fn bridge_methods_are_answered_locally_and_never_forwarded() { + let (input, mut daemon) = stdin_pipe(); + let (bridge_side, broker_side) = UnixStream::pair().unwrap(); + let seen = recording_broker(broker_side); + let finished = spawn_relay(input, bridge_side); + for (id, method) in [ + ("hello", "bridge.hello"), + ("probe", "bridge.probe"), + ("other", "bridge.delete_everything"), + ] { + daemon + .write_all( + format!("{{\"version\":1,\"id\":\"{id}\",\"method\":\"{method}\",\"params\":{{}}}}\n") + .as_bytes(), + ) + .unwrap(); + } + daemon.write_all(&frame("post")).unwrap(); + drop(daemon); + let (result, output) = finished.recv_timeout(Duration::from_secs(30)).unwrap(); + result.unwrap(); + let answers = answers(&output); + assert_eq!(answers.len(), 4, "{answers:?}"); + assert_eq!(answers[0]["id"], "hello"); + assert_eq!( + answers[0]["result"]["capabilities"], + json!(["remote_full_v1", "remote_transfer_v1", "bridge_probe_v1"]) + ); + assert_eq!( + answers[0]["result"]["bridge_version"], + env!("CARGO_PKG_VERSION") + ); + assert_eq!(answers[1]["result"]["schema"], 1); + assert_eq!(answers[2]["error"]["code"], "unsupported"); + assert_eq!(answers[3]["result"]["ok"], true); + let seen = seen.lock().unwrap(); + assert_eq!( + seen.len(), + 1, + "only the ordinary request reached the broker: {seen:?}" + ); + assert_eq!(seen[0]["method"], "message.post"); + } + + /// A `remote.*` frame without a worker credential is refused by the bridge, and the broker + /// is never asked for its scope. + #[test] + fn a_remote_request_without_a_credential_is_refused_locally() { + let (input, mut daemon) = stdin_pipe(); + let (bridge_side, broker_side) = UnixStream::pair().unwrap(); + let seen = recording_broker(broker_side); + let finished = spawn_relay(input, bridge_side); + daemon + .write_all(b"{\"version\":1,\"id\":\"shell\",\"method\":\"remote.shell\",\"params\":{\"command\":\"id\"}}\n") + .unwrap(); + drop(daemon); + let (result, output) = finished.recv_timeout(Duration::from_secs(5)).unwrap(); + result.unwrap(); + let answers = answers(&output); + assert_eq!(answers[0]["error"]["code"], "remote_operation_denied"); + assert!(seen.lock().unwrap().is_empty()); + } + fn frame(id: &str) -> Vec { format!("{{\"version\":1,\"id\":\"{id}\",\"method\":\"message.post\",\"params\":{{}}}}\n") .into_bytes() diff --git a/crates/biorouter-crew/src/lib.rs b/crates/biorouter-crew/src/lib.rs index ca66e42c1..202332e8e 100644 --- a/crates/biorouter-crew/src/lib.rs +++ b/crates/biorouter-crew/src/lib.rs @@ -401,8 +401,12 @@ pub use broker::{bridge, lifecycle, serve, Account, Broker, Connection, Director #[cfg(all(unix, feature = "test-seams"))] pub use broker::{JournalCall, Quotas}; +#[cfg(unix)] +pub mod probe; #[cfg(unix)] pub mod remote; +#[cfg(unix)] +pub mod remote_full; #[cfg(test)] mod tests { diff --git a/crates/biorouter-crew/src/main.rs b/crates/biorouter-crew/src/main.rs index 7418fcf84..a6b4869e7 100644 --- a/crates/biorouter-crew/src/main.rs +++ b/crates/biorouter-crew/src/main.rs @@ -1,7 +1,7 @@ use anyhow::{bail, Result}; #[cfg(unix)] -const USAGE: &str = "usage: biorouter-crew serve|start|status|stop (--state-dir PATH | --name NAME) [--name NAME] [--bootstrap-key HEX]; bridge --stdio --socket PATH --owner-uid UID --workspace-id UUID"; +const USAGE: &str = "usage: biorouter-crew serve|start|status|stop (--state-dir PATH | --name NAME) [--name NAME] [--bootstrap-key HEX]; bridge --stdio --socket PATH --owner-uid UID --workspace-id UUID; probe|doctor [--output-format json]"; /// Where a named workspace keeps its state when `--state-dir` is not given: /// `$HOME/.local/share/biorouter-crew/`. The name is validated first, so it is a plain @@ -22,16 +22,43 @@ fn default_state_dir(name: &str) -> Result { .join(name)) } +/// Whether `--output-format json` was asked for; any other format is refused. +#[cfg(unix)] +fn json_output(args: &[String]) -> Result { + match args.windows(2).find(|pair| pair[0] == "--output-format") { + None if args.iter().any(|arg| arg == "--output-format") => { + bail!("--output-format requires a format (json)") + } + None => Ok(false), + Some(pair) if pair[1] == "json" => Ok(true), + Some(pair) => bail!("unsupported output format {}; use json", pair[1]), + } +} + #[cfg(unix)] fn main() -> Result<()> { let args: Vec = std::env::args().skip(1).collect(); let command = args.first().map(String::as_str).unwrap_or("help"); if matches!(command, "--version" | "-V") { - println!("biorouter-crew {}", env!("CARGO_PKG_VERSION")); + if json_output(&args)? { + // A cheap compatibility check for setup: the broker's capabilities (what `hello` + // advertises) and the bridge's own. + println!( + "{}", + serde_json::json!({ + "version": env!("CARGO_PKG_VERSION"), + "protocol": biorouter_crew::PROTOCOL_VERSION, + "capabilities": biorouter_crew::Broker::capabilities(), + "bridge_capabilities": biorouter_crew::probe::BRIDGE_CAPABILITIES, + }) + ); + } else { + println!("biorouter-crew {}", env!("CARGO_PKG_VERSION")); + } return Ok(()); } if matches!(command, "help" | "--help" | "-h") { - println!("Usage: biorouter-crew start --name NAME --bootstrap-key HEX [--state-dir PATH]\n biorouter-crew serve|start|status|stop --state-dir PATH [--name NAME] [--bootstrap-key HEX]\n biorouter-crew bridge --stdio --socket PATH --owner-uid UID --workspace-id UUID\n biorouter-crew --version\n--name names the workspace (lowercase letters, numbers and hyphens); without --state-dir its state lives in ~/.local/share/biorouter-crew/NAME. start prints the workspace's invitation line (brcrew1:...) to paste into Crew.\nBroker and bridge operations require Linux."); + println!("Usage: biorouter-crew start --name NAME --bootstrap-key HEX [--state-dir PATH]\n biorouter-crew serve|start|status|stop --state-dir PATH [--name NAME] [--bootstrap-key HEX]\n biorouter-crew bridge --stdio --socket PATH --owner-uid UID --workspace-id UUID\n biorouter-crew probe|doctor [--output-format json]\n biorouter-crew --version [--output-format json]\n--name names the workspace (lowercase letters, numbers and hyphens); without --state-dir its state lives in ~/.local/share/biorouter-crew/NAME. start prints the workspace's invitation line (brcrew1:...) to paste into Crew. probe prints fixed facts about this server and account as JSON; doctor adds what blocks hosting here and what limits agents.\nBroker and bridge operations require Linux."); return Ok(()); } @@ -42,6 +69,20 @@ fn main() -> Result<()> { }; if command == "__crew-exec" { biorouter_crew::remote::exec_helper()?; + } else if command == "__crew-run" { + let directory = args + .get(1) + .ok_or_else(|| anyhow::anyhow!("__crew-run requires a job directory"))?; + biorouter_crew::remote_full::supervise(std::path::Path::new(directory))?; + } else if matches!(command, "probe" | "doctor") { + // JSON is the only format, so the flag is optional; anything else is refused. + json_output(&args)?; + let facts = if command == "probe" { + biorouter_crew::probe::probe() + } else { + biorouter_crew::probe::doctor() + }; + println!("{}", serde_json::to_string_pretty(&facts)?); } else if command == "bridge" { biorouter_crew::bridge( std::path::Path::new( diff --git a/crates/biorouter-crew/src/probe.rs b/crates/biorouter-crew/src/probe.rs new file mode 100644 index 000000000..0ac321279 --- /dev/null +++ b/crates/biorouter-crew/src/probe.rs @@ -0,0 +1,1048 @@ +//! Fixed, bounded facts about the server and the account a bridge runs as (design OW-8, OW-9): +//! the answers to `bridge.hello` and `bridge.probe`, and `biorouter-crew probe`, `doctor` and +//! `--version --output-format json`. +//! +//! Every check is a fixed file read, system call or argv with no model input, bounded to +//! [`PROBE_TIMEOUT`] and [`PROBE_OUTPUT`], and none reaches the network. It never reads +//! environment variables (commands are found by a login shell started with a fixed, minimal +//! environment), process lists, other people's homes, SSH configuration or shell history, so +//! what it reports may be shown to every member of a workspace as its first context document. +use anyhow::{anyhow, ensure, Result}; +use serde_json::{json, Value}; +use std::{ + io::Read, + path::{Path, PathBuf}, + process::{Command, Stdio}, + time::{Duration, Instant}, +}; + +/// What the member's own `biorouter-crew bridge` can do beyond relaying, as `bridge.hello` +/// reports it. An older bridge forwards `bridge.hello` to the broker, which refuses it: that +/// refusal is how a daemon tells an old bridge. +pub const BRIDGE_CAPABILITIES: [&str; 3] = + ["remote_full_v1", "remote_transfer_v1", "bridge_probe_v1"]; +/// The longest any one probe may run. +pub const PROBE_TIMEOUT: Duration = Duration::from_secs(5); +/// The most output kept from any one probe command. +pub const PROBE_OUTPUT: usize = 65_536; +/// At most this many filesystems are reported. +const MAX_STORAGE: usize = 12; +/// A filesystem that is not the home's, `/tmp`'s or a scratch area's is reported only from +/// this size, so a server's small system volumes do not crowd out its data storage. +const LARGE_FILESYSTEM_GIB: u64 = 100; +/// Filesystem types that hold data, as `/proc/self/mounts` names them. +const DATA_FILESYSTEMS: [&str; 18] = [ + "ext2", + "ext3", + "ext4", + "xfs", + "btrfs", + "zfs", + "nfs", + "nfs4", + "lustre", + "gpfs", + "beegfs", + "cifs", + "smb3", + "ceph", + "cephfs", + "panfs", + "wekafs", + "fuse.glusterfs", +]; +/// Commands the login-shell probe looks for, by name. Fixed: nothing a model or a member writes +/// reaches it. +const PROBED_COMMANDS: [&str; 20] = [ + "sbatch", + "squeue", + "sinfo", + "srun", + "qsub", + "qstat", + "qconf", + "pbsnodes", + "bsub", + "condor_submit", + "docker", + "podman", + "apptainer", + "singularity", + "nvidia-smi", + "conda", + "mamba", + "python3", + "R", + "julia", +]; +/// Where a conda installation is looked for, relative to the account's home or absolute. +const CONDA_PLACES: [&str; 10] = [ + "~/anaconda3", + "~/miniconda3", + "~/miniforge3", + "~/mambaforge", + "~/micromamba", + "/opt/conda", + "/opt/anaconda3", + "/opt/miniconda3", + "/usr/local/anaconda3", + "/usr/local/miniconda3", +]; +/// Files whose presence means environment modules (Lmod or Tcl modules) are installed. +const MODULE_MARKERS: [&str; 6] = [ + "/etc/profile.d/modules.sh", + "/etc/profile.d/lmod.sh", + "/etc/profile.d/z00_lmod.sh", + "/usr/share/lmod", + "/usr/share/Modules", + "/opt/apps/lmod", +]; + +/// The account this process runs as, from the account database (never from `HOME`/`SHELL`). +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ThisAccount { + pub uid: u32, + pub user: String, + pub home: PathBuf, + /// The login shell, or `None` when the database names none. + pub shell: Option, +} + +/// `getpwuid_r(geteuid())`, with the buffer grown on `ERANGE`. +pub fn this_account() -> Result { + let uid = unsafe { libc::geteuid() }; + let mut size = 16 * 1024; + loop { + let mut entry = std::mem::MaybeUninit::::uninit(); + let mut found = std::ptr::null_mut(); + let mut buffer = vec![0u8; size]; + let status = unsafe { + libc::getpwuid_r( + uid, + entry.as_mut_ptr(), + buffer.as_mut_ptr().cast(), + buffer.len(), + &mut found, + ) + }; + if status == libc::ERANGE && size < 1024 * 1024 { + size *= 4; + continue; + } + ensure!( + status == 0 && !found.is_null(), + "identity_unavailable: This Unix account cannot be resolved." + ); + // SAFETY: on success `found` points at `entry`, whose strings live in `buffer`. + let entry = unsafe { &*found }; + let field = |pointer: *const libc::c_char| { + (!pointer.is_null()).then(|| { + unsafe { std::ffi::CStr::from_ptr(pointer) } + .to_string_lossy() + .trim() + .to_owned() + }) + }; + let home = field(entry.pw_dir).filter(|home| home.starts_with('/')); + return Ok(ThisAccount { + uid, + user: field(entry.pw_name).unwrap_or_default(), + home: PathBuf::from( + home.ok_or_else(|| anyhow!("identity_unavailable: This account has no home."))?, + ), + shell: field(entry.pw_shell) + .filter(|shell| shell.starts_with('/')) + .map(PathBuf::from), + }); + } +} + +/// The highest Landlock ABI the kernel supports, or 0 when it has none (or this is not Linux). +pub fn landlock_abi() -> u32 { + #[cfg(target_os = "linux")] + { + const LANDLOCK_CREATE_RULESET_VERSION: libc::c_uint = 1; + let abi = unsafe { + libc::syscall( + libc::SYS_landlock_create_ruleset, + std::ptr::null::(), + 0usize, + LANDLOCK_CREATE_RULESET_VERSION, + ) + }; + u32::try_from(abi).unwrap_or(0) + } + #[cfg(not(target_os = "linux"))] + { + 0 + } +} + +/// Whether the kernel offers seccomp filters, which confined commands also need. +pub fn seccomp_available() -> bool { + #[cfg(target_os = "linux")] + { + unsafe { libc::prctl(libc::PR_GET_SECCOMP) >= 0 } + } + #[cfg(not(target_os = "linux"))] + { + false + } +} + +/// Whether folder-mode (confined) commands can run here: Landlock ABI 3 and seccomp. +pub fn confined_execution_supported() -> bool { + landlock_abi() >= 3 && seccomp_available() +} + +/// The answer to a bridge-local method (`bridge.*`), which the bridge gives itself and never +/// forwards. +pub fn bridge_method(method: &str) -> Result { + match method { + "bridge.hello" => bridge_hello(), + "bridge.probe" => Ok(probe()), + _ => Err(anyhow!( + "unsupported: This Crew bridge has no method {}.", + method.chars().take(64).collect::() + )), + } +} + +/// `bridge.hello`: this bridge's version and capabilities, whether this kernel can confine +/// commands, and the account it runs as. +pub fn bridge_hello() -> Result { + let account = this_account()?; + Ok(json!({ + "bridge_version": env!("CARGO_PKG_VERSION"), + "capabilities": BRIDGE_CAPABILITIES, + "landlock_abi": landlock_abi(), + "confined_execution_supported": confined_execution_supported(), + "user": account.user, + "uid": account.uid, + "home": account.home, + "shell": account.shell, + })) +} + +/// Run `program` with `args` and a fixed `env` (nothing inherited), stdin `/dev/null`, in its +/// own process group, for at most [`PROBE_TIMEOUT`]; its standard output and error, each cut at +/// [`PROBE_OUTPUT`], or `None` when it could not start or ran out of time. +fn run_bounded(program: &Path, args: &[&str], env: &[(&str, &str)]) -> Option<(String, String)> { + use std::os::unix::process::CommandExt; + let mut command = Command::new(program); + command + .args(args) + .env_clear() + .envs(env.iter().copied()) + .current_dir("/") + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + unsafe { + command.pre_exec(|| { + if libc::setsid() < 0 { + return Err(std::io::Error::last_os_error()); + } + Ok(()) + }); + } + let mut child = command.spawn().ok()?; + let read = |stream: Option>| { + std::thread::spawn(move || { + let mut bytes = Vec::new(); + if let Some(stream) = stream { + let _ = stream.take(PROBE_OUTPUT as u64).read_to_end(&mut bytes); + } + String::from_utf8_lossy(&bytes).into_owned() + }) + }; + let stdout = read( + child + .stdout + .take() + .map(|s| Box::new(s) as Box), + ); + let stderr = read( + child + .stderr + .take() + .map(|s| Box::new(s) as Box), + ); + let deadline = Instant::now() + PROBE_TIMEOUT; + loop { + match child.try_wait() { + Ok(Some(_)) => break, + Ok(None) if Instant::now() < deadline => { + std::thread::sleep(Duration::from_millis(20)); + } + _ => { + unsafe { libc::kill(-(child.id() as i32), libc::SIGKILL) }; + let _ = child.kill(); + let _ = child.wait(); + return None; + } + } + } + // The command has exited; a grandchild still holding its output would keep a reader + // waiting, so the group goes too. + unsafe { libc::kill(-(child.id() as i32), libc::SIGKILL) }; + Some((stdout.join().ok()?, stderr.join().ok()?)) +} + +/// A shell the probe may give a POSIX script with `-lc`: the account's login shell when it is +/// of the Bourne family, else `/bin/sh`. +fn posix_login_shell(account: &ThisAccount) -> PathBuf { + let posix = account.shell.as_ref().filter(|shell| { + let name = shell.file_name().and_then(|n| n.to_str()).unwrap_or(""); + matches!(name, "sh" | "bash" | "zsh" | "dash" | "ksh" | "mksh") && shell.exists() + }); + posix.cloned().unwrap_or_else(|| PathBuf::from("/bin/sh")) +} + +/// The commands a fresh login of this account finds, by name, and whether it has the `module` +/// command. The login shell starts with only `HOME`, `USER`, `LOGNAME`, `SHELL`, a minimal +/// `PATH` and `LANG`, so what it reports is what the account's own startup files set up, and +/// nothing of this process's environment is read. +fn login_commands(account: &ThisAccount) -> (Vec<(String, String)>, bool) { + let mut script = String::from("for c in"); + for name in PROBED_COMMANDS { + script.push(' '); + script.push_str(name); + } + script.push_str( + "; do p=$(command -v \"$c\" 2>/dev/null) && printf '@crew-probe %s %s\\n' \"$c\" \"$p\"; done; \ + command -v module >/dev/null 2>&1 && printf '@crew-probe-modules yes\\n'; exit 0", + ); + let home = account.home.to_string_lossy().into_owned(); + let shell = posix_login_shell(account); + let shell_text = shell.to_string_lossy().into_owned(); + let env = [ + ("HOME", home.as_str()), + ("USER", account.user.as_str()), + ("LOGNAME", account.user.as_str()), + ("SHELL", shell_text.as_str()), + ("PATH", "/usr/local/bin:/usr/bin:/bin"), + ("LANG", "C"), + ]; + let Some((stdout, _)) = run_bounded(&shell, &["-lc", &script], &env) else { + return (Vec::new(), false); + }; + let mut found = Vec::new(); + let mut modules = false; + for line in stdout.lines() { + if line == "@crew-probe-modules yes" { + modules = true; + } else if let Some(rest) = line.strip_prefix("@crew-probe ") { + if let Some((name, path)) = rest.split_once(' ') { + // Only the names asked for, at absolute paths (a shell function or alias, such + // as `conda` after `conda init`, is a name, not a program to run). + if PROBED_COMMANDS.contains(&name) && path.starts_with('/') { + found.push((name.to_owned(), path.to_owned())); + } + } + } + } + (found, modules) +} + +/// The first version number (`3.10.12`) in a command's `--version` text. +fn version_in(text: &str) -> Option { + text.split(|c: char| c.is_whitespace() || c == ',' || c == '(' || c == ')') + .find(|word| { + word.chars().next().is_some_and(|c| c.is_ascii_digit()) + && word.contains('.') + && word + .chars() + .all(|c| c.is_ascii_alphanumeric() || c == '.' || c == '-' || c == '+') + }) + .map(|word| word.chars().take(32).collect()) +} + +/// `path` as the probe reports it: `~/…` inside the account's home, else unchanged, and `None` +/// when it is inside another home. +fn shown_path(path: &Path, home: &Path) -> Option { + if let Ok(inside) = path.strip_prefix(home) { + return Some(if inside.as_os_str().is_empty() { + "~".to_owned() + } else { + format!("~/{}", inside.display()) + }); + } + (!in_another_home(path, home)).then(|| path.display().to_string()) +} + +/// Whether `path` is inside a home directory other than `home`: a sibling of `home` (or below +/// one), or anything below `/home` that is not `home` itself or one of its ancestors. +fn in_another_home(path: &Path, home: &Path) -> bool { + if home.starts_with(path) { + return false; + } + let sibling = home + .parent() + .filter(|parent| parent.components().count() > 1) + .is_some_and(|parent| path.starts_with(parent) && !path.starts_with(home)); + sibling || (path.starts_with("/home") && path != Path::new("/home") && !path.starts_with(home)) +} + +/// One mount from `/proc/self/mounts`: where, and of what type. +#[derive(Clone, Debug, PartialEq, Eq)] +struct Mount { + path: PathBuf, + fs: String, +} + +/// `/proc/self/mounts` text as mounts, with the octal escapes (`\040` for a space) undone. +fn parse_mounts(text: &str) -> Vec { + let unescape = |field: &str| { + let bytes = field.as_bytes(); + let mut out = Vec::with_capacity(bytes.len()); + let mut index = 0; + while index < bytes.len() { + let digits = bytes.get(index + 1..index + 4).unwrap_or_default(); + if bytes[index] == b'\\' + && digits.len() == 3 + && digits.iter().all(|b| (b'0'..=b'7').contains(b)) + { + let value = digits + .iter() + .fold(0u32, |sum, b| sum * 8 + u32::from(b - b'0')); + if let Ok(value) = u8::try_from(value) { + out.push(value); + index += 4; + continue; + } + } + out.push(bytes[index]); + index += 1; + } + String::from_utf8_lossy(&out).into_owned() + }; + text.lines() + .filter_map(|line| { + let mut fields = line.split(' '); + let _source = fields.next()?; + let path = unescape(fields.next()?); + let fs = fields.next()?.to_owned(); + Some(Mount { + path: PathBuf::from(path), + fs, + }) + }) + .collect() +} + +/// The mount `path` is on: the one with the longest mount point that contains it. +fn mount_of<'m>(mounts: &'m [Mount], path: &Path) -> Option<&'m Mount> { + mounts + .iter() + .filter(|mount| path.starts_with(&mount.path)) + .max_by_key(|mount| mount.path.components().count()) +} + +/// Size and free space of the filesystem at `path` in GiB, from `statvfs` on its own thread so +/// a hung network mount costs [`PROBE_TIMEOUT`] and no more. +fn space_gib(path: &Path) -> Option<(u64, u64)> { + let path = std::ffi::CString::new(path.as_os_str().as_encoded_bytes()).ok()?; + let (sender, receiver) = std::sync::mpsc::channel(); + std::thread::spawn(move || { + let mut info = std::mem::MaybeUninit::::uninit(); + let answer = (unsafe { libc::statvfs(path.as_ptr(), info.as_mut_ptr()) } == 0).then(|| { + let info = unsafe { info.assume_init() }; + let unit = info.f_frsize; + let gib = |blocks: u64| blocks.saturating_mul(unit) / (1 << 30); + // Block counts are 64-bit on Linux and 32-bit on macOS, so the conversion is a + // no-op on one of them. + #[allow(clippy::useless_conversion)] + let (blocks, available) = (u64::from(info.f_blocks), u64::from(info.f_bavail)); + (gib(blocks), gib(available)) + }); + let _ = sender.send(answer); + }); + receiver.recv_timeout(PROBE_TIMEOUT).ok().flatten() +} + +/// The filesystems worth naming: the home's, `/tmp`'s, any `/scratch*`, then other data +/// filesystems of at least [`LARGE_FILESYSTEM_GIB`], never one inside another person's home, +/// one entry per distinct filesystem, at most [`MAX_STORAGE`]. +fn storage(mounts: &[Mount], home: &Path) -> Vec { + let mut wanted: Vec<(PathBuf, bool)> = Vec::new(); + for path in [home.to_path_buf(), PathBuf::from("/tmp")] { + if let Some(mount) = mount_of(mounts, &path) { + wanted.push((mount.path.clone(), true)); + } + } + if let Ok(entries) = std::fs::read_dir("/") { + let mut scratch: Vec = entries + .filter_map(|entry| entry.ok()) + .map(|entry| entry.path()) + .filter(|path| { + path.file_name() + .and_then(|name| name.to_str()) + .is_some_and(|name| name.starts_with("scratch")) + && path.is_dir() + }) + .collect(); + scratch.sort(); + wanted.extend(scratch.into_iter().take(4).map(|path| (path, true))); + } + for mount in mounts { + let system = [ + "/proc", + "/sys", + "/dev", + "/run", + "/boot", + "/snap", + "/var/lib/docker", + ] + .iter() + .any(|prefix| mount.path.starts_with(prefix)); + if DATA_FILESYSTEMS.contains(&mount.fs.as_str()) && !system { + wanted.push((mount.path.clone(), false)); + } + } + let mut seen = std::collections::BTreeSet::new(); + let mut reported = Vec::new(); + for (path, always) in wanted.into_iter().take(48) { + if reported.len() >= MAX_STORAGE || in_another_home(&path, home) { + continue; + } + let Some((size_gib, avail_gib)) = space_gib(&path) else { + continue; + }; + if !always && size_gib < LARGE_FILESYSTEM_GIB { + continue; + } + let fs = mount_of(mounts, &path) + .map(|mount| mount.fs.clone()) + .unwrap_or_default(); + // One pool reached through several mounts (ZFS datasets, bind mounts) is one entry. + if !seen.insert((fs.clone(), size_gib, avail_gib)) { + continue; + } + let Some(shown) = shown_path(&path, home) else { + continue; + }; + reported + .push(json!({"path": shown, "fs": fs, "size_gib": size_gib, "avail_gib": avail_gib})); + } + reported +} + +fn os_release() -> Option { + let text = std::fs::read_to_string("/etc/os-release").ok()?; + let value = |key: &str| { + text.lines() + .find_map(|line| line.strip_prefix(key)?.strip_prefix('=')) + .map(|value| value.trim().trim_matches('"').to_owned()) + }; + value("PRETTY_NAME") + .or_else(|| value("NAME")) + .map(|name| name.chars().take(128).collect()) +} + +fn uname() -> (Option, Option) { + let mut name = std::mem::MaybeUninit::::uninit(); + if unsafe { libc::uname(name.as_mut_ptr()) } != 0 { + return (None, None); + } + let name = unsafe { name.assume_init() }; + let field = |chars: &[libc::c_char]| { + unsafe { std::ffi::CStr::from_ptr(chars.as_ptr()) } + .to_string_lossy() + .into_owned() + }; + (Some(field(&name.release)), Some(field(&name.machine))) +} + +fn hostname() -> Option { + let mut buffer = [0u8; 256]; + (unsafe { libc::gethostname(buffer.as_mut_ptr().cast(), buffer.len()) } == 0).then(|| { + let end = buffer.iter().position(|b| *b == 0).unwrap_or(buffer.len()); + String::from_utf8_lossy(&buffer[..end]).into_owned() + }) +} + +fn glibc_version() -> Option { + #[cfg(all(target_os = "linux", target_env = "gnu"))] + { + let version = unsafe { libc::gnu_get_libc_version() }; + (!version.is_null()).then(|| { + unsafe { std::ffi::CStr::from_ptr(version) } + .to_string_lossy() + .into_owned() + }) + } + #[cfg(not(all(target_os = "linux", target_env = "gnu")))] + { + None + } +} + +fn memory_gib() -> Option { + let text = std::fs::read_to_string("/proc/meminfo").ok()?; + let kib: u64 = text + .lines() + .find_map(|line| line.strip_prefix("MemTotal:"))? + .split_whitespace() + .next()? + .parse() + .ok()?; + Some((kib + (1 << 19)) / (1 << 20)) +} + +/// Whether this process is in the group named `name` (the groups its login gave it). +fn in_group(name: &str) -> bool { + let Ok(name) = std::ffi::CString::new(name) else { + return false; + }; + let mut entry = std::mem::MaybeUninit::::uninit(); + let mut found = std::ptr::null_mut(); + let mut buffer = vec![0u8; 64 * 1024]; + let status = unsafe { + libc::getgrnam_r( + name.as_ptr(), + entry.as_mut_ptr(), + buffer.as_mut_ptr().cast(), + buffer.len(), + &mut found, + ) + }; + if status != 0 || found.is_null() { + return false; + } + let gid = unsafe { (*found).gr_gid }; + let count = unsafe { libc::getgroups(0, std::ptr::null_mut()) }; + let mut groups = vec![0 as libc::gid_t; count.max(0) as usize]; + let count = unsafe { libc::getgroups(groups.len() as libc::c_int, groups.as_mut_ptr()) }; + groups.truncate(count.max(0) as usize); + let primary = unsafe { libc::getegid() }; + primary == gid || groups.contains(&gid) +} + +/// GPUs `nvidia-smi` reports: `{name, memory_mib}` each. +fn gpus(nvidia_smi: Option<&str>) -> Vec { + let Some(program) = nvidia_smi else { + return Vec::new(); + }; + let Some((stdout, _)) = run_bounded( + Path::new(program), + &[ + "--query-gpu=name,memory.total", + "--format=csv,noheader,nounits", + ], + &[("PATH", "/usr/bin:/bin"), ("LANG", "C")], + ) else { + return Vec::new(); + }; + stdout + .lines() + .filter_map(|line| { + let (name, memory) = line.rsplit_once(',')?; + Some(json!({"name": name.trim().chars().take(128).collect::(), "memory_mib": memory.trim().parse::().ok()})) + }) + .take(32) + .collect() +} + +/// The scheduler the found commands belong to. +fn scheduler_kind(found: &[&str]) -> Option<&'static str> { + let has = |name: &str| found.contains(&name); + if has("sbatch") { + Some("slurm") + } else if has("bsub") { + Some("lsf") + } else if has("condor_submit") { + Some("htcondor") + } else if has("qconf") { + Some("sge") + } else if has("qsub") { + Some("pbs") + } else { + None + } +} + +/// The server facts `bridge.probe` and `biorouter-crew probe` report (design, "Server probe +/// and context draft"). A fact that cannot be learned is `null` or empty, never an error. +pub fn probe() -> Value { + let account = this_account().ok(); + let home = account.as_ref().map(|a| a.home.clone()).unwrap_or_default(); + let (kernel, arch) = uname(); + let (found, login_modules) = account.as_ref().map(login_commands).unwrap_or_default(); + let path_of = |name: &str| { + found + .iter() + .find(|(found, _)| found == name) + .map(|(_, path)| path.as_str()) + }; + let names: Vec<&str> = found.iter().map(|(name, _)| name.as_str()).collect(); + // Version commands run side by side, so the probe as a whole stays near two timeouts. + let versions: Vec<(&str, std::thread::JoinHandle>)> = [ + ("python3", "--version"), + ("R", "--version"), + ("julia", "--version"), + ] + .into_iter() + .filter_map(|(name, flag)| { + let program = path_of(name)?.to_owned(); + let home = home.to_string_lossy().into_owned(); + Some(( + name, + std::thread::spawn(move || { + let directory = Path::new(&program) + .parent() + .map(|p| p.display().to_string()) + .unwrap_or_default(); + let path = format!("{directory}:/usr/bin:/bin"); + let (stdout, stderr) = run_bounded( + Path::new(&program), + &[flag], + &[ + ("PATH", path.as_str()), + ("HOME", home.as_str()), + ("LANG", "C"), + ], + )?; + version_in(&stdout).or_else(|| version_in(&stderr)) + }), + )) + }) + .collect(); + let gpus_thread = { + let program = path_of("nvidia-smi").map(str::to_owned); + std::thread::spawn(move || gpus(program.as_deref())) + }; + let mounts = std::fs::read_to_string("/proc/self/mounts") + .map(|text| parse_mounts(&text)) + .unwrap_or_default(); + let storage = if home.as_os_str().is_empty() { + Vec::new() + } else { + storage(&mounts, &home) + }; + let mut conda: Vec = Vec::new(); + for place in CONDA_PLACES { + let path = match place.strip_prefix("~/") { + Some(inside) => home.join(inside), + None => PathBuf::from(place), + }; + if path.join("bin").join("conda").exists() || path.join("bin").join("micromamba").exists() { + conda.push(place.to_owned()); + } + } + for name in ["conda", "mamba"] { + let prefix = path_of(name) + .and_then(|path| Path::new(path).parent()?.parent().map(Path::to_path_buf)); + if let Some(shown) = prefix.and_then(|prefix| shown_path(&prefix, &home)) { + if !conda.contains(&shown) { + conda.push(shown); + } + } + } + let modules = login_modules + || MODULE_MARKERS + .iter() + .any(|marker| Path::new(marker).exists()); + let scheduler_commands: Vec<&str> = names + .iter() + .copied() + .filter(|name| { + matches!( + *name, + "sbatch" + | "squeue" + | "sinfo" + | "srun" + | "qsub" + | "qstat" + | "qconf" + | "pbsnodes" + | "bsub" + | "condor_submit" + ) + }) + .collect(); + let mut languages = serde_json::Map::new(); + for (name, thread) in versions { + if let Ok(Some(version)) = thread.join() { + languages.insert(name.to_owned(), json!(version)); + } + } + json!({ + "schema": 1, + "host": hostname(), + "os": os_release(), + "kernel": kernel, + "arch": arch, + "glibc": glibc_version(), + "cpus": unsafe { libc::sysconf(libc::_SC_NPROCESSORS_ONLN) }.max(0), + "mem_gib": memory_gib(), + "gpus": gpus_thread.join().unwrap_or_default(), + "scheduler": {"kind": scheduler_kind(&names), "commands": scheduler_commands}, + "modules": modules, + "conda": conda, + "containers": { + "docker": names.contains(&"docker"), + "docker_group": in_group("docker"), + "apptainer": names.contains(&"apptainer") || names.contains(&"singularity"), + "podman": names.contains(&"podman"), + }, + "languages": languages, + "storage": storage, + "landlock_abi": landlock_abi(), + "user": account.as_ref().map(|a| a.user.clone()), + "home": account.as_ref().map(|a| a.home.clone()), + }) +} + +/// Why a broker cannot host a workspace in this account, each a code, mirroring the checks the +/// broker itself makes when it starts: `linux_required`, `pidfd_unavailable` (a safe `stop` +/// needs `pidfd_open`), `machine_id_unavailable` (node identity), `state_on_network_filesystem` +/// and `home_writable_by_others` (the state directory's ancestors). +pub fn hosting_blockers(account: &ThisAccount) -> Vec<&'static str> { + let mut blockers = Vec::new(); + if !cfg!(target_os = "linux") { + blockers.push("linux_required"); + return blockers; + } + #[cfg(target_os = "linux")] + { + let fd = unsafe { libc::syscall(libc::SYS_pidfd_open, libc::getpid(), 0) }; + if fd < 0 { + blockers.push("pidfd_unavailable"); + } else { + unsafe { libc::close(fd as libc::c_int) }; + } + } + if !machine_id_usable() { + blockers.push("machine_id_unavailable"); + } + let state = account.home.join(".local/share/biorouter-crew"); + let existing = state + .ancestors() + .find(|path| path.exists()) + .map(Path::to_path_buf) + .unwrap_or_else(|| PathBuf::from("/")); + if on_network_filesystem(&existing) { + blockers.push("state_on_network_filesystem"); + } + let canonical = std::fs::canonicalize(&existing).unwrap_or(existing); + let writable = canonical.ancestors().any(|ancestor| { + use std::os::unix::fs::MetadataExt; + std::fs::symlink_metadata(ancestor) + .map(|m| m.mode() & 0o022 != 0 && m.mode() & 0o1000 == 0) + .unwrap_or(false) + }); + if writable { + blockers.push("home_writable_by_others"); + } + blockers +} + +/// The broker's node identity rule (`node_identity` in `broker.rs`): the first of +/// `/etc/machine-id` and `/var/lib/dbus/machine-id` that exists is a root-owned regular file +/// nobody else may write, holding 32 hex digits that are not all zero. +fn machine_id_usable() -> bool { + use std::os::unix::fs::{MetadataExt, OpenOptionsExt}; + for path in ["/etc/machine-id", "/var/lib/dbus/machine-id"] { + let file = match std::fs::OpenOptions::new() + .read(true) + .custom_flags(libc::O_NOFOLLOW | libc::O_CLOEXEC) + .open(path) + { + Ok(file) => file, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => continue, + Err(_) => return false, + }; + let Ok(metadata) = file.metadata() else { + return false; + }; + if !(metadata.is_file() + && metadata.uid() == 0 + && metadata.mode() & 0o022 == 0 + && metadata.len() <= 128) + { + return false; + } + let mut value = String::new(); + if file.take(129).read_to_string(&mut value).is_err() { + return false; + } + let value = value.trim(); + return value.len() == 32 + && value.bytes().all(|b| b.is_ascii_hexdigit()) + && value.bytes().any(|b| b != b'0'); + } + false +} + +/// The broker's network filesystem rule: NFS and SMB/CIFS are refused for workspace state. +fn on_network_filesystem(path: &Path) -> bool { + #[cfg(target_os = "linux")] + { + let Ok(path) = std::ffi::CString::new(path.as_os_str().as_encoded_bytes()) else { + return false; + }; + let mut info = std::mem::MaybeUninit::::uninit(); + if unsafe { libc::statfs(path.as_ptr(), info.as_mut_ptr()) } != 0 { + return false; + } + let kind = unsafe { info.assume_init() }.f_type; + kind == 0x6969 || kind == 0xff534d42_u64 as libc::c_long || kind == 0x517B + } + #[cfg(not(target_os = "linux"))] + { + let _ = path; + false + } +} + +/// What a workspace's agents cannot do on this server, each a code: `landlock_abi_below_3` and +/// `seccomp_unavailable` mean folder-mode (confined) commands cannot run; full access still can. +pub fn execution_limits() -> Vec<&'static str> { + let mut limits = Vec::new(); + if landlock_abi() < 3 { + limits.push("landlock_abi_below_3"); + } + if !seccomp_available() { + limits.push("seccomp_unavailable"); + } + limits +} + +/// `biorouter-crew doctor --output-format json`: the probe, this binary's version, and what +/// blocks hosting or limits agents here. +pub fn doctor() -> Value { + let mut facts = probe(); + facts["version"] = json!(env!("CARGO_PKG_VERSION")); + facts["hosting_blockers"] = match this_account() { + Ok(account) => json!(hosting_blockers(&account)), + Err(_) => json!(["identity_unavailable"]), + }; + facts["execution_limits"] = json!(execution_limits()); + facts +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn version_numbers_are_found_in_version_text() { + assert_eq!(version_in("Python 3.10.12\n").as_deref(), Some("3.10.12")); + assert_eq!( + version_in("R version 4.1.2 (2021-11-01) -- \"Bird Hippie\"").as_deref(), + Some("4.1.2") + ); + assert_eq!(version_in("julia version 1.9.3").as_deref(), Some("1.9.3")); + assert_eq!(version_in("no version here"), None); + } + + #[test] + fn mounts_are_parsed_with_their_escapes() { + let mounts = parse_mounts( + "pool1/home /pool1/home zfs rw 0 0\n/dev/sda1 /mnt/my\\040data ext4 rw 0 0\nproc /proc proc rw 0 0\n", + ); + assert_eq!(mounts.len(), 3); + assert_eq!(mounts[1].path, PathBuf::from("/mnt/my data")); + assert_eq!(mounts[1].fs, "ext4"); + let home = mount_of(&mounts, Path::new("/pool1/home/wanjun/project")).unwrap(); + assert_eq!(home.path, PathBuf::from("/pool1/home")); + } + + /// Another person's home is never named, whatever spelling it takes; the account's own home + /// and the directories above it are. + #[test] + fn other_peoples_homes_are_never_reported() { + let home = Path::new("/pool1/home/wanjun"); + assert!(in_another_home(Path::new("/pool1/home/alice"), home)); + assert!(in_another_home( + Path::new("/pool1/home/alice/anaconda3"), + home + )); + assert!(in_another_home(Path::new("/home/bob"), home)); + assert!(!in_another_home(Path::new("/pool1/home"), home)); + assert!(!in_another_home(Path::new("/pool1/home/wanjun/data"), home)); + assert!(!in_another_home(Path::new("/opt/conda"), home)); + assert_eq!( + shown_path(Path::new("/pool1/home/wanjun/anaconda3"), home).as_deref(), + Some("~/anaconda3") + ); + assert_eq!(shown_path(Path::new("/pool1/home/alice/x"), home), None); + assert_eq!( + shown_path(Path::new("/opt/conda"), home).as_deref(), + Some("/opt/conda") + ); + } + + #[test] + fn the_scheduler_is_named_by_its_commands() { + assert_eq!(scheduler_kind(&["sbatch", "squeue"]), Some("slurm")); + assert_eq!(scheduler_kind(&["qsub", "qconf"]), Some("sge")); + assert_eq!(scheduler_kind(&["qsub", "qstat"]), Some("pbs")); + assert_eq!(scheduler_kind(&["bsub"]), Some("lsf")); + assert_eq!(scheduler_kind(&["docker"]), None); + } + + /// The facts have the shape the context draft reads, with every key present. + #[test] + fn the_probe_has_the_contract_shape() { + let facts = probe(); + for key in [ + "schema", + "host", + "os", + "kernel", + "arch", + "glibc", + "cpus", + "mem_gib", + "gpus", + "scheduler", + "modules", + "conda", + "containers", + "languages", + "storage", + "landlock_abi", + "user", + "home", + ] { + assert!(facts.get(key).is_some(), "missing {key}: {facts}"); + } + assert_eq!(facts["schema"], 1); + assert!(facts["gpus"].is_array() && facts["conda"].is_array()); + assert!(facts["storage"].is_array()); + assert!(facts["scheduler"]["commands"].is_array()); + for key in ["docker", "docker_group", "apptainer", "podman"] { + assert!(facts["containers"][key].is_boolean(), "{key}"); + } + assert!(facts["modules"].is_boolean()); + assert!(serde_json::to_vec(&facts).unwrap().len() < 64 * 1024); + let doctor = doctor(); + assert!(doctor["hosting_blockers"].is_array()); + assert!(doctor["execution_limits"].is_array()); + assert_eq!(doctor["version"], env!("CARGO_PKG_VERSION")); + } + + #[test] + fn bridge_hello_describes_this_bridge_and_account() { + let hello = bridge_method("bridge.hello").unwrap(); + assert_eq!(hello["bridge_version"], env!("CARGO_PKG_VERSION")); + assert_eq!( + hello["capabilities"], + json!(["remote_full_v1", "remote_transfer_v1", "bridge_probe_v1"]) + ); + assert_eq!(hello["uid"], unsafe { libc::geteuid() }); + assert!(hello["home"] + .as_str() + .is_some_and(|home| home.starts_with('/'))); + assert_eq!( + hello["confined_execution_supported"], + landlock_abi() >= 3 && seccomp_available() + ); + let error = bridge_method("bridge.anything").unwrap_err().to_string(); + assert!(error.starts_with("unsupported: "), "{error}"); + } +} diff --git a/crates/biorouter-crew/src/remote.rs b/crates/biorouter-crew/src/remote.rs index 95f07995d..33ba39ff7 100644 --- a/crates/biorouter-crew/src/remote.rs +++ b/crates/biorouter-crew/src/remote.rs @@ -1,4 +1,5 @@ -//! Own-UID scoped file operations and fail-closed, single-process Linux jobs. +//! Own-UID scoped file operations and fail-closed, single-process Linux jobs: folder mode. +//! A run whose scope says `access: "full"` is handed to [`crate::remote_full`] instead. use anyhow::{anyhow, bail, ensure, Result}; use serde_json::{json, Value}; use sha2::{Digest, Sha256}; @@ -236,11 +237,53 @@ fn open_scoped(root: &Path, path: &Path, write: bool) -> Result { } Ok(dir) } -pub fn handle(method: &str, params: &Value, scope: &Value) -> Result { +/// Methods only a full-access run has (design OW-6, OW-7). +const FULL_ACCESS_METHODS: [&str; 4] = [ + "remote.shell", + "remote.stat", + "remote.fetch_blob", + "remote.share_file", +]; + +/// The run's access level, as its scope says. A scope without `access` comes from a broker +/// older than full access and is a folder run, so a newer bridge fails closed. +pub fn access(scope: &Value) -> Result { + match scope.get("access") { + None | Some(Value::Null) => Ok(crate::RemoteAccess::Folder), + Some(value) if value == "folder" => Ok(crate::RemoteAccess::Folder), + Some(value) if value == "full" => Ok(crate::RemoteAccess::Full), + Some(_) => bail!("invalid_scope: unknown remote access level"), + } +} + +/// What a folder-mode command answers when the kernel cannot confine it: said up front, naming +/// the level that can run commands here, instead of the confinement library's own error from +/// inside the helper. +pub fn confinement_refusal(landlock_abi: u32) -> Option { + (landlock_abi < 3).then(|| { + format!( + "unsupported: This server's kernel can't confine commands (Landlock ABI {landlock_abi}). Choose 'As me on the server' in Connection settings to run commands as yourself." + ) + }) +} + +pub fn handle( + method: &str, + params: &Value, + scope: &Value, + link: &crate::remote_full::BrokerLink, +) -> Result { ensure!( cfg!(target_os = "linux"), "unsupported: scoped remote operations require Linux" ); + if access(scope)? == crate::RemoteAccess::Full { + return crate::remote_full::handle(method, params, scope, link); + } + ensure!( + !FULL_ACCESS_METHODS.contains(&method), + "unsupported: {method} needs full access. Choose 'As me on the server' in Connection settings." + ); let root = root(scope)?; let path = relative(params)?; match method { @@ -599,6 +642,9 @@ fn start_job(root: &Path, params: &Value, scope: &Value) -> Result { scope["allow_execute"] == true, "forbidden: remote execution requires an explicit human grant" ); + if let Some(refusal) = confinement_refusal(crate::probe::landlock_abi()) { + bail!(refusal); + } validate_work_tree(root)?; let argv: Vec = serde_json::from_value(params["argv"].clone())?; ensure!( @@ -1071,6 +1117,58 @@ mod tests { refused("/srv", BRIDGE); } + /// A scope from an older broker names no access level and stays in folder mode; only the + /// exact word `full` leaves it, and an unknown level is refused rather than guessed. + #[test] + fn a_scope_without_access_is_folder_mode() { + assert_eq!( + access(&json!({"root": "/x"})).unwrap(), + crate::RemoteAccess::Folder + ); + assert_eq!( + access(&json!({"access": "folder"})).unwrap(), + crate::RemoteAccess::Folder + ); + assert_eq!( + access(&json!({"access": "full"})).unwrap(), + crate::RemoteAccess::Full + ); + for level in [json!("FULL"), json!(true), json!("confined")] { + assert!(access(&json!({ "access": level })).is_err(), "{level}"); + } + } + + /// A kernel without Landlock ABI 3 is refused before anything is reserved or started, with + /// a sentence that names full access. + #[test] + fn a_kernel_that_cannot_confine_is_refused_up_front() { + let refusal = confinement_refusal(1).unwrap(); + assert!(refusal.starts_with("unsupported: "), "{refusal}"); + assert!(refusal.contains("Landlock ABI 1"), "{refusal}"); + assert!(refusal.contains("As me on the server"), "{refusal}"); + assert!(confinement_refusal(0).is_some()); + assert!(confinement_refusal(3).is_none()); + assert!(confinement_refusal(6).is_none()); + } + + /// The methods only full access has are refused in folder mode with a sentence that says + /// so, before the work folder is even looked at. + #[cfg(target_os = "linux")] + #[test] + fn full_access_methods_are_refused_in_folder_mode() { + let link = crate::remote_full::BrokerLink { + connect: Arc::new(|| bail!("no broker in this test")), + credential: String::new(), + }; + let scope = json!({"root": "/nonexistent", "public_provider": false, "access": "folder"}); + for method in FULL_ACCESS_METHODS { + let error = handle(method, &json!({}), &scope, &link) + .unwrap_err() + .to_string(); + assert!(error.contains("needs full access"), "{method}: {error}"); + } + } + #[test] fn the_running_bridge_can_locate_itself() { let bridges = bridge_executables().unwrap(); diff --git a/crates/biorouter-crew/src/remote_full.rs b/crates/biorouter-crew/src/remote_full.rs new file mode 100644 index 000000000..edf4c4dce --- /dev/null +++ b/crates/biorouter-crew/src/remote_full.rs @@ -0,0 +1,1920 @@ +//! Full-access remote operations (design OW-6, OW-7): what a run whose `run.remote_scope` +//! answers `access: "full"` may do through its owner's own bridge. +//! +//! Commands run as the owner's account through its login shell, with the network, child +//! processes and every path the account can reach, under a supervisor process +//! (`biorouter-crew __crew-run `) that records each job in +//! `~/.local/state/biorouter-crew/jobs//`. A job is keyed by its owner and idempotency +//! key, so any later run of the same person can read it, and a detached job outlives the bridge +//! and the grant. File operations take absolute, `~/` or work-folder-relative paths and follow +//! symlinks; nothing here checks ownership, because the account's own permissions are the +//! boundary. Two writes are refused anyway, as accident prevention rather than security: the +//! running bridge executable and `~/.ssh/authorized_keys`. Channel attachments move to and from +//! the server in background transfers over a second broker connection, so the relay never +//! waits on them. +//! +//! Every answer stays well under the 1 MiB frame: reads and writes move at most +//! [`MAX_CHUNK_BYTES`], output tails at most [`MAX_TAIL`] per stream. +use anyhow::{anyhow, bail, ensure, Result}; +use serde_json::{json, Value}; +use sha2::{Digest, Sha256}; +use std::{ + fs::{self, File, OpenOptions}, + io::{BufRead, BufReader, Read, Seek, SeekFrom, Write}, + os::unix::{ + fs::{DirBuilderExt, MetadataExt, OpenOptionsExt, PermissionsExt}, + net::UnixStream, + }, + path::{Component, Path, PathBuf}, + sync::Arc, + time::{Duration, Instant, SystemTime, UNIX_EPOCH}, +}; + +/// The most bytes one `remote.read` returns or one `remote.write` takes: hex-encoded, it fits a +/// frame with room to spare. +pub const MAX_CHUNK_BYTES: usize = 393_216; +/// A command's own default and largest run time, in seconds. +const DEFAULT_TIMEOUT: u64 = 600; +const MAX_TIMEOUT: u64 = 3600; +/// How long `remote.shell` waits for its command before answering `running`. +const DEFAULT_WAIT: u64 = 10; +const MAX_WAIT: u64 = 20; +/// Output kept in an answer, per stream. +const DEFAULT_TAIL: usize = 16_384; +const MAX_TAIL: usize = 65_536; +/// Jobs (commands and transfers) running at once in this account, and records kept. +const MAX_RUNNING: usize = 8; +const MAX_RECORDS: usize = 512; +/// The longest command text, or the most bytes of all argv items together. +const MAX_COMMAND: usize = 65_536; +const MAX_ARGV: usize = 256; +/// Directory entries one listing returns by default and at most. +const DEFAULT_LIST: usize = 500; +const MAX_LIST: usize = 2_000; +/// The largest attachment a transfer moves, as the broker allows. +const MAX_TRANSFER: u64 = 1 << 30; +/// The largest file `remote.hash` reads in one answer. +const MAX_HASH: u64 = 2 << 30; +/// How long a cancelled job's process group has between `SIGTERM` and `SIGKILL`. +const CANCEL_GRACE: Duration = Duration::from_secs(5); +/// A record with neither a process nor an exit this long after it was made never started. +const START_GRACE_SECS: u64 = 30; +/// The broker reads attachments in chunks of this size, and takes uploads in the same. +const TRANSFER_CHUNK: usize = 262_144; +/// A transfer writes its progress at most this often (in bytes moved). +const PROGRESS_EVERY: u64 = 4 * 1024 * 1024; + +/// How a background transfer reaches the broker: a second connection, opened the way the +/// bridge opened its first, and the worker credential of the frame that asked for it. +pub type Connect = Arc Result<(UnixStream, BufReader)> + Send + Sync>; + +/// What a frame's operation may use to talk to the broker on its own. +#[derive(Clone)] +pub struct BrokerLink { + pub connect: Connect, + pub credential: String, +} + +/// Where full-access operations run: the account's home (what `~/` means), the job records +/// directory, and the executable the supervisor is started from (the running bridge), which is +/// also one of the two files a structured write refuses. +#[derive(Clone, Debug)] +pub struct Context { + pub home: PathBuf, + pub jobs: PathBuf, + pub executable: PathBuf, +} + +impl Context { + /// The bridge's own: `HOME` from its SSH login (the account database when unset) and the + /// executable this process was started from. + pub fn of_this_process() -> Result { + let home = std::env::var_os("HOME") + .map(PathBuf::from) + .filter(|home| home.is_absolute()) + .map(Ok) + .unwrap_or_else(|| crate::probe::this_account().map(|account| account.home))?; + let executable = std::env::current_exe() + .map_err(|_| anyhow!("invalid_scope: cannot locate the Crew bridge executable"))?; + Ok(Self { + jobs: home.join(".local/state/biorouter-crew/jobs"), + home, + executable, + }) + } +} + +fn text<'a>(v: &'a Value, key: &str) -> Result<&'a str> { + v[key] + .as_str() + .ok_or_else(|| anyhow!("invalid_params: {key} must be a string")) +} + +fn now() -> u64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|d| d.as_secs()) + .unwrap_or(0) +} + +/// Full access never runs as root: a root bridge would make every command the server's. +pub(crate) fn refuse_root(euid: u32) -> Result<()> { + ensure!( + euid != 0, + "forbidden: Full access never runs commands as root." + ); + Ok(()) +} + +/// A full-access operation, as the bridge's relay calls it with a scope the broker just gave. +pub fn handle(method: &str, params: &Value, scope: &Value, link: &BrokerLink) -> Result { + refuse_root(unsafe { libc::geteuid() })?; + handle_in(&Context::of_this_process()?, method, params, scope, link) +} + +/// [`handle`] in a given [`Context`]; tests pass their own home and job directory. +pub fn handle_in( + context: &Context, + method: &str, + params: &Value, + scope: &Value, + link: &BrokerLink, +) -> Result { + ensure!( + scope["public_provider"] == false, + "forbidden: public providers cannot access remote files or jobs" + ); + ensure!( + scope["access"] == "full", + "invalid_scope: this run does not have full access" + ); + let owner = text(scope, "owner_id") + .ok() + .filter(|owner| !owner.is_empty()) + .ok_or_else(|| anyhow!("invalid_scope: full access needs the run's owner"))?; + let root = PathBuf::from(text(scope, "root")?); + ensure!(root.is_absolute(), "invalid_scope: root must be absolute"); + match method { + "remote.shell" | "remote.execute" => start_command(context, &root, method, params, scope), + "remote.job_status" => { + let dir = job_dir(context, params, owner)?; + job_view(&dir, tail_bytes(params)?) + } + "remote.cancel" => cancel(&job_dir(context, params, owner)?), + "remote.list" => list(&resolve(context, &root, params)?, params), + "remote.stat" => stat(&resolve(context, &root, params)?), + "remote.read" => read(&resolve(context, &root, params)?, params), + "remote.write" => write(context, &resolve(context, &root, params)?, params), + "remote.hash" => hash(&resolve(context, &root, params)?), + "remote.fetch_blob" => fetch_blob(context, &root, params, scope, link), + "remote.share_file" => share_file(context, &root, params, scope, link), + _ => bail!("unsupported: remote operation unavailable"), + } +} + +/// Where `path` points: `~` and `~/…` inside the home, an absolute path as it is, anything +/// else inside the work folder `root`. Nothing is resolved further: symlinks are followed by +/// the operation that opens the path. +pub fn resolve_path(home: &Path, root: &Path, path: &str) -> Result { + ensure!( + !path.contains('\0') && path.len() <= 4096, + "invalid_params: path must be at most 4096 bytes without NUL" + ); + Ok(if path == "~" { + home.to_path_buf() + } else if let Some(inside) = path.strip_prefix("~/") { + home.join(inside) + } else if path.starts_with('~') { + bail!("invalid_params: ~user paths are not supported; give an absolute path instead.") + } else if Path::new(path).is_absolute() { + PathBuf::from(path) + } else { + root.join(path) + }) +} + +fn resolve(context: &Context, root: &Path, params: &Value) -> Result { + let path = match params.get("path") { + None | Some(Value::Null) => ".", + Some(Value::String(path)) => path.as_str(), + Some(_) => bail!("invalid_params: path must be a string"), + }; + resolve_path(&context.home, root, path) +} + +/// `path` with `.` and `..` folded away lexically, for comparing a path that does not exist yet. +fn lexical(path: &Path) -> PathBuf { + let mut out = PathBuf::new(); + for component in path.components() { + match component { + Component::ParentDir => { + out.pop(); + } + Component::CurDir => {} + other => out.push(other), + } + } + out +} + +/// `path` as given and, when it resolves, as the filesystem resolves it. +fn spellings(path: &Path) -> Vec { + let mut variants = vec![lexical(path)]; + if let Ok(resolved) = fs::canonicalize(path) { + if !variants.contains(&resolved) { + variants.push(resolved); + } + } + variants +} + +/// The files a structured write refuses, under every spelling: the running bridge, and the +/// account's `authorized_keys` files. The shell can still change them; this keeps an agent's +/// ordinary file write from locking the person out by accident. +fn protected_writes(context: &Context) -> Vec { + let mut protected = spellings(&context.executable); + for home in spellings(&context.home) { + for name in ["authorized_keys", "authorized_keys2"] { + protected.push(home.join(".ssh").join(name)); + } + } + protected +} + +/// Refuse a write whose target, however it is spelled or linked, is a protected file. +pub(crate) fn refuse_protected_write(context: &Context, target: &Path) -> Result<()> { + let resolved = match fs::canonicalize(target) { + Ok(resolved) => resolved, + Err(_) => { + ensure!( + fs::symlink_metadata(target).is_err(), + "invalid_params: {} is a symbolic link to nothing; write to a real path instead.", + target.display() + ); + let lexical_target = lexical(target); + match (lexical_target.parent(), lexical_target.file_name()) { + (Some(parent), Some(name)) => fs::canonicalize(parent) + .map(|parent| parent.join(name)) + .unwrap_or(lexical_target.clone()), + _ => lexical_target, + } + } + }; + let lexical_target = lexical(target); + for protected in protected_writes(context) { + ensure!( + resolved != protected && lexical_target != protected, + "forbidden: Crew does not write {}, which keeps your Crew bridge or SSH sign-in working. Change it yourself if you mean to.", + protected.display() + ); + } + Ok(()) +} + +fn mtime(metadata: &fs::Metadata) -> i64 { + metadata.mtime() +} + +fn kind_of(file_type: fs::FileType) -> &'static str { + if file_type.is_symlink() { + "symlink" + } else if file_type.is_dir() { + "directory" + } else if file_type.is_file() { + "file" + } else { + "other" + } +} + +fn list(path: &Path, params: &Value) -> Result { + let limit = match params.get("limit") { + None | Some(Value::Null) => DEFAULT_LIST, + Some(value) => value + .as_u64() + .filter(|limit| (1..=MAX_LIST as u64).contains(limit)) + .ok_or_else(|| anyhow!("invalid_params: limit must be 1..{MAX_LIST}"))? + as usize, + }; + let mut entries = Vec::new(); + let mut truncated = false; + for entry in fs::read_dir(path).map_err(|e| anyhow!("remote path unavailable: {e}"))? { + if entries.len() == limit { + truncated = true; + break; + } + let entry = entry?; + let metadata = entry.metadata()?; + let mut item = json!({ + "name": entry.file_name().to_string_lossy(), + "type": kind_of(metadata.file_type()), + "size": metadata.len(), + "mtime": mtime(&metadata), + "mode": format!("{:04o}", metadata.mode() & 0o7777), + }); + if metadata.file_type().is_symlink() { + if let Ok(target) = fs::metadata(entry.path()) { + item["target_type"] = json!(kind_of(target.file_type())); + } + } + entries.push(item); + } + entries.sort_by(|a, b| a["name"].as_str().cmp(&b["name"].as_str())); + Ok(json!({"path": path, "entries": entries, "truncated": truncated})) +} + +fn stat(path: &Path) -> Result { + let Ok(link) = fs::symlink_metadata(path) else { + return Ok(json!({"path": path, "exists": false})); + }; + let symlink = link.file_type().is_symlink(); + let Ok(metadata) = fs::metadata(path) else { + return Ok( + json!({"path": path, "exists": false, "symlink": true, "target": fs::read_link(path).ok()}), + ); + }; + let mut answer = json!({ + "path": path, + "exists": true, + "type": kind_of(metadata.file_type()), + "size": metadata.len(), + "mtime": mtime(&metadata), + "mode": format!("{:04o}", metadata.mode() & 0o7777), + "uid": metadata.uid(), + "gid": metadata.gid(), + "symlink": symlink, + }); + if symlink { + answer["target"] = json!(fs::read_link(path).ok()); + } + Ok(answer) +} + +/// `remote.read {path, offset?, length?}`: at most [`MAX_CHUNK_BYTES`] from `offset`. The bytes +/// come back as hex, and also as text when they are UTF-8 and the frame has room for both. +fn read(path: &Path, params: &Value) -> Result { + let offset = params.get("offset").and_then(Value::as_u64).unwrap_or(0); + let length = match params.get("length") { + None | Some(Value::Null) => MAX_CHUNK_BYTES, + Some(value) => value + .as_u64() + .filter(|length| *length <= MAX_CHUNK_BYTES as u64) + .ok_or_else(|| anyhow!("invalid_params: length must be 0..{MAX_CHUNK_BYTES}"))? + as usize, + }; + let mut file = File::open(path).map_err(|e| anyhow!("remote path unavailable: {e}"))?; + let metadata = file.metadata()?; + ensure!(metadata.is_file(), "not a regular file"); + let file_size = metadata.len(); + ensure!( + offset <= file_size, + "invalid_params: offset beyond the end of the file" + ); + file.seek(SeekFrom::Start(offset))?; + let mut bytes = Vec::with_capacity(length.min(file_size.saturating_sub(offset) as usize)); + (&mut file).take(length as u64).read_to_end(&mut bytes)?; + let next_offset = offset + bytes.len() as u64; + let data_hex = hex::encode(&bytes); + let mut answer = json!({ + "path": path, + "offset": offset, + "size": bytes.len(), + "file_size": file_size, + "next_offset": next_offset, + "eof": next_offset >= file_size, + "sha256": hex::encode(Sha256::digest(&bytes)), + "data_hex": data_hex, + }); + if let Ok(text) = std::str::from_utf8(&bytes) { + // Escaped text can be six times its bytes; it rides along only while the frame has room. + if serde_json::to_string(text)?.len() + data_hex.len() < 960_000 { + answer["text_utf8"] = json!(text); + } + } + Ok(answer) +} + +/// `remote.write {path, text | data_hex, mode?, create_parents?}`: `truncate` (the default) +/// replaces the file, `append` adds to it and `create_new` refuses one that exists. +fn write(context: &Context, path: &Path, params: &Value) -> Result { + ensure!( + params.get("text").is_some() != params.get("data_hex").is_some(), + "invalid_params: provide exactly one of text or data_hex" + ); + let bytes = if let Some(text) = params.get("text") { + text.as_str() + .ok_or_else(|| anyhow!("invalid_params: text must be a string"))? + .as_bytes() + .to_vec() + } else { + hex::decode(text(params, "data_hex")?) + .map_err(|_| anyhow!("invalid_params: data_hex must be hexadecimal"))? + }; + ensure!( + bytes.len() <= MAX_CHUNK_BYTES, + "invalid_params: a write takes at most {MAX_CHUNK_BYTES} bytes; append the rest" + ); + let mode = params + .get("mode") + .and_then(Value::as_str) + .unwrap_or("truncate"); + let mut options = OpenOptions::new(); + options.write(true); + match mode { + "truncate" => options.create(true).truncate(true), + "append" => options.create(true).append(true), + "create_new" => options.create_new(true), + _ => bail!("invalid_params: mode must be truncate, append or create_new"), + }; + refuse_protected_write(context, path)?; + if params.get("create_parents").and_then(Value::as_bool) == Some(true) { + if let Some(parent) = path.parent() { + fs::create_dir_all(parent).map_err(|e| anyhow!("remote path unavailable: {e}"))?; + } + } + let mut file = options.open(path).map_err(|e| { + if e.kind() == std::io::ErrorKind::AlreadyExists { + anyhow!("conflict: {} already exists", path.display()) + } else { + anyhow!("remote path unavailable: {e}") + } + })?; + ensure!(file.metadata()?.is_file(), "not a regular file"); + file.write_all(&bytes)?; + file.sync_all()?; + Ok(json!({ + "path": path, + "mode": mode, + "size": bytes.len(), + "file_size": file.metadata()?.len(), + "sha256": hex::encode(Sha256::digest(&bytes)), + })) +} + +fn hash(path: &Path) -> Result { + let mut file = File::open(path).map_err(|e| anyhow!("remote path unavailable: {e}"))?; + let metadata = file.metadata()?; + ensure!(metadata.is_file(), "not a regular file"); + ensure!( + metadata.len() <= MAX_HASH, + "invalid_params: remote.hash reads files up to 2 GiB; run sha256sum with remote.shell for larger ones" + ); + let mut hasher = Sha256::new(); + let mut buffer = vec![0u8; 1 << 20]; + let mut size = 0u64; + loop { + let n = file.read(&mut buffer)?; + if n == 0 { + break; + } + size += n as u64; + hasher.update(&buffer[..n]); + } + Ok(json!({"path": path, "sha256": hex::encode(hasher.finalize()), "size": size})) +} + +/// The job ID of `key` for `owner`: SHA-256 of `owner:key`, so any run of the same person reaches +/// the same job and nobody else's. +pub fn job_id(owner: &str, key: &str) -> String { + hex::encode(Sha256::digest(format!("{owner}:{key}").as_bytes())) +} + +fn idempotency_key(params: &Value) -> Result<&str> { + let key = text(params, "idempotency_key")?; + ensure!( + !key.is_empty() && key.len() <= 128, + "invalid_params: idempotency_key must be 1 to 128 bytes" + ); + Ok(key) +} + +fn tail_bytes(params: &Value) -> Result { + match params.get("tail_bytes") { + None | Some(Value::Null) => Ok(DEFAULT_TAIL), + Some(value) => value + .as_u64() + .filter(|tail| *tail <= MAX_TAIL as u64) + .map(|tail| tail as usize) + .ok_or_else(|| anyhow!("invalid_params: tail_bytes must be 0..{MAX_TAIL}")), + } +} + +/// The job directory a status or cancel names, refused unless the job is `owner`'s. +fn job_dir(context: &Context, params: &Value, owner: &str) -> Result { + let id = text(params, "job_id")?; + ensure!( + id.len() == 64 && id.bytes().all(|b| b.is_ascii_hexdigit()), + "invalid_params: invalid job ID" + ); + let dir = context.jobs.join(id.to_ascii_lowercase()); + let record = read_json(&dir.join("job.json")) + .map_err(|_| anyhow!("not_found: No Crew job has this ID in this account."))?; + ensure!( + record["owner_id"] == owner, + "forbidden: job belongs to another person" + ); + Ok(dir) +} + +/// A small JSON record the bridge or a supervisor wrote, never through a symlink. +fn read_json(path: &Path) -> Result { + let file = OpenOptions::new() + .read(true) + .custom_flags(libc::O_NOFOLLOW | libc::O_CLOEXEC) + .open(path)?; + let mut data = Vec::new(); + file.take(1_048_577).read_to_end(&mut data)?; + ensure!(data.len() <= 1_048_576, "invalid job record"); + Ok(serde_json::from_slice(&data)?) +} + +/// Replace `path` with `value` atomically: a private temporary file beside it, synced, renamed. +fn write_json(path: &Path, value: &Value) -> Result<()> { + let parent = path + .parent() + .ok_or_else(|| anyhow!("job state directory missing"))?; + let temporary = parent.join(format!(".record-{}.tmp", uuid::Uuid::new_v4().simple())); + let mut file = OpenOptions::new() + .write(true) + .create_new(true) + .mode(0o600) + .custom_flags(libc::O_NOFOLLOW | libc::O_CLOEXEC) + .open(&temporary)?; + file.write_all(&serde_json::to_vec(value)?)?; + file.sync_all()?; + drop(file); + fs::rename(&temporary, path)?; + File::open(parent)?.sync_all()?; + Ok(()) +} + +/// The jobs directory, created owner-only, refused when it is a symlink or someone else's. +fn jobs_directory(context: &Context) -> Result<&Path> { + let jobs = context.jobs.as_path(); + if let Some(parent) = jobs.parent() { + fs::DirBuilder::new() + .recursive(true) + .mode(0o700) + .create(parent)?; + } + match fs::DirBuilder::new().mode(0o700).create(jobs) { + Ok(()) => {} + Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => {} + Err(error) => return Err(error.into()), + } + let metadata = fs::symlink_metadata(jobs)?; + ensure!( + metadata.is_dir() && metadata.uid() == unsafe { libc::geteuid() }, + "unsafe_storage: the Crew jobs directory must be this account's own directory" + ); + if metadata.mode() & 0o077 != 0 { + fs::set_permissions(jobs, fs::Permissions::from_mode(0o700))?; + } + Ok(jobs) +} + +/// `/proc//stat`'s state letter and start time (field 22, in clock ticks since boot). +fn process_identity(pid: u32) -> Option<(char, u64)> { + let stat = fs::read_to_string(format!("/proc/{pid}/stat")).ok()?; + // The command name is in parentheses and may hold spaces or parentheses itself. + let rest = stat.get(stat.rfind(')')? + 1..)?; + let mut fields = rest.split_whitespace(); + let state = fields.next()?.chars().next()?; + let start = fields.nth(18)?.parse().ok()?; + Some((state, start)) +} + +/// Whether `pid` is still the process that started at `start` and has not exited. +fn alive(pid: u64, start: u64) -> bool { + u32::try_from(pid) + .ok() + .and_then(process_identity) + .is_some_and(|(state, started)| started == start && state != 'Z' && state != 'X') +} + +fn alive_in(record: &Value, pid: &str, start: &str) -> bool { + match (record[pid].as_u64(), record[start].as_u64()) { + (Some(pid), Some(start)) => alive(pid, start), + _ => false, + } +} + +/// The last `tail` bytes of an output file, and whether there was more. +fn tail_of(path: &Path, tail: usize) -> (String, bool) { + let Ok(mut file) = File::open(path) else { + return (String::new(), false); + }; + let length = file.metadata().map(|m| m.len()).unwrap_or(0); + let start = length.saturating_sub(tail as u64); + if file.seek(SeekFrom::Start(start)).is_err() { + return (String::new(), false); + } + let mut bytes = Vec::new(); + let _ = file.take(tail as u64).read_to_end(&mut bytes); + (String::from_utf8_lossy(&bytes).into_owned(), start > 0) +} + +/// A job's state as its record, process and exit files tell it: `starting`, `running`, +/// `completed`, `failed`, `timed_out`, `cancelled`, or `lost` when the process that ran it is +/// gone without saying how it ended. +fn job_state(dir: &Path, record: &Value) -> (String, Option) { + if let Ok(exit) = read_json(&dir.join("exit.json")) { + let state = exit["state"].as_str().unwrap_or("failed").to_owned(); + return (state, Some(exit)); + } + if let Ok(process) = read_json(&dir.join("process.json")) { + let running = alive_in(&process, "pid", "start_time") + || alive_in(&process, "supervisor_pid", "supervisor_start_time"); + return (if running { "running" } else { "lost" }.into(), None); + } + let created = record["created_at"].as_u64().unwrap_or(0); + let state = if now().saturating_sub(created) < START_GRACE_SECS { + "starting" + } else { + "lost" + }; + (state.into(), None) +} + +fn is_finished(state: &str) -> bool { + !matches!(state, "starting" | "running") +} + +/// What `remote.job_status` (and every start) answers for the job in `dir`. +fn job_view(dir: &Path, tail: usize) -> Result { + let record = read_json(&dir.join("job.json"))?; + let (state, exit) = job_state(dir, &record); + let mut answer = json!({ + "job_id": record["job_id"], + "kind": record["kind"], + "state": state, + "detach": record["detach"], + "created_at": record["created_at"], + }); + let exit = exit.unwrap_or(Value::Null); + for key in ["exit_code", "signal", "ended_at", "error", "message"] { + if !exit[key].is_null() { + answer[key] = exit[key].clone(); + } + } + match record["kind"].as_str() { + Some("shell" | "execute") => { + let (stdout_tail, stdout_cut) = tail_of(&dir.join("stdout"), tail); + let (stderr_tail, stderr_cut) = tail_of(&dir.join("stderr"), tail); + answer["stdout_tail"] = json!(stdout_tail); + answer["stderr_tail"] = json!(stderr_tail); + answer["truncated"] = json!(stdout_cut || stderr_cut); + answer["output_dir"] = json!(dir); + } + _ => { + let progress = read_json(&dir.join("progress.json")).unwrap_or(Value::Null); + answer["bytes_done"] = progress["bytes_done"].clone(); + answer["size"] = progress["size"].clone(); + for key in ["path", "blob_id", "name"] { + if !record[key].is_null() { + answer[key] = record[key].clone(); + } + } + if !exit["result"].is_null() { + answer["result"] = exit["result"].clone(); + if let Some(result) = exit["result"].as_object() { + for (key, value) in result { + answer[key] = value.clone(); + } + } + } + } + } + if state == "lost" && answer.get("message").is_none() { + answer["message"] = json!( + "The process that ran this job is gone and left no exit status. Inspect its output before retrying." + ); + } + Ok(answer) +} + +/// Make room for one more job: refuse while [`MAX_RUNNING`] run, and drop the oldest finished +/// records once [`MAX_RECORDS`] are kept. +fn make_room(jobs: &Path) -> Result<()> { + let mut finished = Vec::new(); + let mut records = 0usize; + let mut running = 0usize; + for entry in fs::read_dir(jobs)? { + let entry = entry?; + let name = entry.file_name(); + let Some(name) = name.to_str() else { continue }; + if name.len() != 64 || !name.bytes().all(|b| b.is_ascii_hexdigit()) { + continue; + } + records += 1; + let dir = entry.path(); + let Ok(record) = read_json(&dir.join("job.json")) else { + continue; + }; + let (state, exit) = job_state(&dir, &record); + if is_finished(&state) { + let ended = exit + .and_then(|exit| exit["ended_at"].as_u64()) + .or_else(|| record["created_at"].as_u64()) + .unwrap_or(0); + finished.push((ended, dir)); + } else { + running += 1; + } + } + ensure!( + running < MAX_RUNNING, + "quota_exceeded: {MAX_RUNNING} Crew jobs are already running in this account. Wait for one to finish or cancel one." + ); + if records >= MAX_RECORDS { + finished.sort(); + let excess = records + 1 - MAX_RECORDS; + ensure!( + finished.len() >= excess, + "quota_exceeded: This account keeps {MAX_RECORDS} Crew job records and none has finished." + ); + for (_, dir) in finished.into_iter().take(excess) { + let _ = fs::remove_dir_all(dir); + } + } + Ok(()) +} + +/// Create the job's record directory with `record` as its `job.json`, or find the one an earlier +/// request with the same key made: `Ok(None)` when created, `Ok(Some(dir))` when it existed and +/// asked for the same thing. +fn reserve(context: &Context, id: &str, record: &Value) -> Result<(PathBuf, bool)> { + let jobs = jobs_directory(context)?; + let dir = jobs.join(id); + if dir.exists() { + let existing = read_json(&dir.join("job.json"))?; + ensure!( + existing["owner_id"] == record["owner_id"], + "forbidden: job belongs to another person" + ); + ensure!( + existing["digest"] == record["digest"], + "conflict: idempotency key reused with a different request" + ); + return Ok((dir, false)); + } + make_room(jobs)?; + let temporary = jobs.join(format!(".new-{}", uuid::Uuid::new_v4().simple())); + fs::DirBuilder::new().mode(0o700).create(&temporary)?; + let made = (|| -> Result<()> { + write_json(&temporary.join("job.json"), record)?; + fs::rename(&temporary, &dir)?; + File::open(jobs)?.sync_all()?; + Ok(()) + })(); + if let Err(error) = made { + let _ = fs::remove_dir_all(&temporary); + // Another bridge of the same person made it first: answer with theirs. + if dir.join("job.json").exists() { + return reserve(context, id, record); + } + return Err(error); + } + Ok((dir, true)) +} + +/// The account's login shell from the account database, or `/bin/sh`. +fn login_shell() -> PathBuf { + crate::probe::this_account() + .ok() + .and_then(|account| account.shell) + .filter(|shell| shell.exists()) + .unwrap_or_else(|| PathBuf::from("/bin/sh")) +} + +fn shell_name(shell: &Path) -> &str { + shell.file_name().and_then(|n| n.to_str()).unwrap_or("") +} + +/// The argv a job runs: ` -lc `, or for an argv the login shell running it +/// exactly (`exec "$@"`), so modules, conda and `PATH` behave as in the person's own login. +/// csh-family shells take `-l` only alone, so they get `-c`; an argv under a shell that is not +/// of the Bourne family goes through `/bin/sh -l`. +pub(crate) fn job_argv( + shell: &Path, + command: Option<&str>, + argv: Option<&[String]>, +) -> Vec { + let name = shell_name(shell); + let shell_text = shell.to_string_lossy().into_owned(); + match (command, argv) { + (Some(command), _) => { + let flag = if matches!(name, "csh" | "tcsh") { + "-c" + } else { + "-lc" + }; + vec![shell_text, flag.into(), command.into()] + } + (None, Some(argv)) => { + let posix = matches!(name, "sh" | "bash" | "zsh" | "dash" | "ksh" | "mksh"); + let runner = if posix { shell_text } else { "/bin/sh".into() }; + let mut out = vec![ + runner, + "-lc".into(), + "exec \"$@\"".into(), + "biorouter-crew-job".into(), + ]; + out.extend(argv.iter().cloned()); + out + } + (None, None) => Vec::new(), + } +} + +fn start_command( + context: &Context, + root: &Path, + method: &str, + params: &Value, + scope: &Value, +) -> Result { + ensure!( + scope["allow_execute"] == true, + "forbidden: remote execution requires an explicit human grant" + ); + let expiry = scope["expires_at"] + .as_u64() + .ok_or_else(|| anyhow!("grant expiry missing"))?; + ensure!(expiry > now(), "remote execution grant expired"); + let key = idempotency_key(params)?; + let detach = match params.get("detach") { + None | Some(Value::Null) => false, + Some(value) => value + .as_bool() + .ok_or_else(|| anyhow!("invalid_params: detach must be true or false"))?, + }; + let timeout = match params.get("timeout_seconds") { + None | Some(Value::Null) => DEFAULT_TIMEOUT, + Some(value) => value + .as_u64() + .filter(|seconds| (1..=MAX_TIMEOUT).contains(seconds)) + .ok_or_else(|| anyhow!("invalid_params: timeout_seconds must be 1..{MAX_TIMEOUT}"))?, + }; + let wait = params + .get("wait_seconds") + .and_then(Value::as_u64) + .unwrap_or(DEFAULT_WAIT) + .min(MAX_WAIT); + let cwd = match params.get("cwd") { + None | Some(Value::Null) => root.to_path_buf(), + Some(Value::String(cwd)) => resolve_path(&context.home, root, cwd)?, + Some(_) => bail!("invalid_params: cwd must be a string"), + }; + ensure!( + cwd.is_dir(), + "invalid_params: the working directory {} does not exist", + cwd.display() + ); + let (kind, command, argv) = if method == "remote.shell" { + let command = text(params, "command")?; + ensure!( + !command.trim().is_empty() && command.len() <= MAX_COMMAND && !command.contains('\0'), + "invalid_params: command must be 1 to {MAX_COMMAND} bytes without NUL" + ); + ("shell", Some(command.to_owned()), None) + } else { + let argv: Vec = serde_json::from_value(params["argv"].clone()) + .map_err(|_| anyhow!("invalid_params: argv must be a list of strings"))?; + ensure!( + !argv.is_empty() + && argv.len() <= MAX_ARGV + && argv.iter().map(String::len).sum::() <= MAX_COMMAND + && argv.iter().all(|item| !item.contains('\0')), + "invalid_params: bounded argv required" + ); + ("execute", None, Some(argv)) + }; + let shell = login_shell(); + let spec = json!({"kind": kind, "command": command, "argv": argv, "cwd": cwd, "timeout_seconds": timeout, "detach": detach}); + let owner = text(scope, "owner_id")?; + let id = job_id(owner, key); + let record = json!({ + "job_id": id, + "kind": kind, + "owner_id": owner, + "run_id": scope["run_id"], + "digest": hex::encode(Sha256::digest(serde_json::to_vec(&spec)?)), + "created_at": now(), + "detach": detach, + "timeout_seconds": timeout, + "cwd": cwd, + "argv": job_argv(&shell, command.as_deref(), argv.as_deref()), + "parent_pid": std::process::id(), + }); + let (dir, created) = reserve(context, &id, &record)?; + if created { + if let Err(error) = spawn_supervisor(context, &dir, &id) { + let _ = write_json( + &dir.join("exit.json"), + &json!({"state": "failed", "ended_at": now(), "error": format!("The job could not start: {error}")}), + ); + } + } + let deadline = Instant::now() + Duration::from_secs(wait); + let record = read_json(&dir.join("job.json"))?; + while !is_finished(&job_state(&dir, &record).0) && Instant::now() < deadline { + std::thread::sleep(Duration::from_millis(100)); + } + job_view(&dir, DEFAULT_TAIL) +} + +/// Start `biorouter-crew __crew-run `. Its standard streams are `/dev/null`, never the +/// bridge's own (standard output is the frame channel to the daemon). It is reaped on a thread: +/// a job that is not detached stays this bridge's child, and dies with it. +fn spawn_supervisor(context: &Context, dir: &Path, id: &str) -> Result<()> { + let mut child = std::process::Command::new(&context.executable) + .arg("__crew-run") + .arg(dir) + .env("BIOROUTER_CREW_JOB_ID", id) + .current_dir("/") + .stdin(std::process::Stdio::null()) + .stdout(std::process::Stdio::null()) + .stderr(std::process::Stdio::null()) + .spawn()?; + std::thread::spawn(move || { + let _ = child.wait(); + }); + Ok(()) +} + +/// Send `signal` to the job's process group, only while its leader is still the process the +/// supervisor started. +fn signal_group(process: &Value, signal: libc::c_int) -> bool { + if !alive_in(process, "pid", "start_time") { + return false; + } + let Some(pgid) = process["pgid"].as_i64().and_then(|p| i32::try_from(p).ok()) else { + return false; + }; + pgid > 1 && unsafe { libc::kill(-pgid, signal) } == 0 +} + +/// `remote.cancel`: a command's process group gets `SIGTERM`, then `SIGKILL` after +/// [`CANCEL_GRACE`]; a transfer stops at its next chunk. A finished job is left as it is. +fn cancel(dir: &Path) -> Result { + let record = read_json(&dir.join("job.json"))?; + let (state, _) = job_state(dir, &record); + if is_finished(&state) { + return job_view(dir, DEFAULT_TAIL); + } + write_json(&dir.join("cancel.json"), &json!({"requested_at": now()}))?; + if matches!(record["kind"].as_str(), Some("shell" | "execute")) { + if let Ok(process) = read_json(&dir.join("process.json")) { + if signal_group(&process, libc::SIGTERM) { + let deadline = Instant::now() + CANCEL_GRACE; + while alive_in(&process, "pid", "start_time") && Instant::now() < deadline { + std::thread::sleep(Duration::from_millis(100)); + } + signal_group(&process, libc::SIGKILL); + } + } + } + let deadline = Instant::now() + Duration::from_secs(2); + while !dir.join("exit.json").exists() && Instant::now() < deadline { + std::thread::sleep(Duration::from_millis(50)); + } + job_view(dir, DEFAULT_TAIL) +} + +/// The supervisor (`biorouter-crew __crew-run `): runs one job's argv as this account +/// in its own process group, appends its output to the job's files, enforces its timeout and +/// writes `exit.json`. A job that is not detached ends with the bridge that started it. +pub fn supervise(dir: &Path) -> Result<()> { + #[cfg(target_os = "linux")] + { + linux_supervise(dir) + } + #[cfg(not(target_os = "linux"))] + { + let _ = dir; + bail!("unsupported: Crew jobs require Linux") + } +} + +#[cfg(target_os = "linux")] +static STOP_REQUESTED: std::sync::atomic::AtomicBool = std::sync::atomic::AtomicBool::new(false); + +#[cfg(target_os = "linux")] +extern "C" fn request_stop(_: libc::c_int) { + STOP_REQUESTED.store(true, std::sync::atomic::Ordering::SeqCst); +} + +#[cfg(target_os = "linux")] +fn linux_supervise(dir: &Path) -> Result<()> { + use std::os::unix::process::CommandExt; + let metadata = fs::symlink_metadata(dir)?; + ensure!( + metadata.is_dir() + && metadata.uid() == unsafe { libc::geteuid() } + && metadata.mode() & 0o077 == 0, + "unsafe_storage: job directory must be this account's own private directory" + ); + let record = read_json(&dir.join("job.json"))?; + let finish = |state: &str, extra: Value| { + let mut exit = json!({"state": state, "ended_at": now()}); + if let (Some(exit), Some(extra)) = (exit.as_object_mut(), extra.as_object()) { + exit.extend(extra.clone()); + } + write_json(&dir.join("exit.json"), &exit) + }; + let detach = record["detach"] == true; + if detach { + // The second fork: this process exits at once, so the bridge reaps it, and the + // supervisor that carries on is nobody's child but init's (or a subreaper's). + match unsafe { libc::fork() } { + -1 => return Err(std::io::Error::last_os_error().into()), + 0 => {} + _ => return Ok(()), + } + } else { + unsafe { + ensure!( + libc::prctl(libc::PR_SET_PDEATHSIG, libc::SIGTERM) == 0, + "parent lifecycle unavailable" + ); + } + if record["parent_pid"].as_u64() != Some(unsafe { libc::getppid() } as u64) { + return finish( + "cancelled", + json!({"message": "The Crew connection ended before the job started."}), + ); + } + } + unsafe { + libc::setsid(); + for signal in [libc::SIGTERM, libc::SIGINT, libc::SIGHUP] { + libc::signal( + signal, + request_stop as extern "C" fn(libc::c_int) as libc::sighandler_t, + ); + } + } + let argv: Vec = serde_json::from_value(record["argv"].clone())?; + ensure!(!argv.is_empty(), "argv missing"); + let output = |name: &str| { + OpenOptions::new() + .create(true) + .append(true) + .mode(0o600) + .custom_flags(libc::O_NOFOLLOW | libc::O_CLOEXEC) + .open(dir.join(name)) + }; + let (stdout, stderr) = match (output("stdout"), output("stderr")) { + (Ok(stdout), Ok(stderr)) => (stdout, stderr), + (Err(error), _) | (_, Err(error)) => { + return finish( + "failed", + json!({"error": format!("The job's output files could not be opened: {error}")}), + ); + } + }; + let cwd = record["cwd"].as_str().unwrap_or("/").to_owned(); + let mut command = std::process::Command::new(&argv[0]); + command + .args(&argv[1..]) + .current_dir(&cwd) + .stdin(std::process::Stdio::null()) + .stdout(stdout) + .stderr(stderr); + unsafe { + command.pre_exec(move || { + // Its own process group, so a cancel or timeout reaches everything it starts and + // spares this supervisor; and a command the person did not detach dies with it. + if libc::setpgid(0, 0) < 0 { + return Err(std::io::Error::last_os_error()); + } + if !detach { + libc::prctl(libc::PR_SET_PDEATHSIG, libc::SIGKILL); + } + Ok(()) + }); + } + let mut child = match command.spawn() { + Ok(child) => child, + Err(error) => { + return finish( + "failed", + json!({"error": format!("The command could not start: {error}")}), + ); + } + }; + let pid = child.id(); + let identity = |pid: u32| process_identity(pid).map(|(_, start)| start); + write_json( + &dir.join("process.json"), + &json!({ + "pid": pid, + "pgid": pid, + "start_time": identity(pid), + "supervisor_pid": std::process::id(), + "supervisor_start_time": identity(std::process::id()), + "started_at": now(), + }), + )?; + let timeout = Duration::from_secs( + record["timeout_seconds"] + .as_u64() + .unwrap_or(DEFAULT_TIMEOUT), + ); + let deadline = Instant::now() + timeout; + let mut stopped: Option<&str> = None; + let mut kill_at: Option = None; + let status = loop { + if let Some(status) = child.try_wait()? { + break status; + } + let stop = STOP_REQUESTED.load(std::sync::atomic::Ordering::SeqCst); + if stopped.is_none() && (stop || Instant::now() >= deadline) { + stopped = Some(if stop { "cancelled" } else { "timed_out" }); + unsafe { libc::kill(-(pid as i32), libc::SIGTERM) }; + kill_at = Some(Instant::now() + CANCEL_GRACE); + } + if kill_at.is_some_and(|at| Instant::now() >= at) { + unsafe { libc::kill(-(pid as i32), libc::SIGKILL) }; + kill_at = None; + } + std::thread::sleep(Duration::from_millis(100)); + }; + use std::os::unix::process::ExitStatusExt; + let cancelled = dir.join("cancel.json").exists(); + let state = if cancelled { + "cancelled" + } else if let Some(stopped) = stopped { + stopped + } else if status.success() { + "completed" + } else { + "failed" + }; + let mut extra = json!({"exit_code": status.code(), "signal": status.signal()}); + if state == "cancelled" && !cancelled { + extra["message"] = json!( + "Stopped because the Crew connection ended. Use detach for work that must outlive the connection." + ); + } else if state == "timed_out" { + extra["message"] = json!(format!( + "Stopped after its timeout of {} seconds.", + timeout.as_secs() + )); + } + finish(state, extra) +} + +/// Write a transfer's record files for this bridge process, which runs it on a thread. +fn transfer_process(dir: &Path) -> Result<()> { + let pid = std::process::id(); + write_json( + &dir.join("process.json"), + &json!({ + "pid": pid, + "start_time": process_identity(pid).map(|(_, start)| start), + "started_at": now(), + }), + ) +} + +/// One request on a transfer's own broker connection, answered or refused with the broker's +/// own code and sentence. +fn call( + stream: &mut UnixStream, + reader: &mut BufReader, + credential: &str, + sequence: &mut u64, + method: &str, + params: Value, +) -> Result { + *sequence += 1; + let request = json!({ + "version": crate::PROTOCOL_VERSION, + "id": format!("transfer-{sequence}"), + "method": method, + "params": params, + "credential": credential, + }); + let mut bytes = serde_json::to_vec(&request)?; + bytes.push(b'\n'); + stream.write_all(&bytes)?; + stream.flush()?; + let mut frame = Vec::new(); + let n = reader + .take((crate::MAX_FRAME + 1) as u64) + .read_until(b'\n', &mut frame)?; + ensure!( + n > 0 && n <= crate::MAX_FRAME && frame.ends_with(b"\n"), + "broker_unavailable: the workspace server closed the transfer connection" + ); + let response: crate::Response = serde_json::from_slice(&frame)?; + if let Some(error) = response.error { + bail!("{}", error.message); + } + response + .result + .ok_or_else(|| anyhow!("invalid_response: the workspace server sent no result")) +} + +fn progress(dir: &Path, bytes_done: u64, size: u64) { + let _ = write_json( + &dir.join("progress.json"), + &json!({"bytes_done": bytes_done, "size": size}), + ); +} + +fn cancel_requested(dir: &Path) -> bool { + dir.join("cancel.json").exists() +} + +/// Run `body` on a background thread for the transfer in `dir`, recording how it ended. +fn spawn_transfer(dir: PathBuf, body: impl FnOnce(&Path) -> Result + Send + 'static) { + std::thread::spawn(move || { + let outcome = body(&dir); + let exit = match outcome { + Ok(result) => json!({"state": "completed", "ended_at": now(), "result": result}), + Err(_) if cancel_requested(&dir) => json!({"state": "cancelled", "ended_at": now()}), + Err(error) => json!({"state": "failed", "ended_at": now(), "error": error.to_string()}), + }; + let _ = write_json(&dir.join("exit.json"), &exit); + }); +} + +fn start_transfer( + context: &Context, + scope: &Value, + key: &str, + kind: &str, + spec: Value, + mut record: Value, +) -> Result<(PathBuf, bool)> { + let owner = text(scope, "owner_id")?; + let id = job_id(owner, key); + record["job_id"] = json!(id); + record["kind"] = json!(kind); + record["owner_id"] = json!(owner); + record["run_id"] = scope["run_id"].clone(); + record["digest"] = json!(hex::encode(Sha256::digest(serde_json::to_vec(&spec)?))); + record["created_at"] = json!(now()); + record["detach"] = json!(false); + let (dir, created) = reserve(context, &id, &record)?; + if created { + transfer_process(&dir)?; + } + Ok((dir, created)) +} + +/// `remote.fetch_blob {blob_id, path, idempotency_key}`: copy a channel attachment to `path`, +/// in the background. The broker decides, chunk by chunk with the run's own credential, whether +/// this run may read the attachment; the copy lands only whole and only with the attachment's +/// SHA-256, and never replaces a file that is there. +fn fetch_blob( + context: &Context, + root: &Path, + params: &Value, + scope: &Value, + link: &BrokerLink, +) -> Result { + let blob_id = text(params, "blob_id")?; + ensure!( + !blob_id.is_empty() + && blob_id.len() <= 64 + && blob_id.bytes().all(|b| b.is_ascii_hexdigit() || b == b'-'), + "invalid_params: invalid blob_id" + ); + let key = idempotency_key(params)?; + let target = resolve_path(&context.home, root, text(params, "path")?)?; + let spec = json!({"kind": "fetch_blob", "blob_id": blob_id, "path": target}); + let owner = text(scope, "owner_id")?; + // A retry of a copy already made is answered with it, not refused for its finished file. + if !context.jobs.join(job_id(owner, key)).exists() { + refuse_protected_write(context, &target)?; + ensure!( + fs::symlink_metadata(&target).is_err(), + "conflict: {} already exists; choose a new path", + target.display() + ); + } + let record = json!({"path": target, "blob_id": blob_id}); + let (dir, created) = start_transfer(context, scope, key, "fetch_blob", spec, record)?; + if created { + let link = link.clone(); + let blob_id = blob_id.to_owned(); + spawn_transfer(dir.clone(), move |dir| { + run_fetch(dir, &link, &blob_id, &target) + }); + } + job_view(&dir, 0) +} + +fn run_fetch(dir: &Path, link: &BrokerLink, blob_id: &str, target: &Path) -> Result { + let (mut stream, mut reader) = (link.connect)()?; + let parent = target + .parent() + .ok_or_else(|| anyhow!("invalid_params: path needs a directory"))?; + fs::create_dir_all(parent).map_err(|e| anyhow!("remote path unavailable: {e}"))?; + let name = target + .file_name() + .ok_or_else(|| anyhow!("invalid_params: path needs a file name"))? + .to_string_lossy(); + let temporary = parent.join(format!( + ".{name}.crew-{}.part", + uuid::Uuid::new_v4().simple() + )); + let copied = (|| -> Result { + let mut file = OpenOptions::new() + .write(true) + .create_new(true) + .mode(0o600) + .custom_flags(libc::O_NOFOLLOW | libc::O_CLOEXEC) + .open(&temporary)?; + let mut hasher = Sha256::new(); + let mut sequence = 0u64; + let mut offset = 0u64; + let mut reported = 0u64; + let (size, sha256) = loop { + ensure!(!cancel_requested(dir), "cancelled: The copy was cancelled."); + let answer = call( + &mut stream, + &mut reader, + &link.credential, + &mut sequence, + "blob.read", + json!({"blob_id": blob_id, "offset": offset}), + )?; + let size = answer["blob"]["size"] + .as_u64() + .ok_or_else(|| anyhow!("invalid_response: attachment size missing"))?; + let sha256 = text(&answer["blob"], "sha256")?.to_ascii_lowercase(); + ensure!( + size <= MAX_TRANSFER, + "quota_exceeded: attachments over 1 GiB are not copied" + ); + ensure!( + answer["offset"].as_u64() == Some(offset), + "invalid_response: attachment chunk out of order" + ); + let bytes = hex::decode(text(&answer, "data_hex")?)?; + file.write_all(&bytes)?; + hasher.update(&bytes); + offset += bytes.len() as u64; + if offset - reported >= PROGRESS_EVERY || offset == size { + progress(dir, offset, size); + reported = offset; + } + if answer["complete"] == true || offset >= size { + break (size, sha256); + } + ensure!( + !bytes.is_empty(), + "invalid_response: empty attachment chunk" + ); + }; + ensure!( + offset == size && hex::encode(hasher.finalize()) == sha256, + "digest_mismatch: the copy does not match the attachment's SHA-256" + ); + file.sync_all()?; + drop(file); + rename_new(&temporary, target)?; + File::open(parent)?.sync_all()?; + Ok(json!({"path": target, "size": size, "sha256": sha256})) + })(); + if copied.is_err() { + let _ = fs::remove_file(&temporary); + } + copied +} + +/// Rename `from` to `to` only if `to` does not exist. +fn rename_new(from: &Path, to: &Path) -> Result<()> { + #[cfg(target_os = "linux")] + { + let source = std::ffi::CString::new(from.as_os_str().as_encoded_bytes())?; + let target = std::ffi::CString::new(to.as_os_str().as_encoded_bytes())?; + let renamed = unsafe { + libc::syscall( + libc::SYS_renameat2, + libc::AT_FDCWD, + source.as_ptr(), + libc::AT_FDCWD, + target.as_ptr(), + libc::RENAME_NOREPLACE, + ) + }; + if renamed == 0 { + return Ok(()); + } + let error = std::io::Error::last_os_error(); + if error.kind() == std::io::ErrorKind::AlreadyExists { + bail!( + "conflict: {} already exists; choose a new path", + to.display() + ); + } + // A filesystem without RENAME_NOREPLACE (some network filesystems) falls through. + if !matches!(error.raw_os_error(), Some(libc::EINVAL | libc::ENOSYS)) { + return Err(error.into()); + } + } + ensure!( + fs::symlink_metadata(to).is_err(), + "conflict: {} already exists; choose a new path", + to.display() + ); + fs::rename(from, to)?; + Ok(()) +} + +/// A media type for a file name's extension, for how the channel shows the file. +fn media_type_for(name: &str) -> &'static str { + let extension = name + .rsplit_once('.') + .map(|(_, extension)| extension.to_ascii_lowercase()) + .unwrap_or_default(); + match extension.as_str() { + "txt" | "log" => "text/plain", + "md" => "text/markdown", + "csv" => "text/csv", + "tsv" => "text/tab-separated-values", + "json" => "application/json", + "html" | "htm" => "text/html", + "pdf" => "application/pdf", + "png" => "image/png", + "jpg" | "jpeg" => "image/jpeg", + "gif" => "image/gif", + "svg" => "image/svg+xml", + "zip" => "application/zip", + "gz" => "application/gzip", + _ => "application/octet-stream", + } +} + +/// `remote.share_file {path, name?, media_type?, idempotency_key}`: upload a server file of at +/// most 1 GiB to the run's channel as an attachment, in the background. The job's result names +/// the attachment (`blob_id`), which the daemon then posts. Upload rules (quotas, the run's +/// channel, Restricted for a remote run) are the broker's, as for every worker upload. +fn share_file( + context: &Context, + root: &Path, + params: &Value, + scope: &Value, + link: &BrokerLink, +) -> Result { + let key = idempotency_key(params)?; + let path = resolve_path(&context.home, root, text(params, "path")?)?; + let channel = text(scope, "channel_id") + .map_err(|_| anyhow!("unsupported: This workspace server is too old to share server files; ask the host to update Crew."))? + .to_owned(); + let metadata = fs::metadata(&path).map_err(|e| anyhow!("remote path unavailable: {e}"))?; + ensure!(metadata.is_file(), "not a regular file"); + ensure!( + metadata.len() <= MAX_TRANSFER, + "quota_exceeded: maximum attachment is 1 GiB" + ); + let file_name = path + .file_name() + .map(|name| name.to_string_lossy().into_owned()) + .unwrap_or_default(); + let name = match params.get("name") { + None | Some(Value::Null) => file_name, + Some(Value::String(name)) => name.clone(), + Some(_) => bail!("invalid_params: name must be a string"), + }; + let media_type = match params.get("media_type") { + None | Some(Value::Null) => media_type_for(&name).to_owned(), + Some(Value::String(media_type)) => media_type.clone(), + Some(_) => bail!("invalid_params: media_type must be a string"), + }; + let spec = json!({"kind": "share_file", "path": path, "name": name, "media_type": media_type}); + let record = json!({"path": path, "name": name, "media_type": media_type}); + let (dir, created) = start_transfer(context, scope, key, "share_file", spec, record)?; + if created { + let link = link.clone(); + let job = dir + .file_name() + .and_then(|n| n.to_str()) + .unwrap_or_default() + .chars() + .take(16) + .collect::(); + spawn_transfer(dir.clone(), move |dir| { + run_share(dir, &link, &job, &channel, &path, &name, &media_type) + }); + } + job_view(&dir, 0) +} + +fn run_share( + dir: &Path, + link: &BrokerLink, + job: &str, + channel: &str, + path: &Path, + name: &str, + media_type: &str, +) -> Result { + let mut file = File::open(path).map_err(|e| anyhow!("remote path unavailable: {e}"))?; + let mut hasher = Sha256::new(); + let mut buffer = vec![0u8; TRANSFER_CHUNK]; + let mut size = 0u64; + loop { + ensure!( + !cancel_requested(dir), + "cancelled: The upload was cancelled." + ); + let n = file.read(&mut buffer)?; + if n == 0 { + break; + } + size += n as u64; + ensure!( + size <= MAX_TRANSFER, + "quota_exceeded: maximum attachment is 1 GiB" + ); + hasher.update(&buffer[..n]); + } + let sha256 = hex::encode(hasher.finalize()); + progress(dir, 0, size); + let (mut stream, mut reader) = (link.connect)()?; + let mut sequence = 0u64; + let blob = call( + &mut stream, + &mut reader, + &link.credential, + &mut sequence, + "blob.begin", + json!({"channel_id": channel, "size": size, "sha256": sha256, "name": name, "media_type": media_type, "idempotency_key": format!("share-{job}-begin")}), + )?; + let blob_id = text(&blob, "id")?.to_owned(); + file.seek(SeekFrom::Start(0))?; + let mut offset = 0u64; + let mut reported = 0u64; + while offset < size { + ensure!( + !cancel_requested(dir), + "cancelled: The upload was cancelled." + ); + let want = TRANSFER_CHUNK.min((size - offset) as usize); + file.read_exact(&mut buffer[..want])?; + call( + &mut stream, + &mut reader, + &link.credential, + &mut sequence, + "blob.chunk", + json!({"blob_id": blob_id, "offset": offset, "data_hex": hex::encode(&buffer[..want]), "idempotency_key": format!("share-{job}-{offset}")}), + )?; + offset += want as u64; + if offset - reported >= PROGRESS_EVERY || offset == size { + progress(dir, offset, size); + reported = offset; + } + } + call( + &mut stream, + &mut reader, + &link.credential, + &mut sequence, + "blob.finish", + json!({"blob_id": blob_id, "idempotency_key": format!("share-{job}-finish")}), + )?; + Ok( + json!({"blob_id": blob_id, "name": name, "size": size, "sha256": sha256, "media_type": media_type}), + ) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn temp(label: &str) -> PathBuf { + let path = std::env::temp_dir().join(format!( + "crew-full-{label}-{}-{}", + std::process::id(), + uuid::Uuid::new_v4().simple() + )); + fs::create_dir_all(&path).unwrap(); + fs::canonicalize(&path).unwrap() + } + + fn context(home: &Path) -> Context { + Context { + home: home.to_path_buf(), + jobs: home.join(".local/state/biorouter-crew/jobs"), + executable: home.join(".local/bin/biorouter-crew"), + } + } + + #[test] + fn paths_resolve_from_home_absolute_or_the_work_folder() { + let home = Path::new("/home/bob"); + let root = Path::new("/home/bob/crew-work/lab"); + assert_eq!( + resolve_path(home, root, "~").unwrap(), + PathBuf::from("/home/bob") + ); + assert_eq!( + resolve_path(home, root, "~/data/x.csv").unwrap(), + PathBuf::from("/home/bob/data/x.csv") + ); + assert_eq!( + resolve_path(home, root, "/pool1/data/datasets").unwrap(), + PathBuf::from("/pool1/data/datasets") + ); + assert_eq!( + resolve_path(home, root, "results/out.txt").unwrap(), + PathBuf::from("/home/bob/crew-work/lab/results/out.txt") + ); + assert_eq!(resolve_path(home, root, ".").unwrap(), root.join(".")); + assert!(resolve_path(home, root, "~alice/x").is_err()); + assert!(resolve_path(home, root, "a\0b").is_err()); + } + + #[test] + fn job_ids_are_keyed_by_owner_not_run() { + let one = job_id("owner-a", "key-1"); + assert_eq!(one, job_id("owner-a", "key-1")); + assert_ne!(one, job_id("owner-b", "key-1")); + assert_ne!(one, job_id("owner-a", "key-2")); + assert_eq!(one.len(), 64); + assert_eq!(one, hex::encode(Sha256::digest(b"owner-a:key-1"))); + } + + #[test] + fn root_never_gets_full_access() { + assert!(refuse_root(0) + .unwrap_err() + .to_string() + .starts_with("forbidden:")); + refuse_root(1000).unwrap(); + } + + #[test] + fn write_modes_truncate_append_and_create_new() { + let home = temp("write"); + let context = context(&home); + let file = home.join("notes/a.txt"); + let write_with = |params: Value| write(&context, &file, ¶ms); + assert!(write_with(json!({"text": "x"})).is_err(), "no parents yet"); + let answer = write_with(json!({"text": "one\n", "create_parents": true})).unwrap(); + assert_eq!(answer["file_size"], 4); + write_with(json!({"text": "two\n", "mode": "append"})).unwrap(); + assert_eq!(fs::read_to_string(&file).unwrap(), "one\ntwo\n"); + write_with(json!({"data_hex": hex::encode("new\n")})).unwrap(); + assert_eq!(fs::read_to_string(&file).unwrap(), "new\n"); + let refused = write_with(json!({"text": "again", "mode": "create_new"})).unwrap_err(); + assert!(refused.to_string().starts_with("conflict:"), "{refused}"); + assert!(write_with(json!({"text": "a", "data_hex": "61"})).is_err()); + assert!(write_with(json!({"text": "a", "mode": "sideways"})).is_err()); + let large = "a".repeat(MAX_CHUNK_BYTES + 1); + assert!(write_with(json!({"text": large})).is_err()); + let _ = fs::remove_dir_all(&home); + } + + /// The bridge executable and `authorized_keys` are refused however they are reached: by + /// name, through `..`, or through a symlink. + #[test] + fn writes_to_the_bridge_and_authorized_keys_are_refused() { + let home = temp("protect"); + let context = context(&home); + fs::create_dir_all(home.join(".ssh")).unwrap(); + fs::create_dir_all(home.join(".local/bin")).unwrap(); + fs::write(&context.executable, b"bridge").unwrap(); + for target in [ + home.join(".ssh/authorized_keys"), + home.join(".ssh/authorized_keys2"), + home.join(".ssh/../.ssh/authorized_keys"), + context.executable.clone(), + ] { + let error = write(&context, &target, &json!({"text": "x"})).unwrap_err(); + assert!( + error.to_string().starts_with("forbidden:"), + "{}: {error}", + target.display() + ); + } + std::os::unix::fs::symlink(&context.executable, home.join("link-to-bridge")).unwrap(); + assert!(write( + &context, + &home.join("link-to-bridge"), + &json!({"text": "x"}) + ) + .is_err()); + std::os::unix::fs::symlink(home.join("nowhere"), home.join("dangling")).unwrap(); + assert!(write(&context, &home.join("dangling"), &json!({"text": "x"})).is_err()); + assert_eq!(fs::read(&context.executable).unwrap(), b"bridge"); + // Other files in the same folders are ordinary. + write( + &context, + &home.join(".ssh/config-notes"), + &json!({"text": "x"}), + ) + .unwrap(); + let _ = fs::remove_dir_all(&home); + } + + #[test] + fn reads_take_offsets_and_stay_within_a_frame() { + let home = temp("read"); + let file = home.join("data.bin"); + fs::write(&file, b"hello world").unwrap(); + let answer = read(&file, &json!({"offset": 6, "length": 5})).unwrap(); + assert_eq!(answer["text_utf8"], "world"); + assert_eq!(answer["eof"], true); + assert_eq!(answer["file_size"], 11); + assert!(read(&file, &json!({"offset": 12})).is_err()); + assert!(read(&file, &json!({"length": MAX_CHUNK_BYTES + 1})).is_err()); + let big = home.join("big.txt"); + fs::write(&big, "\u{1}".repeat(MAX_CHUNK_BYTES + 10)).unwrap(); + let answer = read(&big, &json!({})).unwrap(); + assert_eq!(answer["size"], MAX_CHUNK_BYTES); + assert_eq!(answer["eof"], false); + assert!( + answer.get("text_utf8").is_none(), + "escaped text would overflow the frame" + ); + assert!(serde_json::to_vec(&answer).unwrap().len() < crate::MAX_FRAME); + let _ = fs::remove_dir_all(&home); + } + + #[test] + fn listing_and_stat_follow_symlinks_and_report_missing_paths() { + let home = temp("list"); + fs::write(home.join("a.txt"), b"a").unwrap(); + fs::create_dir(home.join("sub")).unwrap(); + std::os::unix::fs::symlink(home.join("sub"), home.join("link")).unwrap(); + let listing = list(&home, &json!({})).unwrap(); + let names: Vec<&str> = listing["entries"] + .as_array() + .unwrap() + .iter() + .map(|e| e["name"].as_str().unwrap()) + .collect(); + assert_eq!(names, vec!["a.txt", "link", "sub"]); + assert_eq!(listing["entries"][1]["target_type"], "directory"); + assert_eq!( + list(&home, &json!({"limit": 1})).unwrap()["truncated"], + true + ); + assert_eq!(stat(&home.join("link")).unwrap()["type"], "directory"); + assert_eq!(stat(&home.join("missing")).unwrap()["exists"], false); + assert_eq!( + hash(&home.join("a.txt")).unwrap()["sha256"], + hex::encode(Sha256::digest(b"a")) + ); + let _ = fs::remove_dir_all(&home); + } + + #[test] + fn argv_runs_through_the_login_shell_exactly() { + let argv = vec![ + "python3".to_owned(), + "-c".to_owned(), + "print('a b')".to_owned(), + ]; + assert_eq!( + job_argv(Path::new("/bin/bash"), None, Some(&argv)), + vec![ + "/bin/bash", + "-lc", + "exec \"$@\"", + "biorouter-crew-job", + "python3", + "-c", + "print('a b')" + ] + ); + assert_eq!( + job_argv(Path::new("/usr/bin/fish"), None, Some(&argv))[0], + "/bin/sh" + ); + assert_eq!( + job_argv(Path::new("/bin/zsh"), Some("echo hi"), None), + vec!["/bin/zsh", "-lc", "echo hi"] + ); + assert_eq!( + job_argv(Path::new("/bin/tcsh"), Some("echo hi"), None)[1], + "-c" + ); + } + + /// A folder-mode scope (or one from an older broker, with no `access`) never reaches the + /// full-access code, and a public run never does either. + #[test] + fn full_access_needs_a_full_scope() { + let home = temp("scope"); + let context = context(&home); + let link = BrokerLink { + connect: Arc::new(|| bail!("no broker in this test")), + credential: "credential".into(), + }; + let params = json!({"command": "true", "idempotency_key": "k"}); + for scope in [ + json!({"root": home, "public_provider": false, "owner_id": "o", "allow_execute": true, "expires_at": u64::MAX}), + json!({"root": home, "public_provider": false, "owner_id": "o", "allow_execute": true, "expires_at": u64::MAX, "access": "folder"}), + json!({"root": home, "public_provider": true, "owner_id": "o", "allow_execute": true, "expires_at": u64::MAX, "access": "full"}), + json!({"root": home, "public_provider": false, "allow_execute": true, "expires_at": u64::MAX, "access": "full"}), + ] { + let error = handle_in(&context, "remote.shell", ¶ms, &scope, &link).unwrap_err(); + assert!( + ["invalid_scope:", "forbidden:"] + .iter() + .any(|code| error.to_string().starts_with(code)), + "{scope}: {error}" + ); + } + assert!(!context.jobs.exists(), "nothing was recorded"); + let _ = fs::remove_dir_all(&home); + } + + #[test] + fn job_records_are_owner_only_and_keyed_by_digest() { + let home = temp("reserve"); + let context = context(&home); + let record = json!({"job_id": "x", "owner_id": "o", "digest": "d", "kind": "shell", "created_at": now()}); + let id = job_id("o", "k"); + let (dir, created) = reserve(&context, &id, &record).unwrap(); + assert!(created); + assert_eq!(fs::metadata(&context.jobs).unwrap().mode() & 0o777, 0o700); + assert_eq!(fs::metadata(&dir).unwrap().mode() & 0o777, 0o700); + assert_eq!( + fs::metadata(dir.join("job.json")).unwrap().mode() & 0o777, + 0o600 + ); + let (again, created) = reserve(&context, &id, &record).unwrap(); + assert!(!created); + assert_eq!(again, dir); + let mut other = record.clone(); + other["digest"] = json!("e"); + assert!(reserve(&context, &id, &other) + .unwrap_err() + .to_string() + .starts_with("conflict:")); + let mut stranger = record; + stranger["owner_id"] = json!("p"); + assert!(reserve(&context, &id, &stranger).is_err()); + let found = job_dir(&context, &json!({"job_id": id}), "o").unwrap(); + assert_eq!(found, dir); + assert!(job_dir(&context, &json!({"job_id": id}), "p").is_err()); + assert!( + job_dir(&context, &json!({"job_id": job_id("o", "none")}), "o") + .unwrap_err() + .to_string() + .starts_with("not_found:") + ); + let _ = fs::remove_dir_all(&home); + } + + /// Eight running jobs refuse a ninth; at the record cap the oldest finished one goes. + #[test] + fn running_jobs_and_records_are_capped() { + let home = temp("caps"); + let context = context(&home); + let jobs = jobs_directory(&context).unwrap().to_path_buf(); + let make = |index: usize, finished: bool| { + let dir = jobs.join(job_id("o", &format!("k{index}"))); + fs::create_dir(&dir).unwrap(); + write_json( + &dir.join("job.json"), + &json!({"owner_id": "o", "kind": "shell", "created_at": now()}), + ) + .unwrap(); + if finished { + write_json( + &dir.join("exit.json"), + &json!({"state": "completed", "ended_at": index}), + ) + .unwrap(); + } + dir + }; + for index in 0..MAX_RUNNING { + make(index, false); + } + assert!(make_room(&jobs) + .unwrap_err() + .to_string() + .starts_with("quota_exceeded:")); + for dir in fs::read_dir(&jobs).unwrap() { + fs::remove_dir_all(dir.unwrap().path()).unwrap(); + } + let oldest = make(0, true); + for index in 1..MAX_RECORDS { + make(index, true); + } + make_room(&jobs).unwrap(); + assert!(!oldest.exists(), "the oldest finished record made room"); + assert_eq!(fs::read_dir(&jobs).unwrap().count(), MAX_RECORDS - 1); + let _ = fs::remove_dir_all(&home); + } + + #[test] + fn media_types_follow_the_extension() { + assert_eq!(media_type_for("result.CSV"), "text/csv"); + assert_eq!(media_type_for("plot.png"), "image/png"); + assert_eq!(media_type_for("model"), "application/octet-stream"); + } +} diff --git a/crates/biorouter-crew/tests/full_access_bridge.rs b/crates/biorouter-crew/tests/full_access_bridge.rs new file mode 100644 index 000000000..693c88bcd --- /dev/null +++ b/crates/biorouter-crew/tests/full_access_bridge.rs @@ -0,0 +1,299 @@ +//! End to end through the real binaries (design OW-6, OW-7): a `biorouter-crew serve` broker, +//! a person's signed requests on its socket, and the member's `biorouter-crew bridge` running a +//! full-access command and copying an attachment to the server and back. +//! +//! Linux only, and only as an ordinary account: the broker refuses to serve as root and full +//! access refuses to run as root. As root (a container's default) the test says so and returns; +//! run its binary as an ordinary user to exercise it there. +#![cfg(target_os = "linux")] + +use biorouter_crew::signing_payload; +use ed25519_dalek::{Signer, SigningKey}; +use serde_json::{json, Value}; +use sha2::{Digest, Sha256}; +use std::fs; +use std::io::{BufRead, BufReader, Write}; +use std::os::unix::fs::PermissionsExt; +use std::os::unix::net::UnixStream; +use std::path::{Path, PathBuf}; +use std::process::{Child, ChildStdin, ChildStdout, Command, Stdio}; +use std::time::{Duration, Instant}; + +const BINARY: &str = env!("CARGO_BIN_EXE_biorouter-crew"); + +fn private_dir(label: &str) -> PathBuf { + let path = std::env::temp_dir().join(format!( + "crew-e2e-{label}-{}-{}", + std::process::id(), + uuid::Uuid::new_v4().simple() + )); + fs::create_dir(&path).unwrap(); + fs::set_permissions(&path, fs::Permissions::from_mode(0o700)).unwrap(); + path +} + +/// The broker process, stopped on drop. +struct Served { + child: Child, + socket: PathBuf, + workspace_id: String, +} + +impl Drop for Served { + fn drop(&mut self) { + let _ = self.child.kill(); + let _ = self.child.wait(); + } +} + +fn serve(state: &Path, bootstrap_key: &str) -> Served { + let mut child = Command::new(BINARY) + .args(["serve", "--state-dir"]) + .arg(state) + .args(["--bootstrap-key", bootstrap_key]) + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::inherit()) + .spawn() + .unwrap(); + let mut line = String::new(); + BufReader::new(child.stdout.take().unwrap()) + .read_line(&mut line) + .unwrap(); + let info: Value = serde_json::from_str(&line).unwrap_or_else(|_| panic!("serve said {line}")); + Served { + child, + socket: PathBuf::from(info["socket"].as_str().unwrap()), + workspace_id: info["workspace_id"].as_str().unwrap().to_owned(), + } +} + +/// A person's signed device on one connection to the broker. +struct Person { + stream: UnixStream, + reader: BufReader, + key: SigningKey, + workspace_id: String, + uid: u32, + sequence: usize, +} + +impl Person { + fn frame(&mut self, request: Value) -> Value { + self.stream + .write_all(format!("{request}\n").as_bytes()) + .unwrap(); + let mut line = String::new(); + self.reader.read_line(&mut line).unwrap(); + serde_json::from_str(&line).unwrap() + } + + fn signed(&mut self, method: &str, mut params: Value) -> Value { + self.sequence += 1; + if method != "auth.bootstrap" { + params["idempotency_key"] = json!(format!("e2e-{}", self.sequence)); + } + let device_id = hex::encode(Sha256::digest(self.key.verifying_key().to_bytes())); + let challenge = self.frame(json!({ + "version": 1, "id": format!("challenge-{}", self.sequence), "method": "auth.challenge", + "params": {"device_id": device_id}, + })); + let nonce = challenge["result"]["nonce"].as_str().unwrap().to_owned(); + let signature = self.key.sign(&signing_payload( + &self.workspace_id, + self.uid, + &nonce, + method, + ¶ms, + )); + let answer = self.frame(json!({ + "version": 1, "id": format!("request-{}", self.sequence), "method": method, + "params": params, + "auth": {"device_id": device_id, "nonce": nonce, "signature": hex::encode(signature.to_bytes())}, + })); + assert!(answer["error"].is_null(), "{method}: {answer}"); + answer["result"].clone() + } +} + +/// The member's bridge, spoken to as the daemon does: one JSON line each way. +struct Bridge { + child: Child, + input: ChildStdin, + output: BufReader, + credential: String, + sequence: usize, +} + +impl Bridge { + fn ask(&mut self, method: &str, params: Value) -> Value { + self.sequence += 1; + let request = json!({ + "version": 1, "id": format!("worker-{}", self.sequence), "method": method, + "params": params, "credential": self.credential, + }); + self.input + .write_all(format!("{request}\n").as_bytes()) + .unwrap(); + let mut line = String::new(); + self.output.read_line(&mut line).unwrap(); + serde_json::from_str(&line).unwrap() + } + + fn ok(&mut self, method: &str, params: Value) -> Value { + let answer = self.ask(method, params); + assert!(answer["error"].is_null(), "{method}: {answer}"); + answer["result"].clone() + } + + fn finished(&mut self, job_id: &str) -> Value { + let deadline = Instant::now() + Duration::from_secs(60); + loop { + let status = self.ok("remote.job_status", json!({"job_id": job_id})); + if !matches!(status["state"].as_str(), Some("starting" | "running")) { + return status; + } + assert!(Instant::now() < deadline, "job never finished: {status}"); + std::thread::sleep(Duration::from_millis(200)); + } + } +} + +impl Drop for Bridge { + fn drop(&mut self) { + let _ = self.child.kill(); + let _ = self.child.wait(); + } +} + +#[test] +fn a_full_access_run_runs_commands_and_moves_files_through_the_real_bridge() { + let uid = unsafe { libc::geteuid() }; + if uid == 0 { + eprintln!("skipped: the broker and full access refuse root; run this binary as an ordinary user"); + return; + } + let state = private_dir("state"); + let home = private_dir("home"); + let work = home.join("crew-work/lab"); + fs::create_dir_all(&work).unwrap(); + let key = SigningKey::from_bytes(&[42; 32]); + let served = serve(&state, &hex::encode(key.verifying_key().to_bytes())); + + let stream = UnixStream::connect(&served.socket).unwrap(); + let mut person = Person { + reader: BufReader::new(stream.try_clone().unwrap()), + stream, + key, + workspace_id: served.workspace_id.clone(), + uid, + sequence: 0, + }; + let public_key = hex::encode(person.key.verifying_key().to_bytes()); + person.signed("auth.bootstrap", json!({"public_key": public_key})); + person.signed( + "policy.set", + json!({"mode": "private", "institution_id": "ucsf"}), + ); + let team = person.signed("team.create", json!({"name": "lab"})); + let channel = team["channel"]["id"].as_str().unwrap().to_owned(); + + // A person shares an attachment in the channel. + let attachment: Vec = (0..600_000usize).map(|i| (i * 7 % 253) as u8).collect(); + let sha256 = hex::encode(Sha256::digest(&attachment)); + let blob = person.signed( + "blob.begin", + json!({"channel_id": channel, "size": attachment.len(), "sha256": sha256, "name": "data.bin", "media_type": "application/octet-stream"}), + ); + let blob_id = blob["id"].as_str().unwrap().to_owned(); + for (index, chunk) in attachment.chunks(262_144).enumerate() { + person.signed( + "blob.chunk", + json!({"blob_id": blob_id, "offset": index * 262_144, "data_hex": hex::encode(chunk)}), + ); + } + person.signed("blob.finish", json!({"blob_id": blob_id})); + + let snapshot = person.signed("workspace.snapshot", json!({})); + let epoch = snapshot["workspace"]["policy_epoch"].clone(); + let run = person.signed( + "run.create", + json!({ + "channel_id": channel, "source_channels": [channel], "provider_policy_id": "private-test", + "personal_mode": "private", "public_provider": false, "expires_in": 600, + "expected_workspace_policy_epoch": epoch, "workspace_institution_id": "ucsf", + "connection_institution_id": "ucsf", "provider_affiliation": {"kind": "local"}, + "expected_protected_context": true, "remote_root": work, "remote_execution": true, + "remote_access": "full", + }), + ); + assert_eq!(run["run"]["remote_access"], "full"); + + let mut child = Command::new(BINARY) + .args(["bridge", "--stdio", "--socket"]) + .arg(&served.socket) + .args(["--owner-uid", &uid.to_string(), "--workspace-id", &served.workspace_id]) + .env("HOME", &home) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::inherit()) + .spawn() + .unwrap(); + let mut bridge = Bridge { + input: child.stdin.take().unwrap(), + output: BufReader::new(child.stdout.take().unwrap()), + child, + credential: run["credential"].as_str().unwrap().to_owned(), + sequence: 0, + }; + + let hello = bridge.ok("bridge.hello", json!({})); + assert_eq!(hello["uid"], uid); + assert!(hello["capabilities"] + .as_array() + .unwrap() + .contains(&json!("remote_full_v1"))); + + // A command with the network and a child process, which folder mode could never run. + let shell = bridge.ok( + "remote.shell", + json!({"command": "echo hi; python3 -c 'import socket; socket.socket(); print(\"net\")'; sh -c 'id -u'", "idempotency_key": "shell", "wait_seconds": 20}), + ); + assert_eq!(shell["state"], "completed", "{shell}"); + assert_eq!(shell["stdout_tail"], format!("hi\nnet\n{uid}\n")); + + // The attachment comes to the server whole and checked, and goes back as a new one. + let fetch = bridge.ok( + "remote.fetch_blob", + json!({"blob_id": blob_id, "path": "inbox/data.bin", "idempotency_key": "fetch"}), + ); + let fetched = bridge.finished(fetch["job_id"].as_str().unwrap()); + assert_eq!(fetched["state"], "completed", "{fetched}"); + assert_eq!(fs::read(work.join("inbox/data.bin")).unwrap(), attachment); + let share = bridge.ok( + "remote.share_file", + json!({"path": "inbox/data.bin", "name": "copy.bin", "idempotency_key": "share"}), + ); + let shared = bridge.finished(share["job_id"].as_str().unwrap()); + assert_eq!(shared["state"], "completed", "{shared}"); + assert_eq!(shared["size"], attachment.len()); + let copy = person.signed( + "blob.status", + json!({"blob_id": shared["blob_id"]}), + ); + assert_eq!(copy["complete"], true); + assert_eq!(copy["sha256"], sha256); + assert_eq!(copy["channel_id"], channel.as_str()); + + // Revoking the run ends what its credential can do through the bridge. + person.signed("run.revoke", json!({"run_id": run["run"]["id"]})); + let refused = bridge.ask( + "remote.shell", + json!({"command": "true", "idempotency_key": "after", "wait_seconds": 1}), + ); + assert_eq!(refused["error"]["code"], "remote_operation_denied", "{refused}"); + drop(bridge); + drop(served); + let _ = fs::remove_dir_all(&state); + let _ = fs::remove_dir_all(&home); +} diff --git a/crates/biorouter-crew/tests/full_access_contract.rs b/crates/biorouter-crew/tests/full_access_contract.rs new file mode 100644 index 000000000..5fc6adc80 --- /dev/null +++ b/crates/biorouter-crew/tests/full_access_contract.rs @@ -0,0 +1,709 @@ +//! Full-access jobs and transfers in the member's bridge (design OW-6, OW-7), driven through +//! `remote_full::handle_in` with a private home and the real `biorouter-crew` binary as the +//! supervisor. Linux only: jobs need `/proc`, `prctl` and process groups. +#![cfg(target_os = "linux")] + +use biorouter_crew::remote_full::{handle_in, job_id, BrokerLink, Context}; +use serde_json::{json, Value}; +use sha2::{Digest, Sha256}; +use std::collections::HashMap; +use std::fs; +use std::io::{BufRead, BufReader, Write}; +use std::os::unix::fs::PermissionsExt; +use std::os::unix::net::UnixStream; +use std::path::PathBuf; +use std::sync::{Arc, Mutex}; +use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; + +fn now() -> u64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_secs() +} + +/// A private home with a work folder, removed on drop. +struct Home { + path: PathBuf, + context: Context, +} + +impl Home { + fn new(label: &str) -> Self { + let path = std::env::temp_dir().join(format!( + "crew-full-contract-{label}-{}-{}", + std::process::id(), + uuid::Uuid::new_v4().simple() + )); + fs::create_dir_all(path.join("crew-work/lab")).unwrap(); + let path = fs::canonicalize(path).unwrap(); + let context = Context { + home: path.clone(), + jobs: path.join(".local/state/biorouter-crew/jobs"), + executable: PathBuf::from(env!("CARGO_BIN_EXE_biorouter-crew")), + }; + Self { path, context } + } + + fn work(&self) -> PathBuf { + self.path.join("crew-work/lab") + } + + fn scope(&self, owner: &str, run: &str) -> Value { + json!({ + "root": self.work(), + "allow_execute": true, + "public_provider": false, + "run_id": run, + "policy_epoch": 1, + "expires_at": now() + 600, + "access": "full", + "owner_id": owner, + "channel_id": "channel-1", + }) + } + + fn call(&self, method: &str, params: Value) -> anyhow::Result { + self.call_as("owner-1", "run-1", method, params, &no_broker()) + } + + fn call_as( + &self, + owner: &str, + run: &str, + method: &str, + params: Value, + link: &BrokerLink, + ) -> anyhow::Result { + handle_in( + &self.context, + method, + ¶ms, + &self.scope(owner, run), + link, + ) + } + + /// Poll `job_id` until it is no longer starting or running. + fn finished(&self, job_id: &str) -> Value { + let deadline = Instant::now() + Duration::from_secs(30); + loop { + let status = self + .call("remote.job_status", json!({"job_id": job_id})) + .unwrap(); + let state = status["state"].as_str().unwrap().to_owned(); + if !matches!(state.as_str(), "starting" | "running") { + return status; + } + assert!(Instant::now() < deadline, "job never finished: {status}"); + std::thread::sleep(Duration::from_millis(100)); + } + } +} + +impl Drop for Home { + fn drop(&mut self) { + let _ = fs::remove_dir_all(&self.path); + } +} + +fn no_broker() -> BrokerLink { + BrokerLink { + connect: Arc::new(|| anyhow::bail!("no broker in this test")), + credential: "credential".into(), + } +} + +fn shell(command: &str, key: &str, extra: Value) -> Value { + let mut params = json!({"command": command, "idempotency_key": key, "wait_seconds": 20}); + for (name, value) in extra.as_object().unwrap() { + params[name] = value.clone(); + } + params +} + +/// Whether a process with this PID exists and is not a zombie. +fn running(pid: u64) -> bool { + fs::read_to_string(format!("/proc/{pid}/stat")) + .ok() + .and_then(|stat| { + let rest = stat.get(stat.rfind(')')? + 1..)?.trim_start().to_owned(); + rest.chars().next() + }) + .is_some_and(|state| state != 'Z' && state != 'X') +} + +fn process(home: &Home, job_id: &str) -> Value { + serde_json::from_slice(&fs::read(home.context.jobs.join(job_id).join("process.json")).unwrap()) + .unwrap() +} + +/// A command runs as the account through its login shell, with network sockets and child +/// processes (none of which a confined command may have), in the work folder, and its output +/// lands in owner-only files. +#[test] +fn a_command_runs_as_the_account_with_network_and_children() { + let home = Home::new("shell"); + let answer = home + .call( + "remote.shell", + shell( + "echo hi; echo oops >&2; pwd; sh -c 'echo child'; echo \"job=$BIOROUTER_CREW_JOB_ID\"; \ + python3 -c 'import socket; socket.socket(); print(\"socket ok\")'", + "k1", + json!({}), + ), + ) + .unwrap(); + assert_eq!(answer["state"], "completed", "{answer}"); + assert_eq!(answer["exit_code"], 0); + let stdout = answer["stdout_tail"].as_str().unwrap(); + assert!(stdout.contains("hi\n"), "{stdout}"); + assert!(stdout.contains(home.work().to_str().unwrap()), "{stdout}"); + assert!(stdout.contains("child\n"), "{stdout}"); + assert!(stdout.contains("socket ok"), "{stdout}"); + let id = job_id("owner-1", "k1"); + assert_eq!(answer["job_id"], id.as_str()); + assert!(stdout.contains(&format!("job={id}")), "{stdout}"); + // A login shell's startup files may add to standard error; the command's own line is there. + assert!(answer["stderr_tail"].as_str().unwrap().contains("oops\n"), "{answer}"); + assert_eq!(answer["truncated"], false); + let dir = home.context.jobs.join(&id); + assert_eq!( + fs::metadata(&dir).unwrap().permissions().mode() & 0o777, + 0o700 + ); + for name in ["stdout", "stderr", "job.json", "exit.json", "process.json"] { + let mode = fs::metadata(dir.join(name)).unwrap().permissions().mode() & 0o777; + assert_eq!(mode, 0o600, "{name}"); + } +} + +#[test] +fn a_failing_command_reports_its_exit_code() { + let home = Home::new("fail"); + let answer = home + .call("remote.shell", shell("echo before; exit 3", "k", json!({}))) + .unwrap(); + assert_eq!(answer["state"], "failed"); + assert_eq!(answer["exit_code"], 3); + assert_eq!(answer["stdout_tail"], "before\n"); +} + +/// An argv reaches the program exactly, through the login shell, without being reparsed. +#[test] +fn an_argv_runs_exactly() { + let home = Home::new("argv"); + let answer = home + .call( + "remote.execute", + json!({"argv": ["printf", "%s|", "a b", "$HOME", "c;d"], "idempotency_key": "k", "wait_seconds": 20}), + ) + .unwrap(); + assert_eq!(answer["state"], "completed", "{answer}"); + assert_eq!(answer["stdout_tail"], "a b|$HOME|c;d|"); +} + +/// `cwd` may be `~/`-relative, absolute or work-folder-relative; a missing one is refused +/// before anything runs. +#[test] +fn the_working_directory_resolves_like_every_path() { + let home = Home::new("cwd"); + fs::create_dir(home.work().join("sub")).unwrap(); + for (cwd, expected) in [ + ("~", home.path.clone()), + ("sub", home.work().join("sub")), + (home.path.to_str().unwrap(), home.path.clone()), + ] { + let answer = home + .call("remote.shell", shell("pwd", cwd, json!({"cwd": cwd}))) + .unwrap(); + assert_eq!( + answer["stdout_tail"].as_str().unwrap().trim(), + expected.to_str().unwrap(), + "{cwd}" + ); + } + let error = home + .call( + "remote.shell", + shell("pwd", "missing", json!({"cwd": "nowhere"})), + ) + .unwrap_err(); + assert!(error.to_string().starts_with("invalid_params:"), "{error}"); +} + +/// A job is the owner's, not the run's: a later run of the same person reads it, and another +/// person cannot. +#[test] +fn any_run_of_the_same_owner_reads_a_job_after_it_exits() { + let home = Home::new("status"); + let started = home + .call( + "remote.shell", + shell("sleep 1; echo done", "k", json!({"wait_seconds": 0})), + ) + .unwrap(); + assert!( + matches!(started["state"].as_str(), Some("starting" | "running")), + "{started}" + ); + let id = started["job_id"].as_str().unwrap().to_owned(); + let finished = home.finished(&id); + assert_eq!(finished["state"], "completed"); + let later = home + .call_as( + "owner-1", + "run-2", + "remote.job_status", + json!({"job_id": id, "tail_bytes": 2}), + &no_broker(), + ) + .unwrap(); + assert_eq!(later["stdout_tail"], "e\n"); + assert_eq!(later["truncated"], true); + let stranger = home + .call_as( + "owner-2", + "run-3", + "remote.job_status", + json!({"job_id": id}), + &no_broker(), + ) + .unwrap_err(); + assert!(stranger.to_string().starts_with("forbidden:"), "{stranger}"); + let unknown = home + .call( + "remote.job_status", + json!({"job_id": job_id("owner-1", "nope")}), + ) + .unwrap_err(); + assert!(unknown.to_string().starts_with("not_found:"), "{unknown}"); +} + +/// A retried request is the same job, run once; the same key for a different command is refused. +#[test] +fn an_idempotency_key_names_one_job() { + let home = Home::new("idempotency"); + let command = "echo x >> counter"; + let first = home + .call("remote.shell", shell(command, "k", json!({}))) + .unwrap(); + let second = home + .call("remote.shell", shell(command, "k", json!({}))) + .unwrap(); + assert_eq!(first["job_id"], second["job_id"]); + assert_eq!( + fs::read_to_string(home.work().join("counter")).unwrap(), + "x\n" + ); + let conflict = home + .call("remote.shell", shell("echo y", "k", json!({}))) + .unwrap_err(); + assert!(conflict.to_string().starts_with("conflict:"), "{conflict}"); +} + +/// Cancel stops the whole process group, children included, and only that. +#[test] +fn cancel_stops_the_jobs_process_group() { + let home = Home::new("cancel"); + let started = home + .call( + "remote.shell", + shell( + "sleep 60 & echo $! > child.pid; sleep 60", + "k", + json!({"wait_seconds": 1}), + ), + ) + .unwrap(); + assert_eq!(started["state"], "running", "{started}"); + let id = started["job_id"].as_str().unwrap().to_owned(); + let leader = process(&home, &id)["pid"].as_u64().unwrap(); + let deadline = Instant::now() + Duration::from_secs(5); + while !home.work().join("child.pid").exists() { + assert!(Instant::now() < deadline); + std::thread::sleep(Duration::from_millis(50)); + } + std::thread::sleep(Duration::from_millis(100)); + let child: u64 = fs::read_to_string(home.work().join("child.pid")) + .unwrap() + .trim() + .parse() + .unwrap(); + assert!(running(leader) && running(child)); + let cancelled = home.call("remote.cancel", json!({"job_id": id})).unwrap(); + assert_eq!(cancelled["state"], "cancelled", "{cancelled}"); + let deadline = Instant::now() + Duration::from_secs(5); + while running(child) || running(leader) { + assert!(Instant::now() < deadline, "the group outlived its cancel"); + std::thread::sleep(Duration::from_millis(50)); + } + // Cancelling a finished job changes nothing. + let again = home.call("remote.cancel", json!({"job_id": id})).unwrap(); + assert_eq!(again["state"], "cancelled"); +} + +#[test] +fn a_command_past_its_timeout_is_stopped() { + let home = Home::new("timeout"); + let started = Instant::now(); + let answer = home + .call( + "remote.shell", + shell("sleep 30", "k", json!({"timeout_seconds": 1})), + ) + .unwrap(); + assert_eq!(answer["state"], "timed_out", "{answer}"); + assert!(started.elapsed() < Duration::from_secs(10)); + for bad in [json!(0), json!(3601), json!("10")] { + assert!(home + .call( + "remote.shell", + shell("true", "bad", json!({"timeout_seconds": bad})) + ) + .is_err()); + } +} + +/// A job that is not detached ends with the bridge that started it (here, the thread standing +/// in for the bridge's relay); a detached one carries on and finishes. +#[test] +fn only_a_detached_job_outlives_its_bridge() { + let home = Arc::new(Home::new("detach")); + let start = |command: &'static str, key: &'static str, detach: bool| { + let home = home.clone(); + std::thread::spawn(move || { + let started = home + .call( + "remote.shell", + shell(command, key, json!({"wait_seconds": 1, "detach": detach})), + ) + .unwrap(); + // Running, so the supervisor has armed its parent-death signal before this + // "bridge" goes away. + assert_eq!(started["state"], "running", "{started}"); + started["job_id"].as_str().unwrap().to_owned() + }) + .join() + .unwrap() + }; + let attached = start("sleep 60", "attached", false); + let detached = start("sleep 2; echo survived", "detached", true); + let ended = home.finished(&attached); + assert_eq!(ended["state"], "cancelled", "{ended}"); + assert!( + ended["message"] + .as_str() + .unwrap() + .contains("connection ended"), + "{ended}" + ); + let survived = home.finished(&detached); + assert_eq!(survived["state"], "completed", "{survived}"); + assert_eq!(survived["stdout_tail"], "survived\n"); +} + +/// Eight jobs may run at once in an account; a ninth is refused until one ends. +#[test] +fn at_most_eight_jobs_run_at_once() { + let home = Home::new("concurrency"); + let mut ids = Vec::new(); + for index in 0..8 { + let started = home + .call( + "remote.shell", + shell("sleep 30", &format!("k{index}"), json!({"wait_seconds": 0})), + ) + .unwrap(); + ids.push(started["job_id"].as_str().unwrap().to_owned()); + } + let refused = home + .call("remote.shell", shell("true", "ninth", json!({}))) + .unwrap_err(); + assert!( + refused.to_string().starts_with("quota_exceeded:"), + "{refused}" + ); + for id in &ids { + home.call("remote.cancel", json!({"job_id": id})).unwrap(); + } + home.call("remote.shell", shell("true", "ninth", json!({}))) + .unwrap(); +} + +/// What the fake broker holds: one attachment to read, and the uploads it was given. +#[derive(Default)] +struct Store { + blob: Vec, + blob_sha256: String, + uploads: HashMap, bool)>, + credentials: Vec, + delay: Duration, +} + +/// A broker connection answering `blob.read`, `blob.begin`, `blob.chunk` and `blob.finish` the +/// way the real broker does, from `store`. +fn fake_broker(store: Arc>) -> BrokerLink { + BrokerLink { + connect: Arc::new(move || { + let (ours, theirs) = UnixStream::pair()?; + let store = store.clone(); + std::thread::spawn(move || { + let mut reader = BufReader::new(theirs.try_clone().unwrap()); + let mut writer = theirs; + let mut line = String::new(); + while reader.read_line(&mut line).unwrap_or(0) > 0 { + let request: Value = serde_json::from_str(&line).unwrap(); + line.clear(); + let delay = store.lock().unwrap().delay; + std::thread::sleep(delay); + let mut store = store.lock().unwrap(); + store + .credentials + .push(request["credential"].as_str().unwrap_or("").to_owned()); + let p = &request["params"]; + let answer = match request["method"].as_str().unwrap() { + "blob.read" => { + let offset = p["offset"].as_u64().unwrap() as usize; + let end = (offset + 262_144).min(store.blob.len()); + json!({"result": { + "blob": {"size": store.blob.len(), "sha256": store.blob_sha256}, + "offset": offset, + "data_hex": hex::encode(&store.blob[offset..end]), + "next_offset": end, + "complete": end == store.blob.len(), + }}) + } + "blob.begin" => { + let id = format!("blob-{}", store.uploads.len()); + store.uploads.insert( + id.clone(), + ( + p["size"].as_u64().unwrap(), + p["sha256"].as_str().unwrap().to_owned(), + Vec::new(), + false, + ), + ); + json!({"result": {"id": id, "channel_id": p["channel_id"], "name": p["name"]}}) + } + "blob.chunk" => { + let upload = store + .uploads + .get_mut(p["blob_id"].as_str().unwrap()) + .unwrap(); + assert_eq!(p["offset"].as_u64().unwrap(), upload.2.len() as u64); + upload + .2 + .extend(hex::decode(p["data_hex"].as_str().unwrap()).unwrap()); + json!({"result": {"offset": upload.2.len()}}) + } + "blob.finish" => { + let upload = store + .uploads + .get_mut(p["blob_id"].as_str().unwrap()) + .unwrap(); + if hex::encode(Sha256::digest(&upload.2)) == upload.1 + && upload.2.len() as u64 == upload.0 + { + upload.3 = true; + json!({"result": {"complete": true}}) + } else { + json!({"error": {"code": "digest_mismatch", "message": "digest_mismatch: attachment content differs from declaration"}}) + } + } + other => panic!("unexpected {other}"), + }; + let mut answer = answer; + answer["id"] = request["id"].clone(); + if writer.write_all(format!("{answer}\n").as_bytes()).is_err() { + break; + } + } + }); + Ok((ours.try_clone()?, BufReader::new(ours))) + }), + credential: "worker-credential".into(), + } +} + +fn pattern(size: usize) -> Vec { + (0..size).map(|i| (i * 31 % 251) as u8).collect() +} + +/// A channel attachment is copied to the server in the background, whole and checked, with +/// the run's own credential, and lands owner-only. +#[test] +fn an_attachment_is_copied_to_the_server() { + let home = Home::new("fetch"); + let blob = pattern(700_000); + let store = Arc::new(Mutex::new(Store { + blob_sha256: hex::encode(Sha256::digest(&blob)), + blob, + ..Store::default() + })); + let link = fake_broker(store.clone()); + let started = home + .call_as( + "owner-1", + "run-1", + "remote.fetch_blob", + json!({"blob_id": "0123abcd-0000-4000-8000-000000000001", "path": "inbox/data.bin", "idempotency_key": "fetch"}), + &link, + ) + .unwrap(); + let finished = home.finished(started["job_id"].as_str().unwrap()); + assert_eq!(finished["state"], "completed", "{finished}"); + assert_eq!(finished["size"], 700_000); + let target = home.work().join("inbox/data.bin"); + assert_eq!(fs::read(&target).unwrap(), store.lock().unwrap().blob); + assert_eq!( + fs::metadata(&target).unwrap().permissions().mode() & 0o777, + 0o600 + ); + assert!(store + .lock() + .unwrap() + .credentials + .iter() + .all(|credential| credential == "worker-credential")); + // Nothing but the copy was left in the folder. + let names: Vec<_> = fs::read_dir(home.work().join("inbox")) + .unwrap() + .map(|e| e.unwrap().file_name()) + .collect(); + assert_eq!(names.len(), 1, "{names:?}"); + // A file already there is never replaced. + let refused = home + .call_as( + "owner-1", + "run-1", + "remote.fetch_blob", + json!({"blob_id": "0123abcd-0000-4000-8000-000000000001", "path": "inbox/data.bin", "idempotency_key": "fetch-again"}), + &link, + ) + .unwrap_err(); + assert!(refused.to_string().starts_with("conflict:"), "{refused}"); +} + +#[test] +fn a_copy_that_does_not_match_its_digest_is_discarded() { + let home = Home::new("fetch-bad"); + let store = Arc::new(Mutex::new(Store { + blob: pattern(300_000), + blob_sha256: "00".repeat(32), + ..Store::default() + })); + let started = home + .call_as( + "owner-1", + "run-1", + "remote.fetch_blob", + json!({"blob_id": "abcd", "path": "bad.bin", "idempotency_key": "fetch"}), + &fake_broker(store), + ) + .unwrap(); + let finished = home.finished(started["job_id"].as_str().unwrap()); + assert_eq!(finished["state"], "failed", "{finished}"); + assert!(finished["error"] + .as_str() + .unwrap() + .starts_with("digest_mismatch:")); + assert_eq!( + fs::read_dir(home.work()).unwrap().count(), + 0, + "no partial file left" + ); +} + +/// A server file is uploaded to the run's channel in the background, and the job's result +/// names the attachment the daemon then posts. +#[test] +fn a_server_file_is_shared_to_the_channel() { + let home = Home::new("share"); + let bytes = pattern(600_000); + fs::write(home.work().join("result.csv"), &bytes).unwrap(); + let store = Arc::new(Mutex::new(Store::default())); + let started = home + .call_as( + "owner-1", + "run-1", + "remote.share_file", + json!({"path": "result.csv", "idempotency_key": "share"}), + &fake_broker(store.clone()), + ) + .unwrap(); + let finished = home.finished(started["job_id"].as_str().unwrap()); + assert_eq!(finished["state"], "completed", "{finished}"); + assert_eq!(finished["blob_id"], "blob-0"); + assert_eq!(finished["name"], "result.csv"); + assert_eq!(finished["size"], 600_000); + let store = store.lock().unwrap(); + let upload = &store.uploads["blob-0"]; + assert!(upload.3, "finished"); + assert_eq!(upload.2, bytes); +} + +/// A transfer answers at once with its job; the copy itself never holds the relay. +#[test] +fn a_transfer_returns_before_it_finishes() { + let home = Home::new("share-slow"); + fs::write(home.work().join("big.bin"), pattern(900_000)).unwrap(); + let store = Arc::new(Mutex::new(Store { + delay: Duration::from_millis(400), + ..Store::default() + })); + let asked = Instant::now(); + let started = home + .call_as( + "owner-1", + "run-1", + "remote.share_file", + json!({"path": "big.bin", "idempotency_key": "share"}), + &fake_broker(store), + ) + .unwrap(); + assert!( + asked.elapsed() < Duration::from_millis(400), + "{:?}", + asked.elapsed() + ); + assert!( + matches!(started["state"].as_str(), Some("running" | "starting")), + "{started}" + ); + let finished = home.finished(started["job_id"].as_str().unwrap()); + assert_eq!(finished["state"], "completed", "{finished}"); +} + +/// Files outside the work folder are reachable by absolute and `~/` paths, through symlinks. +#[test] +fn file_operations_reach_the_whole_account() { + let home = Home::new("files"); + let shared = home.path.join("datasets"); + fs::create_dir(&shared).unwrap(); + fs::write(shared.join("table.tsv"), b"a\tb\n").unwrap(); + std::os::unix::fs::symlink(&shared, home.work().join("data")).unwrap(); + let read = home + .call("remote.read", json!({"path": "data/table.tsv"})) + .unwrap(); + assert_eq!(read["text_utf8"], "a\tb\n"); + let stat = home + .call("remote.stat", json!({"path": "~/datasets"})) + .unwrap(); + assert_eq!(stat["type"], "directory"); + home.call( + "remote.write", + json!({"path": shared.join("out.txt"), "text": "x", "mode": "create_new"}), + ) + .unwrap(); + let listing = home + .call("remote.list", json!({"path": "~/datasets"})) + .unwrap(); + assert_eq!(listing["entries"].as_array().unwrap().len(), 2); + let hashed = home + .call("remote.hash", json!({"path": "data/table.tsv"})) + .unwrap(); + assert_eq!(hashed["sha256"], hex::encode(Sha256::digest(b"a\tb\n"))); +} From d641c4dcb934d6d5a889ec7b7d9cfe019f5689dc Mon Sep 17 00:00:00 2001 From: Wanjun Gu Date: Fri, 16 Oct 2026 19:56:26 -0700 Subject: [PATCH 06/32] docs(crew): protocol contract for full access, bridge-local methods and the probe --- docs/crew/design/protocol-contract.md | 47 +++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/docs/crew/design/protocol-contract.md b/docs/crew/design/protocol-contract.md index 324b1206a..66e4cd6d8 100644 --- a/docs/crew/design/protocol-contract.md +++ b/docs/crew/design/protocol-contract.md @@ -231,6 +231,53 @@ The signed `run.create` may additionally include `remote_root` (absolute, narrow The bridge also refuses (`invalid_scope`) a work folder that is, contains or sits inside a protected folder, so a confined worker can never replace the bridge or wrap it through a shell startup file and become the in-path process of the join's threat model (D15, SR4). The protected folders are `~/.ssh`, `~/.aws`, `~/.config`, `~/.local/state`, `~/.local/share/biorouter-crew`, `~/.local/bin`, `~/bin`, `~/.bashrc.d` and `~/.profile.d`, plus the resolved bridge executable's directory and its ancestors inside `HOME`, under every spelling of `HOME` and of the bridge path. The refusal suggests a folder such as `~/crew-work/`. Protecting directories on the remote login `PATH`, or allowing only folders under a Crew-created prefix, is follow-up FU9. +*Amended for open workspaces (design OW-6):* a confined (folder-mode) command on a kernel without Landlock ABI 3 is refused before anything is reserved or started, as `unsupported: This server's kernel can't confine commands (Landlock ABI {n}). Choose 'As me on the server' in Connection settings to run commands as yourself.` Before, the helper started and failed with the Landlock library's own error. Every other folder-mode rule above is unchanged. + +### Full access: running as the member's own account + +*Added for open workspaces ([design](open-workspaces-and-automatic-setup.md), OW-6 and OW-7).* A broker that supports it advertises `remote_access_v1`. The signed `run.create` then also takes `remote_access`: `"folder"` (the default, and what an absent or `null` value means) or `"full"`. Full is accepted only with a `remote_root`, `remote_execution: true` and a private provider; anything else, and any other value, is `invalid_params`. A public provider never gets a remote root, so it is refused earlier, `privacy_denied`. The run persists the level, serialized only when it is full, so a journal written before it replays and re-serializes byte for byte and an older broker ignores the field; the run in the answer echoes it. Full is still a remote root, so the run is protected and Restricted exactly as a folder run is. + +`run.remote_scope` now also returns `access` (`"folder"` or `"full"`), `owner_id` (the run's owner, which keys full-access jobs) and `channel_id` (the run's destination, where a shared server file is uploaded). A scope with no `access`, from an older broker, is a folder run: a newer bridge fails closed, and any `access` other than the two words is refused `invalid_scope`. + +When the scope says `full`, the member's bridge runs the request as the member's own account, with no confinement and no ownership rule, and refuses to run at all as UID 0. The broker still executes nothing (its `hello.unsupported` list is unchanged): only the member's own bridge does, under the member's own UID, after the same scope fetch and recheck as folder mode. A worker credential still reaches only the methods listed above; none of these is a broker method. + +| Method | Params | Answer | +|---|---|---| +| `remote.shell` | `{command, cwd?, timeout_seconds?, wait_seconds?, detach?, idempotency_key}` | The job, below. `command` is 1 to 65,536 bytes; `timeout_seconds` 1 to 3,600 (default 600), refused outside that; `wait_seconds` at most 20 (default 10). | +| `remote.execute` | `{argv, cwd?, timeout_seconds?, wait_seconds?, detach?, idempotency_key}` | As `remote.shell`, with up to 256 argv items of 65,536 bytes in all, passed to the program exactly. | +| `remote.job_status` | `{job_id, tail_bytes?}` | The job. `tail_bytes` is at most 65,536 per stream (default 16,384). | +| `remote.cancel` | `{job_id}` | The job, once stopped. A finished job is left as it is. | +| `remote.list` | `{path?, limit?}` | `{path, entries[{name, type, size, mtime, mode, target_type?}], truncated}`, at most 2,000 entries (default 500). | +| `remote.stat` | `{path}` | `{path, exists, type, size, mtime, mode, uid, gid, symlink, target?}`; a missing path is `exists: false`, not an error. | +| `remote.read` | `{path, offset?, length?}` | At most 393,216 bytes from `offset`: `{path, offset, size, file_size, next_offset, eof, sha256, data_hex, text_utf8?}`. The text rides along only when the bytes are UTF-8 and the frame has room. | +| `remote.write` | `{path, text \| data_hex, mode?, create_parents?}` | At most 393,216 bytes. `mode` is `truncate` (default), `append` or `create_new` (`conflict` when the file exists). `{path, mode, size, file_size, sha256}`. | +| `remote.hash` | `{path}` | `{path, sha256, size}`, for files up to 2 GiB. | +| `remote.fetch_blob` | `{blob_id, path, idempotency_key}` | The transfer job, at once. | +| `remote.share_file` | `{path, name?, media_type?, idempotency_key}` | The transfer job, at once. | + +A `path` (and `cwd`) is absolute, `~` or `~/…` (the bridge's `HOME`), or relative to the run's `remote_root`; `~user` is refused. Symlinks are followed. Two structured writes are refused as accident prevention, not security, since the shell can do either: the running bridge executable and `~/.ssh/authorized_keys` (and `authorized_keys2`), however they are spelled or linked. `remote.fetch_blob` refuses those too, and never replaces a file that exists. Folder mode refuses `remote.shell`, `remote.stat`, `remote.fetch_blob` and `remote.share_file` with a sentence that names full access. + +**Jobs.** A command runs as ` -lc `, the login shell taken from the account database (`getpwuid`), or `/bin/sh`; an argv runs as ` -lc 'exec "$@"' biorouter-crew-job ARGV…`, so modules, conda and `PATH` behave as in the member's own login and the arguments are never reparsed (a csh-family shell gets `-c`, and an argv under a shell outside the Bourne family goes through `/bin/sh -l`). Standard input is `/dev/null`; the environment is the bridge's SSH login environment plus `BIOROUTER_CREW_JOB_ID`. A supervisor, the bridge re-executed as `biorouter-crew __crew-run ` with its standard streams on `/dev/null` (never the bridge's frame channel), starts the command in its own process group, appends its output to `stdout` and `stderr` in the job's directory, enforces the timeout and writes `exit.json`. A job that is not detached ends with its bridge: the supervisor has `PR_SET_PDEATHSIG`, and on that signal stops the job's process group and records `cancelled` with a sentence saying the connection ended. A detached job (`detach: true`) forks twice, so it outlives the bridge and the grant. + +The job ID is SHA-256 of `owner_id:idempotency_key`, so any later run of the same person reads the same job and nobody else's (`forbidden` for another owner, `not_found` for no such job); a retry with the same key and the same command, folder, timeout and detach is the same job, run once, and a different request under the key is `conflict`. Records live in `~/.local/state/biorouter-crew/jobs//` (directory 0700, files 0600): `job.json` (what to run, owner, run and digest), `process.json` (the command's PID, process group and `/proc` start time, and the supervisor's), `exit.json`, and the two output files. A job answer is `{job_id, kind, state, detach, created_at, exit_code?, signal?, ended_at?, error?, message?, stdout_tail, stderr_tail, truncated, output_dir}`, where `state` is `starting`, `running`, `completed`, `failed`, `timed_out`, `cancelled` or `lost` (the process that ran it is gone without an exit record; inspect its output before retrying). `remote.cancel` sends `SIGTERM` to the job's process group and `SIGKILL` five seconds later, each only while the recorded leader's PID still has its recorded start time. At most eight jobs (commands and transfers together) run at once in an account (`quota_exceeded`), and 512 records are kept: at that number the oldest finished record is removed to make room. The bridge answers within `wait_seconds` whatever the job does, so the relay is never held longer than 20 seconds. + +**Transfers.** `remote.fetch_blob` copies a channel attachment to the server and `remote.share_file` uploads a server file of at most 1 GiB to the run's channel. Each answers at once with a job of kind `fetch_blob` or `share_file`, and runs on a background thread over a second connection to the broker, opened as the bridge opened its first, with the frame's own worker credential, so the relay never waits on it. A fetch loops `blob.read` (the broker's `blob_authorized` still decides which attachments the run may read, chunk by chunk), writes a private temporary file beside the target, checks the attachment's `sha256` and size, and renames it into place without replacing anything. A share hashes the file, then sends `blob.begin`, `blob.chunk` and `blob.finish` with idempotency keys derived from the job ID; the broker's worker upload rules (the run's channel, quotas, Restricted for a remote run) apply unchanged. The job reports `bytes_done` and `size` while it runs, and on completion a fetch's `{path, size, sha256}` or a share's `{blob_id, name, size, sha256, media_type}`, which the daemon then posts. A transfer stops at its next chunk when cancelled, and dies with its bridge (`lost`). + +### Bridge-local methods + +*Added for open workspaces (design OW-6, OW-8).* The member's bridge answers `bridge.*` requests itself and never forwards them; an unknown one is refused locally, `unsupported`. An older bridge forwards them, and the broker refuses them (no device signature), which is how a daemon tells an old bridge. + +- `bridge.hello {}` → `{bridge_version, capabilities: ["remote_full_v1", "remote_transfer_v1", "bridge_probe_v1"], landlock_abi, confined_execution_supported, user, uid, home, shell}`. `confined_execution_supported` is Landlock ABI 3 or later with seccomp. The account fields come from the account database, not the environment. +- `bridge.probe {}` → the server facts below. + +A response too large for one frame is answered `response_too_large` under its own ID, instead of ending the bridge as it used to. + +### Server probe + +`bridge.probe`, `biorouter-crew probe [--output-format json]` and `biorouter-crew doctor [--output-format json]` report fixed facts: `{schema: 1, host, os, kernel, arch, glibc, cpus, mem_gib, gpus[{name, memory_mib}], scheduler{kind, commands}, modules, conda[], containers{docker, docker_group, apptainer, podman}, languages{python3?, R?, julia?}, storage[{path, fs, size_gib, avail_gib}], landlock_abi, user, home}`. A fact that cannot be learned is `null` or empty, never an error. `scheduler.kind` is `slurm`, `sge`, `pbs`, `lsf`, `htcondor` or `null`. Every check is a fixed file read, system call or argv with no model input, bounded to five seconds and 64 KiB of output, and none reaches the network: commands are found by the login shell started with only `HOME`, `USER`, `LOGNAME`, `SHELL`, a minimal `PATH` and `LANG`, and run by absolute path with a fixed environment. It never reads environment variables, process lists, other people's homes (a path inside one is left out), SSH configuration or history; paths in the member's home are shown as `~/…`. `storage` names the home's, `/tmp`'s and any `/scratch*` filesystem, then other data filesystems of 100 GiB or more, one entry per filesystem, at most 12. + +`doctor` adds `version`, `hosting_blockers` (codes: `linux_required`, `pidfd_unavailable`, `machine_id_unavailable`, `state_on_network_filesystem`, `home_writable_by_others`, each mirroring a check the broker makes when it starts) and `execution_limits` (`landlock_abi_below_3`, `seccomp_unavailable`: folder-mode commands cannot run, full access still can). `biorouter-crew --version --output-format json` prints `{version, protocol, capabilities, bridge_capabilities}`, where `capabilities` is what this binary's broker advertises in `hello`. Without the flag every older verb prints what it always did. + The journal refuses a delta record or logical state beyond 16 MiB, and total retained audit beyond 1 GiB. All audit records remain retained; no rotation or lossy compaction is performed. Replay keeps one bounded record plus current state in memory, rather than loading the complete journal. ### Workspace quotas From 2781a82b6e4c6c4e96c6b2c1c75e370dafc56bb4 Mon Sep 17 00:00:00 2001 From: Wanjun Gu Date: Fri, 16 Oct 2026 19:56:55 -0700 Subject: [PATCH 07/32] crew broker: open governance, deletion and the workspace context Implements the broker half of OW-1 to OW-5, OW-8 and wide runs from docs/crew/design/open-workspaces-and-automatic-setup.md: - Workspace.governance (managed by default, stored only when open) and workspace.set_governance, host and person only. - Under open governance every active member renames, archives, invites, adds and removes people (never a channel's owner), joins teams and channels on their own (team.join, channel.join) and admits joiners; additions no longer move the policy epoch, and admission, team.create and channel.create put everyone where they belong. The snapshot lists joinable teams and channels. - channel.delete and team.delete remove live state with map removals only, files after the commit, and leave messages unreadable in place; workspace.delete commits a tombstone, refuses everything but hello, and erases only the broker's own files after 30 seconds. biorouter-crew delete does the same offline. - workspace.context.set/get with a revision check, at most 64 KiB, readable by every member and agent. - run.create accepts up to 100 source channels. - Capabilities open_governance_v1, delete_v1, workspace_context_v1 and wide_runs_v1 after presence_v1, and the remote-folder refusal now says where the folder is set. --- crates/biorouter-crew/src/broker.rs | 1019 ++++++++++++++++++++-- crates/biorouter-crew/src/broker/join.rs | 19 +- crates/biorouter-crew/src/lib.rs | 46 +- crates/biorouter-crew/src/main.rs | 13 +- 4 files changed, 1023 insertions(+), 74 deletions(-) diff --git a/crates/biorouter-crew/src/broker.rs b/crates/biorouter-crew/src/broker.rs index 3b23c3ec2..815b57345 100644 --- a/crates/biorouter-crew/src/broker.rs +++ b/crates/biorouter-crew/src/broker.rs @@ -389,6 +389,11 @@ struct State { /// broker built without the feature never drops a pending join from state. #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] pending_joins: BTreeMap, + /// The workspace's context document (OW-8). Serialized only once it is first set, so a + /// journal that never held one replays and re-serializes byte for byte, and an older broker + /// (which ignores unknown fields) replays a journal that did. + #[serde(default, skip_serializing_if = "Option::is_none")] + context: Option, } #[derive(Serialize, Deserialize)] struct Record { @@ -837,22 +842,31 @@ const BLOB_MAX_BYTES: u64 = 1024 * MIB; const WORKSPACE_BLOB_BYTES: u64 = 10 * 1024 * MIB; const WORKSPACE_BLOBS: usize = 10_000; /// The host's own operations that may use the administrative headroom. -const ADMIN_METHODS: [&str; 6] = [ +const ADMIN_METHODS: [&str; 8] = [ "enrollment.invite", "enrollment.approve", "enrollment.cancel", "enrollment.revoke", "policy.set", "workspace.rename", + "workspace.set_governance", + "workspace.delete", ]; /// Operations that only take access away. Anyone may use the administrative headroom for them, /// so a full workspace never keeps a person in a channel or an agent running: each adds at -/// most a bounded idempotency result, and one member's results are capped. -const ACCESS_REMOVAL_METHODS: [&str; 3] = ["run.revoke", "membership.revoke", "channel.archive"]; +/// most a bounded idempotency result, and one member's results are capped. Deleting a channel +/// or a team also shrinks the state. +const ACCESS_REMOVAL_METHODS: [&str; 5] = [ + "run.revoke", + "membership.revoke", + "channel.archive", + "channel.delete", + "team.delete", +]; /// Methods that add records to the actor's share of the state: a member past /// [`Quotas::member_state_bytes`] is refused them. Everything else changes records in place, /// removes them, or adds a bounded amount (a read position, a membership). -const SHARE_METHODS: [&str; 8] = [ +const SHARE_METHODS: [&str; 9] = [ "message.post", "run.project", "run.create", @@ -861,7 +875,50 @@ const SHARE_METHODS: [&str; 8] = [ "invitation.create", "team.create", "channel.create", + "workspace.context.set", ]; +/// At most this many channels one run may read (`wide_runs_v1`; it was 20). Every worker +/// request rechecks each of them, and what an agent reads of them is byte-bounded. +const RUN_SOURCE_CHANNELS: usize = 100; +/// Every request but `hello` once the host has deleted the workspace (OW-5), ahead of +/// authentication, so a member's daemon learns it rather than seeing the broker as offline. +const WORKSPACE_DELETED: &str = + "workspace_deleted: The host deleted this workspace. Remove it from this computer."; +/// How long after `workspace.delete` the broker keeps answering `workspace_deleted` before it +/// erases its files and exits: long enough for connected members' daemons to hear it. +const DELETE_GRACE_SECS: u64 = 30; +/// The marker `workspace.delete` writes into the state directory before erasing anything and +/// removes last, so a directory a crash left half-erased is never mistaken for an empty one +/// (where the next `start` would create a new workspace without a word). +const DELETED_MARKER: &str = "DELETED"; +/// A team's `#general` goes only with its team (`channel.delete`). +const GENERAL_DELETE: &str = + "invalid_params: #general belongs to its team. Delete the team instead."; +/// `channel.delete` or `team.delete` through an agent's grant. The worker allowlist refuses it +/// first; this is the second, method-level wall. +const DELETE_AGENT_REFUSED: &str = "forbidden: only a person can delete channels or teams"; +/// A governance-only method (`team.join`, `channel.join`) in a managed workspace. +const JOIN_NEEDS_OPEN: &str = + "forbidden: In this workspace, the owner or the host adds people. Ask them to add you."; +/// A request to manage teams, channels or people through an agent's grant, refused inside the +/// method behind the worker allowlist. +const PERSON_REQUIRED: &str = "forbidden: only a person can do this"; +/// What a delete names no longer has the name the person confirmed (`expected_name`): the +/// snapshot they chose from was stale, or rewritten on the way. +const CHANNEL_NAME_CHANGED: &str = + "name_changed: This channel was renamed since you chose it. Check its name and try again."; +const TEAM_NAME_CHANGED: &str = + "name_changed: This team was renamed since you chose it. Check its name and try again."; +const WORKSPACE_NAME_CHANGED: &str = + "name_changed: This workspace was renamed since you chose it. Check its name and try again."; +/// The largest workspace context document, in bytes of UTF-8 text. +const CONTEXT_MAX_BYTES: usize = 64 * 1024; +/// `workspace.context.set` from a revision that is no longer the current one. +const CONTEXT_CONFLICT: &str = "revision_conflict: Someone changed the workspace context since you opened it. Reload it, then make your change again."; +/// A run without a remote work folder asked for something only a work folder allows. The +/// folder is the member's own connection setting, not anyone's approval, so the sentence says +/// where to set it. +const NO_REMOTE_FOLDER: &str = "forbidden: This task has no remote work folder. Set one in Connection settings, then start a new task."; const MEMBER_STATE_QUOTA: &str = "quota_exceeded: You have used your share of this workspace's storage. Reading still works; ask the workspace host about starting a new workspace."; const MEMBER_JOURNAL_QUOTA: &str = "quota_exceeded: You have made as many changes as one member's share of this workspace's audit journal allows. Reading still works; ask the workspace host about starting a new workspace."; /// A snapshot's invitations, runs and references are each capped at this many serialized @@ -1128,6 +1185,8 @@ fn recovered_state( institution_id: None, policy_epoch: 1, name: initial_name.map(str::to_owned), + governance: Governance::Managed, + deleted_at: None, }, bootstrap_key: bootstrap_key.into(), workspace_signing_key: digest(token().as_bytes()), @@ -1148,6 +1207,7 @@ fn recovered_state( sequence: 0, read_positions: BTreeMap::new(), pending_joins: BTreeMap::new(), + context: None, } } }; @@ -1264,6 +1324,14 @@ impl Broker { ); } let root = canonical_root.as_path(); + // An erase a crash interrupted (OW-5): never reopened, so a `start` cannot quietly put + // a new workspace where the deleted one was being removed. + if let Some(workspace_id) = deleted_marker(root)? { + bail!( + "workspace_deleted: This state directory holds a workspace its host deleted. Finish removing it: biorouter-crew delete --state-dir {} --workspace-id {workspace_id}", + root.display() + ); + } #[cfg(target_os = "linux")] { let path = std::ffi::CString::new(root.as_os_str().as_encoded_bytes())?; @@ -1494,6 +1562,22 @@ impl Broker { pub fn workspace(&self) -> &Workspace { &self.state.workspace } + /// Whether the host deleted the workspace at least [`DELETE_GRACE_SECS`] before `now`, so + /// its files are due to be erased. + fn erase_due(&self, now: u64) -> bool { + self.state + .workspace + .deleted_at + .is_some_and(|at| now >= at.saturating_add(DELETE_GRACE_SECS)) + } + /// What the runtime watch does once a deleted workspace's grace period is over at `now`: + /// erase the state directory ([`erase_store`]) and return what was left; `None` before + /// then. The broker must not be used afterwards. Test builds only. + #[cfg(feature = "test-seams")] + pub fn erase_if_due(&self, now: u64) -> Option>> { + self.erase_due(now) + .then(|| erase_store(&self.root, &self.state.workspace.id)) + } /// `principal`'s device made a signed request on `conn`. fn note_presence(&mut self, conn: &mut Connection, principal: &str) { let now = Instant::now(); @@ -1584,6 +1668,9 @@ impl Broker { if req.method == "hello" { return self.hello(req); } + // Once the host deleted the workspace (OW-5) nothing but `hello` is answered, ahead of + // every pre-authentication method, so nobody signs in, joins or bootstraps it again. + ensure!(self.state.workspace.deleted_at.is_none(), WORKSPACE_DELETED); if req.method == "auth.challenge" { return self.challenge(uid, conn, req); } @@ -1615,6 +1702,7 @@ impl Broker { | "blob.status" | "reference.get" | "profile.suggest" + | "workspace.context.get" ) { return self.read(&actor, req); } @@ -1659,6 +1747,7 @@ impl Broker { | "blob.status" | "reference.create" | "reference.get" + | "workspace.context.get" ), "forbidden: worker operation unavailable" ); @@ -1784,6 +1873,7 @@ impl Broker { return Err(error); } }; + let pending_file_removals = removed_attachments(&self.state, &state, expired_uploads); let host = self.manager(&self.state, &actor.id).is_ok(); let method = req.method.as_str(); let administrative = host && actor.run.is_none() && ADMIN_METHODS.contains(&method); @@ -1841,7 +1931,7 @@ impl Broker { return Err(error); } // Only once the removal is journaled: a failed commit keeps both. - for blob_id in expired_uploads { + for blob_id in pending_file_removals { let _ = fs::remove_file(self.root.join("blobs").join(blob_id)); } Ok(result) @@ -1853,11 +1943,13 @@ impl Broker { // The host may add people to a channel it is not in (direct add), so its retry of // that add is re-authorized as the host, not as a member. Likewise a member's retry // of leaving a channel (they are no longer in it, and the result is the one they - // were given), and the host's of acting on a channel whose owner left the workspace. + // were given), the host's of acting on a channel whose owner left the workspace, + // and anyone's of deleting a channel, which no longer exists for anyone to be in. let host_add = req.method == "channel.add_member" && self.manager(&self.state, &actor.id).is_ok(); let outside = Self::is_leave(req, &actor.id) - || self.stewards_orphaned_channel(&self.state, actor, req); + || self.stewards_orphaned_channel(&self.state, actor, req) + || req.method == "channel.delete"; if !host_add && !outside { self.channel(&self.state, &actor.id, channel, false)?; } @@ -1928,9 +2020,11 @@ impl Broker { .to_bytes(), ); let mut hello = json!({"protocol":1,"workspace_id":workspace.id,"host_uid":workspace.host_uid,"mode":workspace.mode,"institution_id":workspace.institution_id,"policy_epoch":workspace.policy_epoch,"name":workspace.name,"workspace_public_key":public_key,"node_id":node_id,"workspace_key_fingerprint":digest(&key.verifying_key().to_bytes()),"challenge_nonce":nonce,"signature":signature,"signature_v2":signature_v2,"capabilities":capabilities,"unsupported":["arbitrary_shell","remote_filesystem","network_filesystem","cross_workspace_release"]}); - // Unsigned, for display only: whether this broker is still saving changes. An older - // broker sends no `state`, which a client reads as unknown. + // Unsigned, for display only: whether this broker is still saving changes, or the host + // deleted the workspace and it is about to erase itself. An older broker sends no + // `state`, which a client reads as unknown. match &self.storage_fault { + _ if workspace.deleted_at.is_some() => hello["state"] = json!("deleted"), None => hello["state"] = json!("running"), Some(fault) => { hello["state"] = json!("storage_failed"); @@ -1956,6 +2050,11 @@ impl Broker { "unique_names_v1", "direct_add_v1", "presence_v1", + "open_governance_v1", + "delete_v1", + "workspace_context_v1", + // remote_access_v1 is pushed here by the bridge change + "wide_runs_v1", ]; #[cfg(feature = "join-by-name")] capabilities.push("join_by_name_v1"); @@ -2057,10 +2156,12 @@ impl Broker { (Some(expected), Some(principal)) if principal.id == expected && principal.username == current_username => principal.clone(), (None, None) => { Self::ensure_username_free(&state, uid, ¤t_username)?; - Principal { + let principal = Principal { id: id(), uid, username: current_username.clone(), nickname: current_username, avatar: None, active: true, - } + }; + admit_everywhere(&mut state, &principal.id); + principal } _ => bail!("identity_mismatch: enrollment principal changed; request a new invitation explicitly identifying the existing principal or offboard the old account"), }; @@ -2313,9 +2414,24 @@ impl Broker { "blob.status" => self.read_blob_status(actor, req), "blob.read" => self.read_blob_read(actor, req), "profile.suggest" => self.read_profile_suggest(actor), + "workspace.context.get" => self.read_workspace_context(actor), _ => bail!("unsupported: method unavailable"), } } + /// `workspace.context.get {}`: the workspace context document, or `null` before anyone + /// wrote one. Any member, and any agent's grant, since every member's agent is meant to read + /// it; an agent on a public model only in a Public workspace, since in a Private one the + /// document describes a server that only private models may hear about. + fn read_workspace_context(&self, actor: &Actor) -> Result { + let s = &self.state; + if let Some(run) = &actor.run { + ensure!( + !run.public_provider || s.workspace.mode == Mode::Public, + "privacy_denied: The workspace context is not shared with public models in a Private workspace." + ); + } + Ok(json!({ "context": s.context })) + } fn message_wire(message: &Message) -> Value { let mut value = json!(message); value["sequence"] = json!(message.id); @@ -2479,6 +2595,8 @@ impl Broker { Self::snapshot_invitations(s, &index, &actor.id, now); let mut workspace = json!(s.workspace); workspace["host_principal_id"] = json!(host_principal_id(s)); + // Stored only when open; said either way here, so a client never has to infer it. + workspace["governance"] = json!(s.workspace.governance); let principals = self.snapshot_principals(s, &index, host); let actor_wire = Self::actor_wire(s, &index, &actor.id); // The actor's live runs only (a revoked, expired or superseded run grants nothing), @@ -2510,6 +2628,20 @@ impl Broker { let mut snapshot = json!({"workspace":workspace,"protected_channel_ids":[],"actor":actor_wire,"principals":principals,"former_principals":[],"teams":[],"channels":[],"invitations":invitations_wire,"runs":runs,"read_positions":{},"unread":{},"references":references,"totals":totals}); // Presence is display only: it is in memory, never journaled, and grants nothing. snapshot["online_principal_ids"] = json!(self.online_principal_ids(s)); + // At most 64 KiB of text, so it always fits beside the rest: teams and channels take + // what is left ([`Self::fill_places`]). + snapshot["context"] = json!(s.context); + let open = open_governance(s); + if open { + // What the actor may join on their own (`team.join`, `channel.join`): every team + // they are not in, and every unarchived channel of their teams they are not in. + let (joinable_teams, teams_total) = Self::joinable_teams(s, &actor.id); + let (joinable_channels, channels_total) = Self::joinable_channels(s, &actor.id); + snapshot["joinable_teams"] = Value::Array(joinable_teams); + snapshot["joinable_channels"] = Value::Array(joinable_channels); + snapshot["totals"]["joinable_teams"] = json!(teams_total); + snapshot["totals"]["joinable_channels"] = json!(channels_total); + } if host { snapshot["usage"] = self.usage(s); let refusals: BTreeMap<&str, usize> = self @@ -2519,10 +2651,12 @@ impl Broker { .filter(|(_, count)| *count > 0) .collect(); snapshot["name_collision_refusals"] = json!(refusals); - #[cfg(feature = "join-by-name")] - { - snapshot["pending_joins"] = join::project_for_manager(self, s); - } + } + // Whoever may admit joiners sees who is waiting: the host, or in an open workspace + // every member. Never a code, key, UID or join ID. + #[cfg(feature = "join-by-name")] + if host || open { + snapshot["pending_joins"] = join::project_for_manager(self, s); } self.fill_places( &mut snapshot, @@ -2926,6 +3060,55 @@ impl Broker { }) .collect() } + /// Open governance only: the teams `actor_id` is not in, each `{id, name, display_name, + /// handle, member_count}` (active members), as many as fit in one snapshot section, with how + /// many there are in all. Every member may join any of them, so naming them is the point. + fn joinable_teams(s: &State, actor_id: &str) -> (Vec, usize) { + let joinable: Vec = s + .teams + .values() + .filter(|team| !team.members.contains(actor_id)) + .map(|team| { + json!({ + "id": team.id, + "name": team.name, + "display_name": sanitize_team_name(&team.name), + "handle": name_key(&team.name), + "member_count": active_count(s, &team.members), + }) + }) + .collect(); + let total = joinable.len(); + (within_budget(joinable), total) + } + /// Open governance only: the unarchived channels of `actor_id`'s teams that they are not in, + /// each `{id, team_id, name, display_name, handle, classification, member_count}`, within + /// one snapshot section, with the total. + fn joinable_channels(s: &State, actor_id: &str) -> (Vec, usize) { + let joinable: Vec = s + .channels + .values() + .filter(|channel| !channel.archived && !channel.members.contains(actor_id)) + .filter(|channel| { + s.teams + .get(&channel.team_id) + .is_some_and(|team| team.members.contains(actor_id)) + }) + .map(|channel| { + json!({ + "id": channel.id, + "team_id": channel.team_id, + "name": channel.name, + "display_name": sanitize_channel_name(&channel.name), + "handle": name_key(&channel.name), + "classification": channel.classification, + "member_count": active_count(s, &channel.members), + }) + }) + .collect(); + let total = joinable.len(); + (within_budget(joinable), total) + } /// An invitation as its invitee or inviter sees it: the target's current name (the /// team's, or the channel's plus its team's), who invited, and whether it has expired. /// Naming the target to its intended invitee is the purpose of an invitation. @@ -3067,7 +3250,7 @@ impl Broker { let root = run .remote_root .as_ref() - .ok_or_else(|| anyhow!("forbidden: run has no remote path approval"))?; + .ok_or_else(|| anyhow!(NO_REMOTE_FOLDER))?; Ok( json!({"root":root,"allow_execute":run.remote_execution,"public_provider":run.public_provider,"run_id":run.id,"policy_epoch":run.policy_epoch,"expires_at":run.expires_at}), ) @@ -3195,6 +3378,13 @@ impl Broker { "channel.create" => self.mutate_channel_create(s, actor, req), "channel.rename" => self.mutate_channel_rename(s, actor, req), "workspace.rename" => self.mutate_workspace_rename(s, actor, req), + "workspace.set_governance" => self.mutate_workspace_set_governance(s, actor, req), + "workspace.delete" => self.mutate_workspace_delete(s, actor, req), + "workspace.context.set" => self.mutate_workspace_context_set(s, actor, req), + "team.join" => self.mutate_team_join(s, actor, req), + "channel.join" => self.mutate_channel_join(s, actor, req), + "channel.delete" => self.mutate_channel_delete(s, actor, req), + "team.delete" => self.mutate_team_delete(s, actor, req), "invitation.create" => self.mutate_invitation_create(s, actor, req), "invitation.accept" => self.mutate_invitation_accept(s, actor, req), "team.add_member" => self.mutate_team_add_member(s, actor, req), @@ -3410,7 +3600,17 @@ impl Broker { let name = Self::team_name_for(s, p, None)?; let team_id = id(); let channel_id = id(); - let members = BTreeSet::from([who.clone()]); + let mut members = BTreeSet::from([who.clone()]); + if open_governance(s) { + // OW-3: a new team in an open workspace has everyone in it, and so does its + // `#general`. Nobody's existing grant reads more for it, so the epoch stays. + members.extend( + s.principals + .values() + .filter(|p| p.active) + .map(|p| p.id.clone()), + ); + } let team = Team { id: team_id.clone(), name, @@ -3459,13 +3659,24 @@ impl Broker { Some(v) => serde_json::from_value(v.clone())?, None => Classification::Restricted, }; + let mut members = BTreeSet::from([who.clone()]); + if open_governance(s) { + // OW-3: a new channel in an open workspace has every active member of its team. + let team = s.teams.get(team_id).expect("checked team"); + members.extend( + team.members + .iter() + .filter(|id| s.principals.get(*id).is_some_and(|p| p.active)) + .cloned(), + ); + } let c = Channel { id: id(), team_id: team_id.into(), name, created_by: who.clone(), owner_id: who.clone(), - members: BTreeSet::from([who.clone()]), + members, archived: false, classification, pending_owner: None, @@ -3517,16 +3728,18 @@ impl Broker { ); Ok(name) } - /// `team.rename {team_id, name}`: the team's creator only. The ID, memberships and - /// invitations are unchanged; the old name is free at once. + /// `team.rename {team_id, name}`: the team's creator only, or in an open workspace any of + /// its members. The ID, memberships and invitations are unchanged; the old name is free at + /// once. fn mutate_team_rename(&self, s: &mut State, actor: &Actor, req: &Request) -> Result { let p = &req.params; let who = &actor.id; let team_id = text(p, "team_id")?; + let open = open_to_people(s, actor); ensure!( - s.teams - .get(team_id) - .is_some_and(|team| team.created_by == *who && team.members.contains(who)), + s.teams.get(team_id).is_some_and(|team| { + (team.created_by == *who || open) && team.members.contains(who) + }), "forbidden: team creator required" ); let name = Self::team_name_for(s, p, Some(team_id))?; @@ -3534,15 +3747,16 @@ impl Broker { team.name = name; Ok(json!(team)) } - /// `channel.rename {channel_id, name}`: the channel's current owner only, and not while - /// archived (an archived channel is read-only and keeps its name reserved). + /// `channel.rename {channel_id, name}`: the channel's current owner only, or in an open + /// workspace any of its members, and not while archived (an archived channel is read-only + /// and keeps its name reserved). fn mutate_channel_rename(&self, s: &mut State, actor: &Actor, req: &Request) -> Result { let p = &req.params; let who = &actor.id; let channel_id = text(p, "channel_id")?; let channel = self.channel(s, who, channel_id, true)?; ensure!( - channel.owner_id == *who, + channel.owner_id == *who || open_to_people(s, actor), "forbidden: current owner required" ); let team_id = channel.team_id.clone(); @@ -3583,6 +3797,126 @@ impl Broker { } Ok(json!(s.workspace)) } + /// `workspace.set_governance {governance: "open"|"managed"}` (`open_governance_v1`, OW-1): + /// the host only, as a person. It changes who may manage teams, channels and joiners, never + /// what any grant may read, so the epoch stays. Setting the governance the workspace already + /// has is a successful no-op. + fn mutate_workspace_set_governance( + &self, + s: &mut State, + actor: &Actor, + req: &Request, + ) -> Result { + self.manager(s, &actor.id)?; + ensure!( + actor.run.is_none(), + "forbidden: human host decision required" + ); + let governance = match req.params.get("governance").and_then(Value::as_str) { + Some("open") => Governance::Open, + Some("managed") => Governance::Managed, + _ => bail!("invalid_params: governance must be open or managed"), + }; + s.workspace.governance = governance; + Ok(json!({ "governance": governance })) + } + /// `workspace.delete {workspace_id, expected_name}` (`delete_v1`, OW-5): the host only, as a + /// person, naming the workspace by its ID and by its current name (`null` for an unnamed + /// one), both inside the signature. + /// + /// It commits the tombstone: every device, legacy enrollment, pending join and grant goes + /// and every run is revoked, so nobody gets in again, even through a broker without + /// `delete_v1` replaying this journal. From then on every request but `hello` is refused + /// [`WORKSPACE_DELETED`], and [`DELETE_GRACE_SECS`] later the broker erases its files and + /// exits ([`finish_deletion`]). + fn mutate_workspace_delete( + &self, + s: &mut State, + actor: &Actor, + req: &Request, + ) -> Result { + let p = &req.params; + self.manager(s, &actor.id)?; + ensure!( + actor.run.is_none(), + "forbidden: human host decision required" + ); + ensure!( + text(p, "workspace_id")? == s.workspace.id, + "invalid_params: That isn't this workspace's ID. Refresh and try again." + ); + let expected = match p.get("expected_name") { + Some(Value::Null) => None, + Some(Value::String(name)) => Some(name.as_str()), + _ => bail!("invalid_params: expected_name must be the workspace's name, or null for a workspace without one"), + }; + ensure!( + expected == s.workspace.name.as_deref(), + WORKSPACE_NAME_CHANGED + ); + s.workspace.deleted_at = Some(now()); + s.devices.clear(); + s.enrollments.clear(); + s.pending_joins.clear(); + s.grants.clear(); + for run in s.runs.values_mut() { + run.revoked = true; + } + s.workspace.policy_epoch += 1; + Ok(json!({ + "deleted": true, + "workspace_id": s.workspace.id, + "erase_after_seconds": DELETE_GRACE_SECS, + })) + } + /// `workspace.context.set {text, expected_revision}` (`workspace_context_v1`, OW-8): a + /// person replaces the workspace context document, the host in a managed workspace and any + /// member in an open one. `expected_revision` is the revision the editor started from (0 + /// before the first), so of two people editing at once the second is refused + /// [`CONTEXT_CONFLICT`] rather than silently overwriting the first. + /// + /// At most [`CONTEXT_MAX_BYTES`] of text; line breaks and tabs are kept and any other control + /// character is refused. The text counts toward its editor's share of the state while the + /// revision is theirs, and every revision stays in the journal. The epoch stays: the + /// document is reference information and authorizes nothing. + fn mutate_workspace_context_set( + &self, + s: &mut State, + actor: &Actor, + req: &Request, + ) -> Result { + let p = &req.params; + let who = &actor.id; + ensure!(actor.run.is_none(), PERSON_REQUIRED); + if !open_governance(s) { + self.manager(s, who)?; + } + let text = p + .get("text") + .and_then(Value::as_str) + .ok_or_else(|| anyhow!("invalid_params: text must be a string"))?; + ensure!( + text.len() <= CONTEXT_MAX_BYTES, + "invalid_params: The workspace context is longer than 64 KiB. Shorten it, then save it again." + ); + ensure!( + !text + .chars() + .any(|c| c.is_control() && !matches!(c, '\n' | '\r' | '\t')), + "invalid_params: The workspace context has a control character in it. Remove it, then save it again." + ); + let current = s.context.as_ref().map_or(0, |context| context.revision); + ensure!(number(p, "expected_revision")? == current, CONTEXT_CONFLICT); + let context = WorkspaceContext { + text: text.to_owned(), + revision: current + 1, + updated_by: who.clone(), + updated_at: now(), + }; + let result = json!({"revision": context.revision, "updated_at": context.updated_at}); + s.context = Some(context); + Ok(result) + } fn mutate_invitation_create( &self, s: &mut State, @@ -3598,14 +3932,21 @@ impl Broker { s.principals.get(principal).is_some_and(|p| p.active), "forbidden: principal unavailable" ); + // In an open workspace any member of the team or channel invites (OW-2). + let open = open_to_people(s, actor); match kind { "team" => ensure!( - s.teams.get(target).is_some_and(|t| t.created_by == *who), + s.teams + .get(target) + .is_some_and(|t| t.created_by == *who || (open && t.members.contains(who))), "forbidden: team owner required" ), "channel" => { let c = self.channel(s, who, target, true)?; - ensure!(c.owner_id == *who, "forbidden: current owner required"); + ensure!( + c.owner_id == *who || open, + "forbidden: current owner required" + ); ensure!( s.teams .get(&c.team_id) @@ -3662,13 +4003,21 @@ impl Broker { .filter(|i| i.principal_id == *who && i.expires_at >= now()) .ok_or_else(|| anyhow!("forbidden: invitation unavailable"))? .clone(); + // The inviter must still hold the authority they invited with: the team's creator or the + // channel's owner, or in an open workspace still an active member of it (OW-2). + let open = open_governance(s); + let inviter_active = s + .principals + .get(&invitation.inviter_id) + .is_some_and(|p| p.active); if invitation.kind == "team" { let team = s .teams .get_mut(&invitation.target_id) .ok_or_else(|| anyhow!("forbidden: team unavailable"))?; ensure!( - team.created_by == invitation.inviter_id, + team.created_by == invitation.inviter_id + || (open && inviter_active && team.members.contains(&invitation.inviter_id)), "forbidden: invitation authority changed" ); team.members.insert(who.clone()); @@ -3683,13 +4032,19 @@ impl Broker { .get_mut(&invitation.target_id) .ok_or_else(|| anyhow!("forbidden: channel unavailable"))?; ensure!( - c.owner_id == invitation.inviter_id && !c.archived, + (c.owner_id == invitation.inviter_id + || (open && inviter_active && c.members.contains(&invitation.inviter_id))) + && !c.archived, "forbidden: invitation authority changed" ); c.members.insert(who.clone()); } s.invitations.remove(&invitation.id); - s.workspace.policy_epoch += 1; + // OW-3: an addition in an open workspace widens no existing grant, so it leaves every + // agent running. In a managed one every membership change ends them, as it always has. + if !open { + s.workspace.policy_epoch += 1; + } Ok(json!({"accepted":true})) } /// `team.add_member {team_id, principal_id, expected_username, channel_ids?}` (direct add, @@ -3701,11 +4056,15 @@ impl Broker { /// Every check runs before anything changes, so one refused channel refuses the whole add. /// A member already in the team and every listed channel is a successful no-op that leaves /// the policy epoch alone; any real addition moves it, as `invitation.accept` does. + /// + /// In an open workspace (OW-2) any member of the team adds people, into the channels of it + /// they are in themselves, and an addition leaves the epoch alone (OW-3). fn mutate_team_add_member(&self, s: &mut State, actor: &Actor, req: &Request) -> Result { let p = &req.params; let who = &actor.id; ensure!(actor.run.is_none(), DIRECT_ADD_AGENT_REFUSED); let host = self.manager(s, who).is_ok(); + let open = open_governance(s); let team_id = text(p, "team_id")?; let team = s .teams @@ -3713,7 +4072,7 @@ impl Broker { .filter(|team| host || team.members.contains(who)) .ok_or_else(|| anyhow!("forbidden: team unavailable"))?; ensure!( - host || team.created_by == *who, + host || open || team.created_by == *who, "forbidden: Only the team's owner or the workspace host can add people to it." ); let principal_id = text(p, "principal_id")?; @@ -3738,7 +4097,7 @@ impl Broker { .filter(|c| c.team_id == team_id && (host || c.members.contains(who))) .ok_or_else(|| anyhow!(DIRECT_ADD_UNKNOWN_CHANNEL))?; ensure!( - host || channel.owner_id == *who, + host || open || channel.owner_id == *who, "forbidden: You can only add people to channels you own. Uncheck #{} and try again.", channel.name ); @@ -3772,7 +4131,9 @@ impl Broker { || (invitation.target_id != team_id && !added_channels.contains(&invitation.target_id)) }); - s.workspace.policy_epoch += 1; + if !open { + s.workspace.policy_epoch += 1; + } } Ok(json!({ "team_id": team_id, @@ -3782,9 +4143,10 @@ impl Broker { })) } /// `channel.add_member {channel_id, principal_id, expected_username}` (direct add): the - /// channel's owner or the workspace host adds a member of the channel's team. As - /// [`Self::mutate_team_add_member`], an existing member is a no-op that leaves the epoch - /// alone. + /// channel's owner or the workspace host adds a member of the channel's team, or in an open + /// workspace any member of the channel does. As [`Self::mutate_team_add_member`], an + /// existing member is a no-op that leaves the epoch alone, and so is any addition in an + /// open workspace. fn mutate_channel_add_member( &self, s: &mut State, @@ -3795,6 +4157,7 @@ impl Broker { let who = &actor.id; ensure!(actor.run.is_none(), DIRECT_ADD_AGENT_REFUSED); let host = self.manager(s, who).is_ok(); + let open = open_governance(s); let channel_id = text(p, "channel_id")?; let channel = if host { s.channels @@ -3804,7 +4167,7 @@ impl Broker { self.channel(s, who, channel_id, false)? }; ensure!( - host || channel.owner_id == *who, + host || open || channel.owner_id == *who, "forbidden: Only the channel's owner or the workspace host can add people to it." ); ensure!( @@ -3831,7 +4194,9 @@ impl Broker { s.invitations.retain(|_, invitation| { invitation.principal_id != target || invitation.target_id != channel_id }); - s.workspace.policy_epoch += 1; + if !open { + s.workspace.policy_epoch += 1; + } } Ok(json!({ "channel_id": channel_id, @@ -3879,7 +4244,10 @@ impl Broker { /// longer an active member of the workspace has nobody left to manage it, so the host /// archives it or removes its members, from outside it too /// ([`Self::stewards_orphaned_channel`]). Nobody removes a channel's owner, the host never - /// overrides an owner who is still a member, and only the owner transfers. + /// overrides an owner who is still a member, and only the owner transfers. In an open + /// workspace every member of the channel may archive it or remove anyone but its owner. + /// + /// Each of them bumps the epoch, open or not: a removal or an archive takes access away. fn mutate_channel_archive(&self, s: &mut State, actor: &Actor, req: &Request) -> Result { let p = &req.params; let who = &actor.id; @@ -3895,12 +4263,15 @@ impl Broker { }; let owner = c.owner_id == *who; let leaving = !owner && Self::is_leave(req, who); + // In an open workspace any member of the channel archives it or removes someone from + // it (OW-2). Nobody still removes its owner, and only the owner transfers it. + let open = method != "channel.transfer" && open_to_people(s, actor); ensure!( leaving || !c.archived, "channel_archived: channel is read-only" ); ensure!( - owner || leaving || stewarding, + owner || leaving || stewarding || open, "forbidden: current owner required" ); match method { @@ -3959,13 +4330,16 @@ impl Broker { req.method == "membership.revoke" && req.params.get("principal_id").and_then(Value::as_str) == Some(actor_id) } - /// Whether `actor` is the host, as a person, archiving (`channel.archive`) or removing a - /// member from (`membership.revoke`) a channel whose owner is no longer an active member of - /// the workspace, and so can act on it without being in it. An offboarded owner's channels - /// would otherwise stay as they are for good, with every member someone added to them. + /// Whether `actor` is the host, as a person, archiving (`channel.archive`), deleting + /// (`channel.delete`) or removing a member from (`membership.revoke`) a channel whose owner + /// is no longer an active member of the workspace, and so can act on it without being in + /// it. An offboarded owner's channels would otherwise stay as they are for good, with every + /// member someone added to them. fn stewards_orphaned_channel(&self, s: &State, actor: &Actor, req: &Request) -> bool { - matches!(req.method.as_str(), "channel.archive" | "membership.revoke") - && actor.run.is_none() + matches!( + req.method.as_str(), + "channel.archive" | "channel.delete" | "membership.revoke" + ) && actor.run.is_none() && self.manager(s, &actor.id).is_ok() && req .params @@ -3998,6 +4372,152 @@ impl Broker { s.workspace.policy_epoch += 1; Ok(json!(c)) } + /// `team.join {team_id}` (`open_governance_v1`): in an open workspace any member joins a + /// team, and its `#general`, on their own, and their invitations to either are spent. The + /// epoch stays, as for every addition in an open workspace (OW-3). In a managed workspace + /// the owner or the host adds people instead. + fn mutate_team_join(&self, s: &mut State, actor: &Actor, req: &Request) -> Result { + let who = &actor.id; + ensure!(actor.run.is_none(), PERSON_REQUIRED); + ensure!(open_governance(s), JOIN_NEEDS_OPEN); + let team_id = text(&req.params, "team_id")?; + let team = s + .teams + .get_mut(team_id) + .ok_or_else(|| anyhow!("forbidden: team unavailable"))?; + team.members.insert(who.clone()); + let general = team.general_channel_id.clone(); + s.channels + .get_mut(&general) + .ok_or_else(|| anyhow!("storage_corrupt"))? + .members + .insert(who.clone()); + s.invitations.retain(|_, invitation| { + invitation.principal_id != *who + || (invitation.target_id != team_id && invitation.target_id != general) + }); + Ok(json!({"team_id": team_id, "joined": true})) + } + /// `channel.join {channel_id}` (`open_governance_v1`): in an open workspace any member of a + /// channel's team joins it on their own, unless it is archived, and their invitations to it + /// are spent. The epoch stays (OW-3). + fn mutate_channel_join(&self, s: &mut State, actor: &Actor, req: &Request) -> Result { + let who = &actor.id; + ensure!(actor.run.is_none(), PERSON_REQUIRED); + ensure!(open_governance(s), JOIN_NEEDS_OPEN); + let channel_id = text(&req.params, "channel_id")?; + // A channel of a team the actor is not in reads exactly as one that doesn't exist. + let channel = s + .channels + .get(channel_id) + .filter(|channel| { + s.teams + .get(&channel.team_id) + .is_some_and(|team| team.members.contains(who)) + }) + .ok_or_else(|| anyhow!("forbidden: channel unavailable"))?; + ensure!( + !channel.archived, + "channel_archived: #{} is archived, so no one can join it.", + channel.name + ); + s.channels + .get_mut(channel_id) + .expect("checked channel") + .members + .insert(who.clone()); + s.invitations.retain(|_, invitation| { + invitation.principal_id != *who || invitation.target_id != channel_id + }); + Ok(json!({"channel_id": channel_id, "joined": true})) + } + /// `channel.delete {channel_id, expected_name?}` (`delete_v1`, OW-4): a person removes a + /// channel, archived or not: its owner, the host for a channel whose owner left the + /// workspace ([`Self::stewards_orphaned_channel`]), or in an open workspace any of its + /// members. A team's `#general` goes only with its team. `expected_name` (inside the + /// signature) must still be the channel's name, so a stale or rewritten snapshot cannot + /// steer the delete onto a channel that was renamed since. + /// + /// Only map entries are removed ([`remove_channel`]): the channel's messages stay in the + /// state and the journal, readable by nobody because their channel is gone. The epoch + /// stays, since every run that named the channel is revoked and nobody else's reads + /// change. + fn mutate_channel_delete(&self, s: &mut State, actor: &Actor, req: &Request) -> Result { + let p = &req.params; + let who = &actor.id; + ensure!(actor.run.is_none(), DELETE_AGENT_REFUSED); + let channel_id = text(p, "channel_id")?; + let stewarding = self.stewards_orphaned_channel(s, actor, req); + let c = if stewarding { + s.channels + .get(channel_id) + .ok_or_else(|| anyhow!("forbidden: channel unavailable"))? + } else { + self.channel(s, who, channel_id, false)? + }; + ensure!( + c.owner_id == *who || stewarding || open_governance(s), + "forbidden: current owner required" + ); + let team_id = c.team_id.clone(); + ensure!( + s.teams + .get(&team_id) + .is_some_and(|team| team.general_channel_id != channel_id), + GENERAL_DELETE + ); + check_expected_name(p, &c.name, Some('#'), CHANNEL_NAME_CHANGED)?; + let removed = remove_channel(s, channel_id); + Ok(json!({ + "deleted": true, + "channel_id": channel_id, + "team_id": team_id, + "removed_attachments": removed.attachments, + "removed_references": removed.references, + })) + } + /// `team.delete {team_id, expected_name?}` (`delete_v1`, OW-4): a person removes a team + /// with every channel in it, its `#general` and channels the deleter is not in included: + /// its creator while still in it, the host when the creator is no longer an active member, + /// or in an open workspace any member of the team. As [`Self::mutate_channel_delete`], map + /// entries only, and the epoch stays. + fn mutate_team_delete(&self, s: &mut State, actor: &Actor, req: &Request) -> Result { + let p = &req.params; + let who = &actor.id; + ensure!(actor.run.is_none(), DELETE_AGENT_REFUSED); + let team_id = text(p, "team_id")?; + let host = self.manager(s, who).is_ok(); + let team = s + .teams + .get(team_id) + .ok_or_else(|| anyhow!("forbidden: team unavailable"))?; + let member = team.members.contains(who); + let stewarding = host + && team.created_by != *who + && !s + .principals + .get(&team.created_by) + .is_some_and(|creator| creator.active); + ensure!(member || stewarding, "forbidden: team unavailable"); + ensure!( + stewarding || team.created_by == *who || open_governance(s), + "forbidden: team creator required" + ); + check_expected_name(p, &team.name, None, TEAM_NAME_CHANGED)?; + let channels: Vec = s + .channels + .values() + .filter(|channel| channel.team_id == team_id) + .map(|channel| channel.id.clone()) + .collect(); + for channel_id in &channels { + remove_channel(s, channel_id); + } + s.teams.remove(team_id); + s.invitations + .retain(|_, invitation| invitation.target_id != team_id); + Ok(json!({"deleted": true, "team_id": team_id, "channels": channels.len()})) + } fn message_restricted( &self, s: &State, @@ -4170,7 +4690,7 @@ impl Broker { let mut sources = strings(p, "source_channels")?; sources.insert(channel.into()); ensure!( - sources.len() <= 20, + sources.len() <= RUN_SOURCE_CHANNELS, "quota_exceeded: too many context channels" ); let consent = Self::run_policy_consent(s, p)?; @@ -4307,7 +4827,7 @@ impl Broker { let root = run .remote_root .as_ref() - .ok_or_else(|| anyhow!("forbidden: run has no remote path approval"))?; + .ok_or_else(|| anyhow!(NO_REMOTE_FOLDER))?; ensure!( Path::new(path).starts_with(root), "forbidden: reference outside approved remote root" @@ -4625,6 +5145,19 @@ fn prune_retained(s: &mut State, now: u64, quotas: &Quotas) -> Pruned { } Pruned { uploads, runs } } +/// The attachments whose files go once `after` is journaled: the unfinished uploads that aged +/// out (`expired`), and every other attachment `before` held and `after` does not, which +/// `channel.delete` and `team.delete` remove. +fn removed_attachments(before: &State, after: &State, mut expired: Vec) -> Vec { + let removed: Vec = before + .blobs + .keys() + .filter(|blob_id| !after.blobs.contains_key(*blob_id) && !expired.contains(*blob_id)) + .cloned() + .collect(); + expired.extend(removed); + expired +} /// What [`prune_retained`] removed that outlives the state. struct Pruned { /// Unfinished uploads, whose files are deleted once the commit lands. @@ -4683,8 +5216,9 @@ fn remember(s: &mut State, quotas: &Quotas, actor: &str, key: String, cached: Ca /// The serialized bytes of the records `principal` created and holds against /// [`Quotas::member_state_bytes`]: their messages (their agents' included), runs with their -/// grants, attachments, references, invitations sent, and the teams and channels they -/// created. Idempotency results and read positions are bounded separately. +/// grants, attachments, references, invitations sent, the teams and channels they created, +/// and the workspace context while the revision is theirs. Idempotency results and read +/// positions are bounded separately. fn member_state_bytes(s: &State, principal: &str) -> usize { fn total(items: impl Iterator) -> usize { items.fold(0, |sum, item| sum.saturating_add(json_len(&item))) @@ -4701,6 +5235,7 @@ fn member_state_bytes(s: &State, principal: &str) -> usize { total(s.invitations.values().filter(|i| i.inviter_id == principal)), total(s.teams.values().filter(|t| t.created_by == principal)), total(s.channels.values().filter(|c| c.created_by == principal)), + total(s.context.iter().filter(|c| c.updated_by == principal)), ] .into_iter() .fold(0, usize::saturating_add) @@ -4759,6 +5294,38 @@ fn shared_name<'a>(p: &'a Value, key: &str, what: SharedName) -> Result<&'a str> Ok(value) } +/// Whether the workspace is open (OW-1): every active member manages its teams and channels. +fn open_governance(s: &State) -> bool { + s.workspace.governance == Governance::Open +} +/// Whether an open workspace's relaxed rule applies to `actor`: the workspace is open and the +/// request is a person's own. An agent's grant never manages anything, open or not; the worker +/// allowlist refuses it first, and this is the second wall. +fn open_to_people(s: &State, actor: &Actor) -> bool { + open_governance(s) && actor.run.is_none() +} +/// How many of `members` are active members of the workspace. +fn active_count(s: &State, members: &BTreeSet) -> usize { + members + .iter() + .filter(|id| s.principals.get(*id).is_some_and(|p| p.active)) + .count() +} +/// OW-3: in an open workspace, a member is admitted (`auth.join`, legacy `auth.enroll`) into +/// every team and every unarchived channel at once. Adding them widens no existing grant, so +/// the epoch stays. A managed workspace adds them nowhere. +fn admit_everywhere(s: &mut State, principal_id: &str) { + if !open_governance(s) { + return; + } + for team in s.teams.values_mut() { + team.members.insert(principal_id.to_owned()); + } + for channel in s.channels.values_mut().filter(|channel| !channel.archived) { + channel.members.insert(principal_id.to_owned()); + } +} + /// The host's active principal, injected into the snapshot's `workspace` at projection time /// and never stored (it would be journaled otherwise). fn host_principal_id(s: &State) -> Option<&str> { @@ -4787,6 +5354,72 @@ fn check_expected_username(s: &State, params: &Value, principal_id: &str) -> Res Ok(()) } +/// If `params` carries `expected_name` (the team or channel name the person confirmed, inside +/// their signature, with one leading `sigil` allowed), `name` must still be exactly that, as +/// [`check_expected_username`] does for people; `changed` otherwise. +fn check_expected_name( + params: &Value, + name: &str, + sigil: Option, + changed: &'static str, +) -> Result<()> { + let expected = match params.get("expected_name") { + None | Some(Value::Null) => return Ok(()), + Some(Value::String(expected)) => expected.as_str(), + Some(_) => bail!("invalid_params: expected_name must be a string"), + }; + let expected = sigil + .and_then(|sigil| expected.strip_prefix(sigil)) + .unwrap_or(expected); + ensure!(expected == name, changed); + Ok(()) +} + +/// What [`remove_channel`] removed besides the channel itself. +struct RemovedChannel { + attachments: usize, + references: usize, +} + +/// Remove `channel_id` from the live state, for `channel.delete` and `team.delete` (OW-4), with +/// map removals only, which a broker without `delete_v1` replays: the channel, every attachment +/// and reference posted in it or derived from it (nobody could ever read them again), +/// invitations to it, and everyone's read position in it. Every run that writes to it or reads +/// it is revoked. +/// +/// Its messages stay. `messages` is one array, which the journal could only rewrite whole, and +/// once the channel is gone nobody can read them, nor any message elsewhere derived from it +/// ([`Broker::visible`]). The attachments' files go once the removal is journaled +/// ([`Broker::apply_mutation`]). +fn remove_channel(s: &mut State, channel_id: &str) -> RemovedChannel { + let derived = + |own: &str, sources: &BTreeSet| own == channel_id || sources.contains(channel_id); + s.channels.remove(channel_id); + let before = s.blobs.len(); + s.blobs + .retain(|_, blob| !derived(&blob.channel_id, &blob.source_channels)); + let attachments = before - s.blobs.len(); + let before = s.references.len(); + s.references + .retain(|_, reference| !derived(&reference.channel_id, &reference.source_channels)); + let references = before - s.references.len(); + s.invitations + .retain(|_, invitation| invitation.target_id != channel_id); + let suffix = format!(":{channel_id}"); + s.read_positions.retain(|key, _| !key.ends_with(&suffix)); + for run in s + .runs + .values_mut() + .filter(|run| derived(&run.channel_id, &run.source_channels)) + { + run.revoked = true; + } + RemovedChannel { + attachments, + references, + } +} + /// The teams and channels a snapshot lists ([`Broker::places_within`]), with their wires. struct Places<'s> { teams: Vec<&'s Team>, @@ -5316,6 +5949,25 @@ pub fn serve(root: &Path, bootstrap_key: &str, name: Option<&str>) -> Result<()> ); let node_id = node_identity()?; let mut broker = Broker::open_inner(root, bootstrap_key, Box::new(SystemDirectory), name)?; + if broker.state.workspace.deleted_at.is_some() { + // The broker stopped between the tombstone and the erase (OW-5). Nobody can be + // connected to a broker that is starting, so the erase is finished now. + let workspace_id = broker.state.workspace.id.clone(); + let runtime = broker.state.runtime_basename.clone(); + let leftovers = erase_store(&broker.root, &workspace_id)?; + if let Some(basename) = runtime { + remove_stale_runtime( + &broker.runtime_root, + broker.state.workspace.host_uid, + &basename, + ); + } + println!( + "{}", + json!({"deleted": true, "workspace_id": workspace_id, "leftovers": leftovers}) + ); + return Ok(()); + } if let Some(name) = name { match broker.state.workspace.name.as_deref() { Some(stored) if stored == name => {} @@ -5392,24 +6044,195 @@ fn path_identity(path: &Path) -> Option<(u64, u64)> { fn runtime_intact(socket: &Path, bound: (u64, u64)) -> bool { path_identity(socket) == Some(bound) } +/// How often a running broker checks whether a workspace its host deleted is due to be erased. +const DELETE_WATCH_INTERVAL: Duration = Duration::from_secs(1); /// Stop the broker once its socket path no longer leads to the socket it bound: the runtime /// directory was removed (by root or a `/tmp` cleaner) and perhaps re-created by another /// account. Nobody can connect any more and `stop` can no longer verify the process, while it /// still holds the writer lock, so it exits and lets the next `start` restore the path or move /// the workspace to a new one. It waits for the request in progress, so no commit is cut short. +/// +/// The same thread ends a deleted workspace ([`finish_deletion`]) once its grace period is +/// over, so it wakes every [`DELETE_WATCH_INTERVAL`] and looks at the socket every +/// [`RUNTIME_WATCH_INTERVAL`]. fn watch_runtime(socket: PathBuf, bound: (u64, u64), broker: Arc>) { - std::thread::spawn(move || loop { - std::thread::sleep(RUNTIME_WATCH_INTERVAL); - if !runtime_intact(&socket, bound) { - let _held = broker.lock(); - eprintln!( - "runtime_lost: {} no longer leads to this broker; stopping so the next start can restore or move it", - socket.display() - ); - std::process::exit(75); + std::thread::spawn(move || { + let mut since_check = Duration::ZERO; + loop { + std::thread::sleep(DELETE_WATCH_INTERVAL); + since_check += DELETE_WATCH_INTERVAL; + if let Ok(held) = broker.lock() { + if held.erase_due(now()) { + finish_deletion(&held, &socket, bound); + } + } + if since_check < RUNTIME_WATCH_INTERVAL { + continue; + } + since_check = Duration::ZERO; + if !runtime_intact(&socket, bound) { + let _held = broker.lock(); + eprintln!( + "runtime_lost: {} no longer leads to this broker; stopping so the next start can restore or move it", + socket.display() + ); + std::process::exit(75); + } } }); } +/// The end of a workspace its host deleted (OW-5), once connected members have had +/// [`DELETE_GRACE_SECS`] to hear `workspace_deleted`: erase the state directory +/// ([`erase_store`]), remove the runtime socket and its directory if they are still the ones +/// this broker bound, and exit. Called with the broker's lock held, so no request is answered +/// in between. +fn finish_deletion(broker: &Broker, socket: &Path, bound: (u64, u64)) -> ! { + let workspace_id = &broker.state.workspace.id; + let root = broker.root.display(); + let erased = erase_store(&broker.root, workspace_id); + match &erased { + Ok(leftovers) if leftovers.is_empty() => { + eprintln!("workspace_deleted: erased workspace {workspace_id} from {root}") + } + Ok(leftovers) => eprintln!( + "workspace_deleted: erased workspace {workspace_id}; {root} stays, holding files Crew did not create: {}", + leftovers.join(", ") + ), + Err(error) => eprintln!( + "workspace_deleted: could not finish erasing workspace {workspace_id} ({error:#}); finish with biorouter-crew delete --state-dir {root} --workspace-id {workspace_id}" + ), + } + if runtime_intact(socket, bound) { + let _ = fs::remove_file(socket); + if let Some(directory) = socket.parent() { + let _ = fs::remove_dir(directory); + } + } + std::process::exit(if erased.is_ok() { 0 } else { 74 }) +} +/// The workspace ID in `root`'s [`DELETED_MARKER`], or `None` when there is no marker: an +/// erase that a crash interrupted left it. +fn deleted_marker(root: &Path) -> Result> { + let file = match OpenOptions::new() + .read(true) + .custom_flags(libc::O_NOFOLLOW | libc::O_CLOEXEC) + .open(root.join(DELETED_MARKER)) + { + Ok(file) => file, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None), + Err(error) => { + return Err(error).context("unsafe_storage: cannot read the deleted-workspace marker") + } + }; + let mut id = String::new(); + file.take(128).read_to_string(&mut id)?; + Ok(Some(id.trim().to_owned())) +} +/// Erase a deleted workspace's state directory `root`, removing only the files the broker +/// creates and never anything recursively, since `--state-dir` may name a directory that held +/// other files first. The [`DELETED_MARKER`] goes in first and comes out last, so a crash part +/// of the way leaves a directory every later open refuses, never one a `start` would quietly +/// fill with a new workspace. +/// +/// A file is removed only when it is a regular file this account owns with one link. What is +/// left (anything else, and whatever a removal failed for) keeps `root` in place, and the +/// names are returned: the leftovers. +fn erase_store(root: &Path, workspace_id: &str) -> Result> { + let euid = unsafe { libc::geteuid() }; + let marker = root.join(DELETED_MARKER); + let mut file = private_file(&marker, false)?; + file.set_len(0)?; + file.write_all(workspace_id.as_bytes())?; + file.sync_all()?; + sync_dir(root)?; + let ours = |path: &Path| { + fs::symlink_metadata(path).is_ok_and(|m| m.is_file() && m.uid() == euid && m.nlink() == 1) + }; + let blobs = root.join("blobs"); + if fs::symlink_metadata(&blobs).is_ok_and(|m| m.is_dir() && m.uid() == euid) { + for entry in fs::read_dir(&blobs)? { + let path = entry?.path(); + if ours(&path) { + let _ = fs::remove_file(&path); + } + } + let _ = fs::remove_dir(&blobs); + } + for entry in fs::read_dir(root)? { + let entry = entry?; + let created = entry.file_name().to_str().is_some_and(|name| { + matches!(name, "journal.jsonl" | "runtime.json" | "broker.log") + || name.starts_with("torn-tail-") + || (name.starts_with(".runtime-") && name.ends_with(".tmp")) + }); + if created && ours(&entry.path()) { + let _ = fs::remove_file(entry.path()); + } + } + // Still held by a running broker, which is fine: unlinking a locked file releases nothing. + let lock = root.join("writer.lock"); + if ours(&lock) { + let _ = fs::remove_file(&lock); + } + fs::remove_file(&marker)?; + sync_dir(root)?; + if fs::remove_dir(root).is_ok() { + return Ok(Vec::new()); + } + let mut leftovers = Vec::new(); + for entry in fs::read_dir(root)? { + let entry = entry?; + let name = entry.file_name().to_string_lossy().into_owned(); + match fs::read_dir(entry.path()) { + Ok(inside) if name == "blobs" => leftovers.extend( + inside + .filter_map(|entry| entry.ok()) + .map(|entry| format!("blobs/{}", entry.file_name().to_string_lossy())), + ), + _ => leftovers.push(name), + } + } + leftovers.sort(); + Ok(leftovers) +} +/// Remove a deleted workspace's runtime directory `runtime_root/basename` when nothing listens +/// in it: a directory `uid` owns that no other account can write into, holding at most a stale +/// `broker.sock` of `uid`'s. Anything else is left exactly as it is, as +/// [`reclaim_runtime_socket`] leaves it. +fn remove_stale_runtime(runtime_root: &Path, uid: u32, basename: &str) { + use std::os::unix::fs::FileTypeExt; + if !runtime_basename_of(basename, uid) { + return; + } + let directory = runtime_root.join(basename); + let Ok(metadata) = fs::symlink_metadata(&directory) else { + return; + }; + if !metadata.is_dir() || metadata.uid() != uid || metadata.mode() & 0o022 != 0 { + return; + } + let Ok(entries) = fs::read_dir(&directory) else { + return; + }; + for entry in entries { + if !entry.is_ok_and(|entry| entry.file_name() == "broker.sock") { + return; + } + } + let socket = directory.join("broker.sock"); + if let Ok(metadata) = fs::symlink_metadata(&socket) { + if !(metadata.file_type().is_socket() && metadata.uid() == uid) { + return; + } + match UnixStream::connect(&socket) { + Err(error) if error.kind() == std::io::ErrorKind::ConnectionRefused => { + let _ = fs::remove_file(&socket); + } + _ => return, + } + } + let _ = fs::remove_dir(&directory); +} fn serve_client(stream: UnixStream, uid: u32, broker: Arc>) -> Result<()> { let mut connection = Connection::new(); let served = serve_requests(stream, uid, &broker, &mut connection); @@ -5746,9 +6569,75 @@ pub fn lifecycle(command: &str, root: &Path, key: &str, name: Option<&str>) -> R } } +/// `biorouter-crew delete (--state-dir PATH | --name NAME) --workspace-id UUID` (OW-5): erase a +/// stopped broker's workspace, as `workspace.delete` does for a running one, and finish an +/// erase a crash interrupted. The ID is the confirmation: it must be the one the state +/// directory holds, in its journal or in the [`DELETED_MARKER`] the interrupted erase left. +/// Refused `writer_active` while a broker holds the directory. Answers `{deleted, workspace_id, +/// leftovers}`, the leftovers being what the broker did not create and so left in place. +pub fn delete_workspace(root: &Path, workspace_id: &str) -> Result { + match fs::symlink_metadata(root) { + Ok(metadata) => ensure!( + metadata.is_dir() + && metadata.uid() == unsafe { libc::geteuid() } + && metadata.mode() & 0o077 == 0, + "unsafe_storage: state directory must be owner-only and not a symlink" + ), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => bail!( + "not_found: There is no workspace state directory at {}.", + root.display() + ), + Err(error) => return Err(error.into()), + } + // Past an interrupted erase the marker names the workspace, and the journal may already be + // gone or cut short; before one, the journal is what names it. + let marker = deleted_marker(root)?; + let stored = match (&marker, stored_workspace(root)) { + (Some(_), stored) => stored.ok().flatten(), + (None, stored) => stored?, + }; + let held = marker + .or_else(|| stored.as_ref().map(|stored| stored.id.clone())) + .ok_or_else(|| anyhow!("not_found: This state directory holds no workspace."))?; + ensure!( + held == workspace_id, + "identity_mismatch: This state directory holds workspace {held}, not {workspace_id}." + ); + // The same lock a broker holds for as long as it runs. + let lock = private_file(&root.join("writer.lock"), false)?; + ensure!( + unsafe { libc::flock(lock.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) } == 0, + "writer_active: A broker is running from this state directory. Stop it first: biorouter-crew stop --state-dir {}", + root.display() + ); + let _lock = WriterLock { + file: lock, + owner_pid: unsafe { libc::getpid() }, + }; + // The runtime directory the workspace last listened in, found before the erase removes + // the journal and the runtime descriptor that name it. + let runtime = stored + .and_then(|stored| stored.runtime_basename) + .or_else(|| { + read_runtime_descriptor(root) + .ok() + .flatten() + .and_then(|info| { + let socket = PathBuf::from(info.get("socket")?.as_str()?); + Some(socket.parent()?.file_name()?.to_str()?.to_owned()) + }) + }); + let leftovers = erase_store(root, workspace_id)?; + if let Some(basename) = runtime { + remove_stale_runtime(Path::new("/tmp"), unsafe { libc::geteuid() }, &basename); + } + Ok(json!({"deleted": true, "workspace_id": workspace_id, "leftovers": leftovers})) +} + /// What `status` prints for the broker `info` describes, which answered `hello`: the runtime -/// descriptor with `"state":"running"` and the workspace's name, or, when the broker has -/// stopped saving changes, a refusal telling the host what to do. +/// descriptor with `"state":"running"` (or `"deleted"`, while a deleted workspace counts down +/// to its erase) and the workspace's name, or, when the broker has stopped saving changes, a +/// refusal telling the host what to do. fn status_answer(root: &Path, mut info: Value, hello: &Value) -> Result { if hello.get("state").and_then(Value::as_str) == Some("storage_failed") { let full = hello["storage"]["code"].as_str() == Some("storage_full"); @@ -5772,7 +6661,11 @@ fn status_answer(root: &Path, mut info: Value, hello: &Value) -> Result { fault.host_instruction(&root) ); } - info["state"] = json!("running"); + info["state"] = if hello.get("state").and_then(Value::as_str) == Some("deleted") { + json!("deleted") + } else { + json!("running") + }; info["name"] = hello.get("name").cloned().unwrap_or(Value::Null); Ok(info) } diff --git a/crates/biorouter-crew/src/broker/join.rs b/crates/biorouter-crew/src/broker/join.rs index 6bac4a916..337ee49cd 100644 --- a/crates/biorouter-crew/src/broker/join.rs +++ b/crates/biorouter-crew/src/broker/join.rs @@ -26,8 +26,9 @@ //! - `hello` advertises `join_by_name_v1` whenever this module is compiled. //! //! Account lookups go through `broker.directory` (the [`Directory`] seam), never through libc -//! directly, and only after `broker.manager(..)` has admitted the caller. `enrollment.pending` -//! makes none; `auth.join` makes exactly one, the fresh `by_uid` of check 2. +//! directly, and only after [`human_manager`] has admitted the caller: the host, or in an open +//! workspace any member (OW-2). `enrollment.pending` makes none; `auth.join` makes exactly one, +//! the fresh `by_uid` of check 2. use super::*; use unicode_properties::{GeneralCategory, UnicodeGeneralCategory}; @@ -280,7 +281,8 @@ fn code_matches(s: &State, join: &PendingJoin, key: &[u8; 32]) -> Result { /// Bind the key: a new principal (nickname = username, D2) or the existing one for an added /// device, the device labeled `invitation_code`, the join and any legacy enrollment for the -/// UID removed, committed as `uid:`. +/// UID removed, committed as `uid:`. In an open workspace a new principal also joins every +/// team and every unarchived channel (OW-3). fn bind( broker: &mut Broker, join: &PendingJoin, @@ -308,6 +310,7 @@ fn bind( state .principals .insert(principal.id.clone(), principal.clone()); + admit_everywhere(&mut state, &principal.id); principal } }; @@ -354,10 +357,14 @@ pub(super) fn mutate( Some(result) } -/// Only the host's own signed device: never an agent's grant, and checked before any account -/// lookup. +/// Only the host's own signed device, or in an open workspace any member's (OW-2; every signed +/// request is an active member's): never an agent's grant, and checked before any account +/// lookup. The approved code must still be the joiner's own device code, so whoever approves, +/// the joiner's screen decides which key is let in. fn human_manager(broker: &Broker, s: &State, actor: &Actor) -> Result<()> { - broker.manager(s, &actor.id)?; + if !open_governance(s) { + broker.manager(s, &actor.id)?; + } ensure!(actor.run.is_none(), AGENT_REFUSED); Ok(()) } diff --git a/crates/biorouter-crew/src/lib.rs b/crates/biorouter-crew/src/lib.rs index fc1fe6b36..8be074507 100644 --- a/crates/biorouter-crew/src/lib.rs +++ b/crates/biorouter-crew/src/lib.rs @@ -102,6 +102,25 @@ impl ProviderAffiliation { } } +/// Who manages a workspace's teams, channels and joiners (OW-1, OW-2). +/// +/// `Managed` is today's rules: owners, team creators and the host manage. `Open` lets every +/// active member rename, archive and delete teams and channels, add and remove people in them, +/// join any of them, and invite and admit joiners; removing someone from the workspace, privacy, +/// governance, renaming and deleting the workspace stay the host's. +#[derive(Clone, Copy, Debug, Default, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum Governance { + #[default] + Managed, + Open, +} +impl Governance { + pub fn is_managed(&self) -> bool { + *self == Self::Managed + } +} + #[derive(Clone, Debug, Serialize, Deserialize)] pub struct Workspace { pub id: String, @@ -115,6 +134,29 @@ pub struct Workspace { /// existed replays and re-serializes byte for byte, and an older broker ignores the field. #[serde(default, skip_serializing_if = "Option::is_none")] pub name: Option, + /// Serialized only when `open`, so a journal written before governance existed replays and + /// re-serializes byte for byte, and an older broker (which ignores the field) enforces the + /// managed rules, which are stricter. + #[serde(default, skip_serializing_if = "Governance::is_managed")] + pub governance: Governance, + /// When the host deleted the workspace (seconds since the Unix epoch): the tombstone + /// `workspace.delete` commits before the broker erases its files. Absent until then. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub deleted_at: Option, +} + +/// The workspace's context document (OW-8): Markdown the members write about the server, which +/// every member's agent receives as reference information. Stored by the broker, at most +/// 64 KiB of text. +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +pub struct WorkspaceContext { + pub text: String, + /// 1 for the first text, one more for every change. + pub revision: u64, + /// The principal who wrote this revision. + pub updated_by: String, + /// When this revision was written (seconds since the Unix epoch). + pub updated_at: u64, } #[derive(Clone, Debug, Serialize, Deserialize)] pub struct Principal { @@ -369,7 +411,9 @@ fn canonical(value: &Value) -> Value { #[cfg(unix)] mod broker; #[cfg(unix)] -pub use broker::{bridge, lifecycle, serve, Account, Broker, Connection, Directory}; +pub use broker::{ + bridge, delete_workspace, lifecycle, serve, Account, Broker, Connection, Directory, +}; #[cfg(all(unix, feature = "test-seams"))] pub use broker::{JournalCall, Quotas}; diff --git a/crates/biorouter-crew/src/main.rs b/crates/biorouter-crew/src/main.rs index 7418fcf84..08329feb2 100644 --- a/crates/biorouter-crew/src/main.rs +++ b/crates/biorouter-crew/src/main.rs @@ -1,7 +1,7 @@ use anyhow::{bail, Result}; #[cfg(unix)] -const USAGE: &str = "usage: biorouter-crew serve|start|status|stop (--state-dir PATH | --name NAME) [--name NAME] [--bootstrap-key HEX]; bridge --stdio --socket PATH --owner-uid UID --workspace-id UUID"; +const USAGE: &str = "usage: biorouter-crew serve|start|status|stop (--state-dir PATH | --name NAME) [--name NAME] [--bootstrap-key HEX]; delete (--state-dir PATH | --name NAME) --workspace-id UUID; bridge --stdio --socket PATH --owner-uid UID --workspace-id UUID"; /// Where a named workspace keeps its state when `--state-dir` is not given: /// `$HOME/.local/share/biorouter-crew/`. The name is validated first, so it is a plain @@ -31,7 +31,7 @@ fn main() -> Result<()> { return Ok(()); } if matches!(command, "help" | "--help" | "-h") { - println!("Usage: biorouter-crew start --name NAME --bootstrap-key HEX [--state-dir PATH]\n biorouter-crew serve|start|status|stop --state-dir PATH [--name NAME] [--bootstrap-key HEX]\n biorouter-crew bridge --stdio --socket PATH --owner-uid UID --workspace-id UUID\n biorouter-crew --version\n--name names the workspace (lowercase letters, numbers and hyphens); without --state-dir its state lives in ~/.local/share/biorouter-crew/NAME. start prints the workspace's invitation line (brcrew1:...) to paste into Crew.\nBroker and bridge operations require Linux."); + println!("Usage: biorouter-crew start --name NAME --bootstrap-key HEX [--state-dir PATH]\n biorouter-crew serve|start|status|stop --state-dir PATH [--name NAME] [--bootstrap-key HEX]\n biorouter-crew delete (--state-dir PATH | --name NAME) --workspace-id UUID\n biorouter-crew bridge --stdio --socket PATH --owner-uid UID --workspace-id UUID\n biorouter-crew --version\n--name names the workspace (lowercase letters, numbers and hyphens); without --state-dir its state lives in ~/.local/share/biorouter-crew/NAME. start prints the workspace's invitation line (brcrew1:...) to paste into Crew. delete erases a stopped workspace's state directory, only the files Crew created; --workspace-id must be the ID of the workspace it holds.\nBroker and bridge operations require Linux."); return Ok(()); } @@ -52,7 +52,7 @@ fn main() -> Result<()> { .parse()?, option("--workspace-id").ok_or_else(|| anyhow::anyhow!("--workspace-id required"))?, )?; - } else if matches!(command, "serve" | "start" | "status" | "stop") { + } else if matches!(command, "serve" | "start" | "status" | "stop" | "delete") { let name = option("--name"); if args.iter().any(|arg| arg == "--name") && name.is_none() { bail!("--name requires a workspace name"); @@ -77,7 +77,12 @@ fn main() -> Result<()> { ), }; let key = option("--bootstrap-key").unwrap_or(""); - if command == "serve" { + if command == "delete" { + let workspace_id = option("--workspace-id").ok_or_else(|| { + anyhow::anyhow!("--workspace-id required: the ID of the workspace to delete") + })?; + println!("{}", biorouter_crew::delete_workspace(&root, workspace_id)?); + } else if command == "serve" { biorouter_crew::serve(&root, key, name)?; } else { let name = if command == "start" { name } else { None }; From 12402d4bcd95ee2da184f49ee3b4d728e557e226 Mon Sep 17 00:00:00 2001 From: Wanjun Gu Date: Fri, 16 Oct 2026 19:57:50 -0700 Subject: [PATCH 08/32] crew(ui): start an agent task from the composer with @agent, /agent and /task (OW-10) --- .../Composer.agentInvocation.test.tsx | 496 ++++++++++++++++ .../src/components/crew/composer/Composer.tsx | 533 +++++++++++++++++- .../composer/ComposerSuggestions.test.tsx | 219 +++++++ .../crew/composer/ComposerSuggestions.tsx | 267 +++++++++ .../crew/composer/agentInvocation.test.ts | 142 +++++ .../crew/composer/agentInvocation.ts | 154 +++++ .../src/components/crew/composer/composer.css | 106 ++++ .../src/components/crew/composer/copy.ts | 58 ++ 8 files changed, 1959 insertions(+), 16 deletions(-) create mode 100644 ui/desktop/src/components/crew/composer/Composer.agentInvocation.test.tsx create mode 100644 ui/desktop/src/components/crew/composer/ComposerSuggestions.test.tsx create mode 100644 ui/desktop/src/components/crew/composer/ComposerSuggestions.tsx create mode 100644 ui/desktop/src/components/crew/composer/agentInvocation.test.ts create mode 100644 ui/desktop/src/components/crew/composer/agentInvocation.ts diff --git a/ui/desktop/src/components/crew/composer/Composer.agentInvocation.test.tsx b/ui/desktop/src/components/crew/composer/Composer.agentInvocation.test.tsx new file mode 100644 index 000000000..351de4f35 --- /dev/null +++ b/ui/desktop/src/components/crew/composer/Composer.agentInvocation.test.tsx @@ -0,0 +1,496 @@ +import { act, fireEvent, screen, waitFor, within } from '@testing-library/react'; +import userEvent from '@testing-library/user-event'; +import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest'; +import { CrewHttpError, type CrewMessage, type Snapshot } from '../crewApi'; +import { + channelReady, + connection, + currentCrew, + ids, + installDaemon, + mocked, + ownedRun, + renderCrew, + richSnapshot, + type ScriptedDaemon, +} from '../integration/harness'; +import { agentCopy, unknownOutcomeCopy } from '../pane/copy'; +import { crewActionCopy } from '../state/copy'; +import { installResizeObserverStub } from '../test/crewTestUtils'; +import { composerCopy } from './copy'; + +vi.mock('../crewApi', async () => { + const actual = await vi.importActual('../crewApi'); + return { ...actual, crewHttp: vi.fn(), crewRequest: vi.fn(), observeCrew: vi.fn() }; +}); +// Stable across renders, as the real context's callbacks are; each test sets what they answer. +const config = vi.hoisted(() => { + const settings = { + providers: [] as unknown[], + values: {} as Record, + }; + return { + settings, + api: { + getProviders: async () => settings.providers, + read: async (key: string) => settings.values[key] ?? null, + getProviderModels: async () => [], + }, + }; +}); +vi.mock('../../ConfigContext', async () => { + const actual = await vi.importActual('../../ConfigContext'); + return { ...actual, useConfig: () => config.api }; +}); +vi.mock('../CrewAuthentication', () => ({ default: () =>
})); + +installResizeObserverStub(); + +const original = Element.prototype.scrollIntoView; +beforeAll(() => { + Element.prototype.scrollIntoView = vi.fn(); +}); +afterAll(() => { + Element.prototype.scrollIntoView = original; +}); + +/** + * `@agent`, `/agent` and `/task` in the composer (design OW-10), in the real layout against the + * real controller: the consent line says what Enter does before Enter, Enter starts the task with + * Ask my agent's own request, and every case the pane must decide opens the pane instead. + * SECURITY-SENSITIVE (human review): nothing but the person's own draft ever starts a task. + */ + +const versa = { + name: 'versa_azure', + is_configured: true, + resolved_tier: 'private', + affiliation: { kind: 'institutions', institutions: [{ id: 'ucsf', display_name: 'UCSF' }] }, + metadata: { display_name: 'Versa', known_models: [{ name: 'gpt-5.5' }] }, +}; +const openRouter = { + name: 'openrouter', + is_configured: true, + resolved_tier: 'public', + metadata: { display_name: 'OpenRouter', known_models: [{ name: 'free-model' }] }, +}; + +const RUNS = `/connections/${connection.id}/runs`; + +function runPosts(): Record[] { + return mocked.crewHttp.mock.calls + .filter(([path, method]) => path === RUNS && method === 'POST') + .map(([, , body]) => body as Record); +} +const posts = () => mocked.crewRequest.mock.calls.filter(([, method]) => method === 'message.post'); + +const consent = () => document.querySelector('.crew-compose-agent'); +const startButton = () => screen.getByRole('button', { name: composerCopy.startTask }); + +function enter(composer: HTMLElement) { + fireEvent.keyDown(composer, { key: 'Enter', code: 'Enter', keyCode: 13 }); +} + +async function typeTask(composer: HTMLElement, body: string) { + fireEvent.change(composer, { target: { value: body } }); + // The models are read the first time a draft could need them. + await waitFor(() => expect(consent()).not.toBeNull()); +} + +let daemon: ScriptedDaemon; + +beforeEach(() => { + vi.clearAllMocks(); + window.localStorage.clear(); + config.settings.providers = [versa, openRouter]; + config.settings.values = { + BIOROUTER_PROVIDER: 'openrouter', + BIOROUTER_MODEL: 'free-model', + CREW_TASK_MODEL: { provider: 'versa_azure', model: 'gpt-5.5' }, + }; + daemon = installDaemon({ + snapshot: richSnapshot({ pending_joins: [] }), + http: (path, method) => + path === RUNS && method === 'POST' ? { ...ownedRun, status: 'running' } : undefined, + }); +}); + +describe('@agent in the composer (OW-10)', () => { + it('starts the task on the model for Crew tasks with the words after the trigger, and posts nothing', async () => { + renderCrew(); + const composer = await channelReady(); + await typeTask(composer, '@agent summarize counts.csv'); + await waitFor(() => + expect(consent()).toHaveTextContent('Enter starts your agent with gpt-5.5 · Versa') + ); + expect(consent()).toHaveTextContent( + 'It reads #general and posts its result there, for this task only.' + ); + // The model's tier mark, as Ask my agent shows it. + expect(consent()?.querySelector('[data-crew-model-tier="private"]')).not.toBeNull(); + + enter(composer); + await waitFor(() => expect(runPosts()).toHaveLength(1)); + expect(runPosts()[0]).toMatchObject({ + channel_id: ids.general, + prompt: 'summarize counts.csv', + provider: 'versa_azure', + model: 'gpt-5.5', + context_channels: [ids.general], + posting_grant: true, + expected_mode: 'private', + expected_workspace_policy_epoch: 1, + }); + expect(typeof runPosts()[0].request_id).toBe('string'); + await waitFor(() => expect(composer).toHaveValue('')); + expect(posts()).toHaveLength(0); + expect(screen.queryByLabelText(agentCopy.task)).toBeNull(); + }); + + it.each([['/agent'], ['/task']])('starts the same way from %s', async (trigger) => { + renderCrew(); + const composer = await channelReady(); + await typeTask(composer, `${trigger} plot counts by sample`); + await waitFor(() => expect(consent()).toHaveTextContent('Enter starts your agent')); + fireEvent.click(startButton()); + await waitFor(() => expect(runPosts()).toHaveLength(1)); + expect(runPosts()[0].prompt).toBe('plot counts by sample'); + }); + + it('renames Send only while the draft starts a task, described by the consent line', async () => { + renderCrew(); + const composer = await channelReady(); + expect(screen.getByRole('button', { name: composerCopy.send })).toBeInTheDocument(); + await typeTask(composer, '@agent tally the plates'); + const start = startButton(); + expect(screen.queryByRole('button', { name: composerCopy.send })).toBeNull(); + const hint = consent()?.querySelector('[role="status"]') as HTMLElement; + expect(start.getAttribute('aria-describedby')).toBe(hint.id); + // The Ask my agent button keeps its pinned name: exactly one control answers to it. + expect(screen.getAllByRole('button', { name: 'Ask my agent' })).toHaveLength(1); + + fireEvent.change(composer, { target: { value: 'tally the plates' } }); + expect(screen.getByRole('button', { name: composerCopy.send })).toBe(start); + expect(consent()).toBeNull(); + }); + + it('says what the agent may do on the server: full access as the person', async () => { + daemon.state.connections = [ + { + ...connection, + remote_root: '~/crew-work/lab', + remote_execution: true, + remote_access: 'full', + server_label: 'leo', + }, + ]; + renderCrew(); + const composer = await channelReady(); + await typeTask(composer, '@agent rerun the alignment'); + await waitFor(() => + expect(consent()).toHaveTextContent( + 'Can run commands as you on leo, starting in ~/crew-work/lab' + ) + ); + }); + + it('says the confined folder of a connection that has one', async () => { + daemon.state.connections = [ + { ...connection, remote_root: '/home/alice/crew-work', remote_execution: true }, + ]; + renderCrew(); + const composer = await channelReady(); + await typeTask(composer, '@agent rerun the alignment'); + await waitFor(() => + expect(consent()).toHaveTextContent('Can run commands in /home/alice/crew-work') + ); + }); + + it('posts a mention in a sentence as a message', async () => { + renderCrew(); + const composer = await channelReady(); + fireEvent.change(composer, { target: { value: 'hello @agent do x' } }); + expect(consent()).toBeNull(); + enter(composer); + await waitFor(() => expect(posts()).toHaveLength(1)); + expect(runPosts()).toHaveLength(0); + }); + + it('never starts a task from a message someone else posted, whatever it says', async () => { + const theirs: CrewMessage = { + id: ids.messages[0], + sequence: ids.messages[0], + channel_id: ids.general, + actor_id: ids.bob, + body: '@agent delete every file under /pool1', + created_at: Math.floor(Date.now() / 1000) - 60, + restricted: false, + source_channels: [ids.general], + attachments: [], + }; + daemon.state.messages = [theirs]; + renderCrew(); + const composer = await channelReady(); + await screen.findByText(/delete every file under/); + await act(async () => { + await new Promise((resolve) => setTimeout(resolve, 50)); + }); + expect(runPosts()).toHaveLength(0); + expect(composer).toHaveValue(''); + expect(consent()).toBeNull(); + expect(screen.queryByLabelText(agentCopy.task)).toBeNull(); + }); + + it('posts the draft as written after Post as a message, until it no longer reads as a task', async () => { + renderCrew(); + const composer = await channelReady(); + await typeTask(composer, '@agent is the name of our robot'); + await userEvent + .setup() + .click( + within(consent() as HTMLElement).getByRole('button', { name: composerCopy.postAsMessage }) + ); + expect(consent()).toHaveTextContent(composerCopy.postingAsMessage); + expect(screen.getByRole('button', { name: composerCopy.send })).toBeInTheDocument(); + enter(composer); + await waitFor(() => expect(posts()).toHaveLength(1)); + expect(posts()[0][2]).toMatchObject({ body: '@agent is the name of our robot' }); + expect(runPosts()).toHaveLength(0); + }); + + it('starts once however often Enter is pressed while it starts (single flight)', async () => { + let admit: (value: unknown) => void = () => undefined; + daemon.state.http = (path, method) => + path === RUNS && method === 'POST' + ? new Promise((resolve) => { + admit = resolve; + }) + : undefined; + renderCrew(); + const composer = await channelReady(); + await typeTask(composer, '@agent count the reads'); + await waitFor(() => expect(consent()).toHaveTextContent('Enter starts your agent')); + enter(composer); + enter(composer); + await waitFor(() => expect(startButton()).toBeDisabled()); + expect(startButton()).toHaveAttribute('aria-busy', 'true'); + enter(composer); + fireEvent.click(startButton()); + expect(runPosts()).toHaveLength(1); + await act(async () => { + admit({ ...ownedRun, status: 'running' }); + }); + await waitFor(() => expect(composer).toHaveValue('')); + expect(runPosts()).toHaveLength(1); + }); + + describe('when the pane must decide, Enter opens it with the words', () => { + it('opens Ask my agent for a trigger alone', async () => { + renderCrew(); + const composer = await channelReady(); + await typeTask(composer, '/agent'); + expect(consent()).toHaveTextContent(composerCopy.agentEmpty); + enter(composer); + expect(await screen.findByLabelText(agentCopy.task)).toHaveValue(''); + expect(runPosts()).toHaveLength(0); + }); + + it('explains a refused model before Enter, then opens the picker with the task, which starts there and clears the draft', async () => { + config.settings.values = { + CREW_TASK_MODEL: { provider: 'openrouter', model: 'free-model' }, + }; + renderCrew(); + const composer = await channelReady(); + await typeTask(composer, '@agent plot it'); + await waitFor(() => + expect(consent()).toHaveTextContent( + `${agentCopy.publicWorkspace('lab')} ${composerCopy.agentOpensPicker}` + ) + ); + expect(consent()).toHaveAttribute('data-tone', 'warning'); + enter(composer); + expect(await screen.findByLabelText(agentCopy.task)).toHaveValue('plot it'); + // The picker opens on its own, after the pane has taken focus. + const option = await screen.findByRole('option', { name: /^gpt-5\.5/ }); + expect(runPosts()).toHaveLength(0); + + const user = userEvent.setup(); + await user.click(option); + await user.click(screen.getByRole('button', { name: agentCopy.start })); + await waitFor(() => expect(runPosts()).toHaveLength(1)); + expect(runPosts()[0]).toMatchObject({ prompt: 'plot it', provider: 'versa_azure' }); + // The pane's seed was the whole `@agent …` draft, so it clears. + await waitFor(() => expect(composer).toHaveValue('')); + }); + + it('opens it when no model is set up', async () => { + config.settings.providers = []; + config.settings.values = {}; + renderCrew(); + const composer = await channelReady(); + await typeTask(composer, '@agent plot it'); + await waitFor(() => expect(consent()).toHaveTextContent(composerCopy.agentNoModel)); + enter(composer); + expect(await screen.findByText(agentCopy.noModels)).toBeInTheDocument(); + expect(screen.getByLabelText(agentCopy.task)).toHaveValue('plot it'); + expect(runPosts()).toHaveLength(0); + }); + }); + + describe('a draft that cannot start, said before Enter', () => { + it('refuses files: a task is words only', async () => { + renderCrew(); + const composer = await channelReady(); + act(() => currentCrew().addAttachment({ id: ids.blob, name: 'counts.csv' })); + await typeTask(composer, '@agent analyze it'); + expect(consent()).toHaveTextContent(composerCopy.agentFiles); + expect(startButton()).toBeDisabled(); + enter(composer); + await act(async () => { + await new Promise((resolve) => setTimeout(resolve, 20)); + }); + expect(runPosts()).toHaveLength(0); + expect(posts()).toHaveLength(0); + expect(screen.queryByLabelText(agentCopy.task)).toBeNull(); + }); + + it('refuses a task over 32 KB', async () => { + renderCrew(); + const composer = await channelReady(); + await typeTask(composer, `@agent ${'x'.repeat(33_000)}`); + expect(consent()).toHaveTextContent(composerCopy.agentTooLong); + expect(startButton()).toBeDisabled(); + enter(composer); + expect(runPosts()).toHaveLength(0); + }); + }); + + describe('a start that fails is told in the composer', () => { + it('leads with "Couldn’t start your agent." and rewords a model refusal, with Choose another model', async () => { + daemon.state.http = (path, method) => { + if (path === RUNS && method === 'POST') + throw new CrewHttpError( + "Daemon returned 400: Crew institution does not match the model's resolved affiliation; choose a local model or a model approved for this institution", + 400, + 'crew_institution_mismatch' + ); + return undefined; + }; + renderCrew(); + const composer = await channelReady(); + await typeTask(composer, '@agent summarize counts.csv'); + await waitFor(() => expect(consent()).toHaveTextContent('Enter starts your agent')); + enter(composer); + const note = await screen.findByText(composerCopy.startErrorLead); + const alert = note.closest('[role="alert"]') as HTMLElement; + expect(alert).toHaveTextContent(agentCopy.institutionRefused('gpt-5.5', 'UCSF')); + expect(alert).not.toHaveTextContent(composerCopy.sendErrorLead); + expect(composer).toHaveValue('@agent summarize counts.csv'); + expect(currentCrew().error).toMatchObject({ source: 'composer', action: 'task-start' }); + + await userEvent + .setup() + .click(within(alert).getByRole('button', { name: composerCopy.chooseAnotherModel })); + expect(await screen.findByLabelText(agentCopy.task)).toHaveValue('summarize counts.csv'); + }); + + it('says an unknown outcome and leads to Ask my agent’s gate, where Enter now goes too', async () => { + daemon.state.http = (path, method) => { + if (path === RUNS && method === 'POST') + throw new CrewHttpError( + 'The request was received, but Crew cannot tell whether the task started.', + 409, + 'crew_start_outcome_unknown' + ); + return undefined; + }; + renderCrew(); + const composer = await channelReady(); + await typeTask(composer, '@agent summarize counts.csv'); + await waitFor(() => expect(consent()).toHaveTextContent('Enter starts your agent')); + enter(composer); + const title = await screen.findByText(unknownOutcomeCopy.title); + const alert = title.closest('[role="alert"]') as HTMLElement; + await waitFor(() => expect(consent()).toHaveTextContent(composerCopy.agentUnknownOutcome)); + + // The note's own way to the gate, and Enter's, which now opens the pane rather than starting. + await userEvent + .setup() + .click(within(alert).getByRole('button', { name: composerCopy.openAskMyAgent })); + expect(await screen.findByLabelText(agentCopy.task)).toHaveValue('summarize counts.csv'); + expect(screen.getAllByRole('checkbox')).toHaveLength(1); + act(() => currentCrew().closePane()); + await waitFor(() => expect(screen.queryByLabelText(agentCopy.task)).toBeNull()); + enter(composer); + expect(await screen.findByLabelText(agentCopy.task)).toHaveValue('summarize counts.csv'); + expect(runPosts()).toHaveLength(1); + + // The lock is module-scoped (C10): a deliberate restart after inspection is what lifts it. + daemon.state.http = (path, method) => + path === RUNS && method === 'POST' ? { ...ownedRun, status: 'running' } : undefined; + const user = userEvent.setup(); + await user.click(screen.getByRole('checkbox')); + await user.click(screen.getByRole('button', { name: unknownOutcomeCopy.restart })); + await waitFor(() => expect(currentCrew().unknownRunDestination).toBeNull()); + expect(runPosts()).toHaveLength(2); + }); + + it('names the channel in the connection bar when the person moved on before it failed', async () => { + let refuse: (reason: unknown) => void = () => undefined; + daemon.state.http = (path, method) => + path === RUNS && method === 'POST' + ? new Promise((_resolve, reject) => { + refuse = reject; + }) + : undefined; + renderCrew(); + const composer = await channelReady(); + await typeTask(composer, '@agent summarize counts.csv'); + await waitFor(() => expect(consent()).toHaveTextContent('Enter starts your agent')); + enter(composer); + await waitFor(() => expect(runPosts()).toHaveLength(1)); + act(() => currentCrew().selectChannel(ids.methods)); + await channelReady('methods'); + await act(async () => { + refuse(new CrewHttpError('Daemon returned 500: boom', 500)); + }); + await waitFor(() => expect(currentCrew().error?.source).toBe('global')); + expect(currentCrew().error?.message).toBe( + crewActionCopy.startFailedIn('#general', 'Daemon returned 500: boom') + ); + }); + }); + + it('reads every channel the person is in, in an open workspace whose broker reads wide', async () => { + const base = richSnapshot({ pending_joins: [] }); + daemon.state.snapshot = { + ...base, + workspace: { ...base.workspace, governance: 'open' } as Snapshot['workspace'], + }; + renderCrew(); + const composer = await channelReady(); + act(() => + daemon.emit({ + type: 'state', + connection_id: connection.id, + connection_mode: 'private', + connection_policy_epoch: 1, + connection_institution_id: connection.institution_id, + snapshot: daemon.state.snapshot, + runs: [], + cursor: null, + capabilities: ['presence_v1', 'wide_runs_v1'], + }) + ); + await waitFor(() => expect(currentCrew().capabilities).toContain('wide_runs_v1')); + await typeTask(composer, '@agent summarize the week'); + await waitFor(() => + expect(consent()).toHaveTextContent( + 'It reads 2 channels and posts its result in #general, for this task only.' + ) + ); + enter(composer); + await waitFor(() => expect(runPosts()).toHaveLength(1)); + expect(runPosts()[0].context_channels).toEqual([ids.general, ids.methods]); + }); +}); diff --git a/ui/desktop/src/components/crew/composer/Composer.tsx b/ui/desktop/src/components/crew/composer/Composer.tsx index 6e03b1652..8a146e27a 100644 --- a/ui/desktop/src/components/crew/composer/Composer.tsx +++ b/ui/desktop/src/components/crew/composer/Composer.tsx @@ -1,6 +1,7 @@ import { useCallback, useEffect, + useId, useLayoutEffect, useMemo, useRef, @@ -12,16 +13,33 @@ import { type ReactNode, type Ref, } from 'react'; +import type { ProviderDetails } from '../../../api'; import { Button } from '../../ui/button'; import { Note } from '../../ui/note'; import { Tooltip, TooltipContent, TooltipTrigger } from '../../ui/Tooltip'; import { ArrowUp, Bot, Loader2, X } from '../../icons/app-icons'; import { cn } from '../../../utils'; -import { channelSlug } from '../identity'; +import { channelName, channelNamesAcrossTeams, channelSlug } from '../identity'; +import { agentCopy, unknownOutcomeCopy } from '../pane/copy'; +import { ModelTierMarks } from '../pane/CrewModelPicker'; +import { + isAffiliationRefusal, + isPublicModelRefusal, + modelDisplay, + modelRefusalText, + remoteAccessText, + taskStartCheck, + usePanePresentation, + wideRunChannels, + workspaceInstitutionLabel, + knownInstitutions, + type TaskStartCheck, +} from '../pane/presentation'; +import { useConfiguredModels } from '../pane/useConfiguredModels'; import { crewActionCopy } from '../state/copy'; import { POST_NOTE_CODES, postDestination } from '../state/crewSend'; import { draftTooLongToKeep } from '../state/draftStash'; -import type { CrewActionError, DraftFile, DraftReference } from '../state/types'; +import type { AgentPaneSeed, CrewActionError, DraftFile, DraftReference } from '../state/types'; import { useCrew, useCrewErrorSlot, useCrewSurfaceReset } from '../state/CrewControllerContext'; import { postedLabel, useAttachmentIndexVersion } from '../files/attachmentIndex'; import { filesCopy } from '../files/copy'; @@ -34,11 +52,29 @@ import { } from '../files/FileDropZone'; import { canShareDroppedFiles, useCrewUpload, type CrewShareNames } from '../files/useCrewUpload'; import { serverLabel, workspaceTitle } from '../sidebar/sidebarView'; +import { + agentNameTaken, + agentStartState, + parseAgentInvocation, + type AgentStartState, + type SuggestionQuery, +} from './agentInvocation'; import { AttachMenu } from './AttachMenu'; import { ComposerChips } from './ComposerChips'; +import { + buildSuggestions, + ComposerSuggestions, + suggestionId, + useComposerSuggestions, + type ComposerSuggestion, +} from './ComposerSuggestions'; import { composerCopy } from './copy'; import './composer.css'; +/** A draft whose first word is `@` or `/` something: the agent's models may be about to matter. */ +const LEADING_SIGIL = /^\s*[@/]/; +const NO_CHANNELS: string[] = []; + /** The daemon's attachment limit (`local_files::MAX_SIZE`): 1 GiB. */ export const CREW_ATTACHMENT_LIMIT = 1024 * 1024 * 1024; @@ -119,8 +155,19 @@ export interface ComposerProps { * resolves the dropped `File` itself, and only Share yields the file capability. Both then * start the upload with the exact legacy `beginTransfer` payload. The renderer never reads a * file or hands a path to anything. + * - **`@agent`, `/agent`, `/task`** as the draft's first word start the person's own agent on a + * task in this channel (design OW-10, `agentInvocation.ts`), on their model for Crew tasks + * (OW-11). Before Enter a consent line inside the card says what Enter will do — the model and its + * tier, where the agent posts, what it reads and may do on the server — and Send reads "Start + * agent task". Enter is the consent. It starts the task through `askAgent` (Ask my agent's request + * exactly), or opens Ask my agent with the words when the pane must decide: no task yet, a model + * that would be refused here or none, a start whose outcome is unknown. A draft with files, or + * longer than a task may be, starts nothing and the line says why. "Post as a message" posts the + * draft as written. Typing `@` or `/` offers the agent and the channel's people in a suggestion + * list (`ComposerSuggestions`), which takes Enter only to insert a row. * - * React authorizes nothing here: the daemon and broker decide every post and every upload. + * React authorizes nothing here: the daemon and broker decide every post, every upload and every + * task. */ export function Composer({ note, inputRef, agentPaneId }: ComposerProps) { const controller = useCrew(); @@ -160,6 +207,9 @@ export function Composer({ note, inputRef, agentPaneId }: ComposerProps) { // note went with it. So it goes on the person's own edits (typing, removing a chip), a send, its // dismiss control, a new drop or paste, and the surface resets listed below. const [extraFiles, setExtraFiles] = useState(''); + // "Post as a message" (OW-10): this draft posts as written, `@agent` and all, until it no longer + // reads as a task. + const [literal, setLiteral] = useState(false); const latestUpload = useRef(upload); latestUpload.current = upload; // What the native share confirmation calls this channel and its workspace: display text only. @@ -234,8 +284,11 @@ export function Composer({ note, inputRef, agentPaneId }: ComposerProps) { reason === 'protected-cleared' || reason === 'channel-changed' || reason === 'channel-revoked' - ) + ) { setExtraFiles(''); + // "Post as a message" was said of one channel's draft. + setLiteral(false); + } }); // The observer clears the draft when the verified privacy scope changes under it (a new @@ -351,6 +404,121 @@ export function Composer({ note, inputRef, agentPaneId }: ComposerProps) { } const agentOpen = ui.pane?.mode === 'agent'; + // `@agent`, `/agent`, `/task` (OW-10): the person's own draft, and nothing else, can start their + // agent. A message from anyone else is never parsed. + const { workspace, dir } = usePanePresentation(); + const nameTaken = useMemo(() => agentNameTaken(snapshot), [snapshot]); + const parsed = useMemo( + () => parseAgentInvocation(draft.body, { agentNameTaken: nameTaken }), + [draft.body, nameTaken] + ); + useEffect(() => { + if (!parsed) setLiteral(false); + }, [parsed]); + const invocation = literal ? null : parsed; + // The models are read the first time a draft could need them, not on every mount. + const models = useConfiguredModels({ + optional: true, + enabled: parsed !== null || LEADING_SIGIL.test(draft.body), + }); + const taskModel = models.taskDefault; + const taskProvider = models.providers?.find((item) => item.name === taskModel?.provider); + const shownModel = taskModel ? modelDisplay(taskModel, taskProvider) : null; + const shownModelName = shownModel + ? agentCopy.modelChoice(shownModel.model, shownModel.provider) + : ''; + // An open workspace whose broker reads wide: `@agent` reads every channel the person is in. + const wide = useMemo( + () => wideRunChannels({ snapshot, channel, capabilities: controller.capabilities }), + [snapshot, channel, controller.capabilities] + ); + const contextChannels = wide ?? NO_CHANNELS; + const channelLabels = useMemo( + () => channelNamesAcrossTeams(snapshot?.channels ?? [], snapshot?.teams ?? []), + [snapshot] + ); + // The pane's own checks, on the model the consent line names (OW-10): the two never disagree. + const check = taskStartCheck({ + provider: taskProvider, + providers: models.providers, + connection: controller.connection, + snapshot, + channel, + contextChannels, + workspace, + channelLabel: (item) => channelLabels.get(item.id) ?? channelName(item), + model: shownModel?.model ?? null, + }); + const agentState: AgentStartState | null = invocation + ? agentStartState({ + invocation, + hasFiles: draft.attachments.length > 0 || draft.references.length > 0, + unknownOutcome: controller.unknownRunDestination !== null, + modelsKnown: models.providers !== null || models.failure !== null, + hasModel: taskModel !== null, + blockText: check.blockText, + }) + : null; + const startingTask = isPending('run.start'); + const starting = useRef(false); + const openAgentPane = (seed: AgentPaneSeed) => openPane({ mode: 'agent', seed }); + /** + * Enter, or the send button: a message as before, else what the consent line said it would do. + * A start is single flight: the pending `run.start` (shared with the pane) and, until that + * renders, a ref. + */ + const submitNow = () => { + if (!invocation || !agentState) return sendNow(); + latestUpload.current.dismissError(); + setExtraFiles(''); + const seed = { task: invocation.prompt, draft: draft.body }; + switch (agentState.kind) { + case 'files': + case 'too-long': + // Nothing starts; the line under the text says why. + return; + case 'empty': + case 'unknown-outcome': + case 'loading': + return openAgentPane(seed); + case 'no-model': + case 'blocked': + return openAgentPane({ ...seed, pickModel: true }); + case 'ready': + break; + } + if (!taskModel || startingTask || starting.current) return; + const ask = controller.askAgent; + if (!ask) return openAgentPane(seed); + starting.current = true; + // Exactly the model the line showed, never one read again at Enter. + void ask({ + prompt: invocation.prompt, + draftBody: draft.body, + model: taskModel, + contextChannels, + }).finally(() => { + starting.current = false; + }); + }; + const known = useMemo(() => knownInstitutions(models.providers), [models.providers]); + const taskError = (failure: CrewActionError) => + taskStartNote({ + error: failure, + check, + model: shownModel?.model ?? agentCopy.model, + institution: + check.institution?.label ?? + workspaceInstitutionLabel(controller.connection, snapshot, known), + onChooseModel: () => + openAgentPane({ + task: parsed?.prompt ?? '', + draft: draft.body, + pickModel: true, + }), + onOpenPane: () => openAgentPane({ task: parsed?.prompt ?? '', draft: draft.body }), + }); + // Another channel's failure never shows here, nor comes along into it (QA M5). const composerError = ownsError && @@ -363,6 +531,7 @@ export function Composer({ note, inputRef, agentPaneId }: ComposerProps) { }); const notes = composerNote({ error: composerError, + taskError, tooLongToKeep: draftTooLongToKeep(draft.body), uploadError: upload.error, onDismissUpload: dismissUploadError, @@ -410,6 +579,66 @@ export function Composer({ note, inputRef, agentPaneId }: ComposerProps) { onRemoveAttachment={removeOwnAttachment} onRemoveReference={removeOwnReference} onSend={sendNow} + onSubmit={submitNow} + agentMode={invocation !== null} + agentRefused={agentState?.kind === 'files' || agentState?.kind === 'too-long'} + startingTask={startingTask} + renderAgentLine={(hintId) => + agentState ? ( + + openAgentPane({ + task: invocation?.prompt ?? '', + draft: draft.body, + pickModel: true, + }) + } + onPostAsMessage={() => setLiteral(true)} + /> + ) : literal && parsed ? ( +
+

+ {composerCopy.postingAsMessage} +

+ + + +
+ ) : null + } + buildRows={(query) => + buildSuggestions({ + query, + snapshot, + channel, + slug: name, + dir, + agentNameTaken: nameTaken, + agentDetail: shownModelName ? ( + <> + {composerCopy.agentRowDetail(shownModelName, name)}{' '} + + + ) : ( + composerCopy.agentRowDetail(null, name) + ), + }) + } posting={isPending('send')} agentOpen={agentOpen} agentPaneId={agentPaneId} @@ -498,6 +727,175 @@ function sendNote(error: CrewActionError): ReactNode { } } +/** + * A start from the composer that failed (`@agent`, OW-10), above the card: "Couldn't start your + * agent." and the reason in its own node, a model refusal in Ask my agent's words with **Choose + * another model**, and a start whose outcome is unknown as the pane's gate title with **Open Ask my + * agent**, where the gate is (C9 to C12). The draft keeps the task's words either way. + */ +function taskStartNote({ + error, + check, + model, + institution, + onChooseModel, + onOpenPane, +}: { + error: CrewActionError; + check: TaskStartCheck; + model: string; + institution: string | null; + onChooseModel(): void; + onOpenPane(): void; +}): ReactNode { + if (error.code === 'crew_start_outcome_unknown') + return ( + + {composerCopy.openAskMyAgent} + + } + > + {unknownOutcomeCopy.title} + + ); + const modelRefused = isAffiliationRefusal(error) || isPublicModelRefusal(error); + const reason = + modelRefusalText({ + error, + mismatch: check.mismatchText, + publicText: check.publicText, + model, + institution, + }) ?? error.message; + return ( + + {composerCopy.chooseAnotherModel} + + ) : undefined + } + > + {composerCopy.startErrorLead} {reason} + + ); +} + +/** + * The consent line under the text while the draft starts a task (OW-10): what Enter will do, said + * before Enter, because Enter is the consent. When Enter starts the task it names the model and + * its tier mark (the same chip Ask my agent shows), where the agent posts, what it reads and what + * it may do on the server; otherwise it says what Enter does instead, or why nothing starts. The + * words are a live region, so they are heard as they change; the links beside them are not. + * + * **Change** opens Ask my agent on its model picker with the task's words; **Post as a message** + * sends this draft as written. + */ +function AgentConsentLine({ + id, + state, + slug, + model, + provider, + privateOnly, + reads, + access, + onChange, + onPostAsMessage, +}: { + id: string; + state: AgentStartState; + slug: string; + model: string; + provider: ProviderDetails | undefined; + privateOnly: boolean; + /** How many channels the task reads, the one it posts in included. */ + reads: number; + access: string | null; + onChange(): void; + onPostAsMessage(): void; +}) { + let tone: 'neutral' | 'warning' = 'neutral'; + let text: ReactNode; + switch (state.kind) { + case 'ready': + text = ( + <> + + {composerCopy.agentStartsWith(model)}{' '} + . + {' '} + + {reads > 1 ? composerCopy.agentReadsWide(reads, slug) : composerCopy.agentReads(slug)} + + {access && {access}} + + ); + break; + case 'blocked': + tone = 'warning'; + text = `${state.text} ${composerCopy.agentOpensPicker}`; + break; + case 'no-model': + text = composerCopy.agentNoModel; + break; + case 'loading': + text = composerCopy.agentLoading; + break; + case 'empty': + text = composerCopy.agentEmpty; + break; + case 'unknown-outcome': + tone = 'warning'; + text = composerCopy.agentUnknownOutcome; + break; + case 'files': + tone = 'warning'; + text = composerCopy.agentFiles; + break; + case 'too-long': + tone = 'warning'; + text = composerCopy.agentTooLong; + break; + } + return ( +
+ +

+ {text} +

+ + {state.kind === 'ready' && ( + + )} + + +
+ ); +} + /** * The one note directly above the card, in priority order: the send's answer, an upload failure, * the note about several dropped files, a draft too long to keep across a channel switch @@ -519,6 +917,7 @@ function sendNote(error: CrewActionError): ReactNode { */ function composerNote({ error, + taskError, tooLongToKeep, uploadError, onDismissUpload, @@ -528,6 +927,8 @@ function composerNote({ note, }: { error: CrewActionError | null; + /** How a failed `@agent` start reads (`taskStartNote`); a send's answer is `sendNote`'s. */ + taskError(error: CrewActionError): ReactNode; tooLongToKeep: boolean; uploadError: string; onDismissUpload(): void; @@ -538,7 +939,7 @@ function composerNote({ }): ReactNode { let content: ReactNode = null; if (error) { - content = sendNote(error); + content = error.action === 'task-start' ? taskError(error) : sendNote(error); } else if (uploadError) { content = ( }> @@ -577,6 +978,18 @@ interface ComposerCardProps { onRemoveAttachment(id: string): void; onRemoveReference(id: string): void; onSend(): void; + /** Enter and Send: `onSend` for a message, the agent's start for `@agent` (OW-10). */ + onSubmit(): void; + /** The draft starts a task: Send reads "Start agent task" and the consent line shows. */ + agentMode: boolean; + /** Nothing starts for this draft (files, too long); the consent line says why. */ + agentRefused: boolean; + /** A task is being started, from here or from Ask my agent. */ + startingTask: boolean; + /** The consent line (or the "posts as a message" line), whose words carry `hintId`. */ + renderAgentLine(hintId: string): ReactNode; + /** The suggestion rows for the word before the caret. */ + buildRows(query: SuggestionQuery): ComposerSuggestion[]; posting: boolean; agentOpen: boolean; agentPaneId?: string; @@ -661,6 +1074,12 @@ function ComposerCard({ onRemoveAttachment, onRemoveReference, onSend, + onSubmit, + agentMode, + agentRefused, + startingTask, + renderAgentLine, + buildRows, posting, agentOpen, agentPaneId, @@ -678,6 +1097,21 @@ function ComposerCard({ [inputRef] ); const hasContent = Boolean(body.trim()) || attachments.length > 0 || references.length > 0; + const hintId = useId(); + const listId = useId(); + const agentLine = renderAgentLine(hintId); + + // The suggestion list follows the caret (`suggestionQuery`); `null` is "at the end". + const [caretAt, setCaretAt] = useState(null); + const [focused, setFocused] = useState(false); + const suggestions = useComposerSuggestions({ + body, + caret: caretAt ?? body.length, + focused, + build: buildRows, + }); + // Where the caret goes once an accepted row's text is in the box. + const placeCaret = useRef(null); // Auto-grow: one row at rest, up to the stylesheet's 40vh cap, then it scrolls. Not animated // (the spec lists composer auto-grow among what must not move). Where there is no layout @@ -691,8 +1125,35 @@ function ComposerCard({ }, [body]); const caret = useRestoredDraftCaret(textarea, body); + // After an accepted row: the caret goes after what it inserted, not to the end. + useLayoutEffect(() => { + const at = placeCaret.current; + const node = textarea.current; + if (at === null || !node) return; + placeCaret.current = null; + node.setSelectionRange(at, at); + setCaretAt(at); + }, [body]); + /** Insert a suggestion row's text. It starts nothing: Enter does, once the list is closed. */ + const acceptRow = (index: number) => { + const next = suggestions.accept(index); + if (!next) return; + // The person's own edit, so the restored-draft rule leaves the caret to `placeCaret`. + caret.typed(next.body); + placeCaret.current = next.caret; + setBody(next.body); + textarea.current?.focus(); + }; const onKeyDown = (e: KeyboardEvent) => { + // An open suggestion list takes its keys first (Enter and Tab accept a row); it ignores every + // key while an input method is composing, as the rest of this handler does. + const listed = suggestions.onKeyDown(e); + if (listed === 'accept') { + acceptRow(suggestions.active); + return; + } + if (listed) return; if ( e.key !== 'Enter' || e.shiftKey || @@ -701,7 +1162,7 @@ function ComposerCard({ ) return; e.preventDefault(); - if (!e.repeat) onSend(); + if (!e.repeat) onSubmit(); }; /** * A pasted file goes to the upload path; pasted words stay words. Copying cells or rich text @@ -740,6 +1201,15 @@ function ComposerCard({ onPauseUpload={(transfer) => void upload.pause(transfer)} onResumeUpload={(transfer) => void upload.resume(transfer)} /> + {suggestions.open && ( + + )}