diff --git a/README.md b/README.md index d04cb5d..72352b7 100644 --- a/README.md +++ b/README.md @@ -132,7 +132,7 @@ and `--ssh my-mac` checks a Mac over SSH). - ✅ **Computer Use app**: ~/.codex/computer-use/Codex Computer Use.app (26.916.1001103) - ✅ **Computer Use runtime**: Codex app-server running with `cua_repl` - ✅ **App access**: Codex approves app access itself (approval_policy = "never") -- ✅ **Native apps**: Engine reachable, 17 apps listed +- ✅ **Native apps**: Engine reachable (Finder attached) - ✅ **Screenshots**: Finder screenshot captured (image/jpeg, 132 KB) - ✅ **OCR**: 61 text lines recognized in 894 ms - ✅ **Browser tabs**: Connected: Chrome (extension) @@ -150,6 +150,7 @@ Pass options with the object form of the plugin entry. Every option is optional; "options": { "surfaces": ["apps", "browser"], "screenshots": "image", + "appAccess": "decline", "maxOutputKB": 128, "idleShutdownMinutes": 0, "callTimeoutSeconds": 300, @@ -164,6 +165,7 @@ Pass options with the object form of the plugin entry. Every option is optional; | --------------------- | --------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | `surfaces` | `["apps", "browser"]` | What the agent may operate: native apps, browser tabs, or both. Turning one off removes it from the tool description and makes its `cua` functions throw. This scopes the model; it is not a security boundary. | | `screenshots` | `"image"` | How screenshots reach the model. `"image"`: as images. `"ocr"`: as text recognized on-device (macOS Vision), one line per text block with its `[x,y]` position, which is also a valid click coordinate; for models without vision, or to keep pixels off your model provider. `"both"`: image plus text. `"off"`: replaced by a placeholder, so the model relies on the accessibility tree. | +| `appAccess` | `"decline"` | How to answer Codex's per-app access prompts. `"decline"` leaves them to Codex: apps you have not approved are declined, and the agent tells you which app needs permission. `"accept"` approves every prompt and asks Codex to remember the app; this makes apps usable on Codex builds that cannot surface approvals, but removes that confirmation step. | | `maxOutputKB` | `128` | Maximum text one call returns. Longer output is cut, with a note telling the model to narrow its query. `0` disables the limit. Images are not counted. | | `idleShutdownMinutes` | `0` (never) | Stop the background Codex process after this many minutes without Computer Use calls. The default keeps it running until OpenCode stops, so work can wait indefinitely for you (for example, a login in a tab the agent handed over). | | `callTimeoutSeconds` | `300` | Maximum time for one `computer_use` call. | @@ -181,6 +183,10 @@ question, as they do inside Codex: - Any other app is declined, and the agent tells you which app needs permission. OpenCode's plugin API cannot show these prompts as OpenCode permission questions yet; once it can ([anomalyco/opencode#46530](https://github.com/anomalyco/opencode/pull/46530)), they will appear in OpenCode instead. +- `appAccess: "accept"` answers every forwarded prompt with approval and asks Codex to remember the app. Use it when + your Codex build cannot surface app-access approvals and every native app fails with "was not approved"; it removes + the per-app confirmation step, so only enable it if you are comfortable with the agent using any app it is asked to + operate. The default is `"decline"`. Apps blocked by the engine or your organization stay blocked. `/computer-use-doctor` shows which policy applies. @@ -299,7 +305,7 @@ Start with `/computer-use-doctor`; it names the missing piece. | `Could not reach my-mac over SSH` | On the OpenCode machine, `ssh my-mac true` must succeed without a prompt; see [Run Computer Use on another Mac (SSH)](#run-computer-use-on-another-mac-ssh). | | `Codex has no cua_repl MCP server` | Turn on Computer Use in the ChatGPT/Codex app, then run `/computer-use-doctor` again. | | Permission errors from the engine | Grant Accessibility and Screen Recording to _Codex Computer Use_, then retry. | -| The agent says an app needs permission | Approve the app permanently in ChatGPT/Codex, or set `approval_policy = "never"`; see [App access](#app-access). | +| The agent says an app needs permission | Approve the app permanently in ChatGPT/Codex, set `approval_policy = "never"`, or set `appAccess: "accept"`; see [App access](#app-access). | | No browsers listed / Chrome tab calls fail | Install and connect the ChatGPT for Chrome extension from the ChatGPT/Codex app, keep Chrome running, then run `/computer-use-doctor`. | | Anything else | Plugin messages are prefixed `[codex-computer-use]` in `~/.local/share/opencode/log/opencode.log`; set `"debug": true` to log the full exchange with Codex. | diff --git a/src/bridge.ts b/src/bridge.ts index de26961..1a25e94 100644 --- a/src/bridge.ts +++ b/src/bridge.ts @@ -1,6 +1,7 @@ import { AppServerClient, AppServerError } from "./app-server" import { BUNDLED_CODEX_PATHS, CODEX_PATH_ENV, resolveCodexPath } from "./codex-path" import type { Host } from "./host" +import type { AppAccess } from "./options" /** Codex's MCP server that hosts the Computer Use JavaScript runtime (`cua`). */ export const CUA_SERVER = "cua_repl" @@ -12,6 +13,8 @@ export interface BridgeOptions { codexPath?: string idleShutdownMs: number callTimeoutMs: number + /** How to answer Codex's app-access prompts (see the `appAccess` option). */ + appAccess: AppAccess /** The OpenCode location directory; Codex's working folder when Codex runs on this machine. */ cwd: string version: string @@ -108,7 +111,11 @@ export class ComputerUseBridge { threadId: threadID, server: CUA_SERVER, tool: "js", - arguments: { code: this.options.codePrefix ? `${this.options.codePrefix}\n${code}` : code }, + arguments: { + code: this.options.codePrefix ? `${this.options.codePrefix}\n${code}` : code, + // The runtime defaults to 30s, too short for large accessibility trees; keep both timeouts in step. + timeout_ms: this.options.callTimeoutMs, + }, _meta: turnMetadata(threadID, state?.turnID, input.callID), }, { signal: input.signal, timeoutMs: this.options.callTimeoutMs }, @@ -347,9 +354,14 @@ export class ComputerUseBridge { if (method !== "mcpServer/elicitation/request") { throw new Error(`${method} is not supported by opencode-codex-computer-use`) } + const message = typeof params?.message === "string" ? params.message : "Computer Use requested approval" + if (this.options.appAccess === "accept") { + // Opt-in: treat every forwarded app-access prompt as approved and ask Codex to remember the app. + this.options.log(`accepted: ${message}`) + return { action: "accept", content: null, _meta: { persist: "always" } } + } // OpenCode's plugin API cannot raise a permission question yet, so a prompt Codex forwards // (because approval_policy is not "never") is declined, and the model is told how the user can allow the app. - const message = typeof params?.message === "string" ? params.message : "Computer Use requested approval" this.addApprovalNote(params?.threadId, declinedNote(message, appName(params))) this.options.log(`declined: ${message}`) return { action: "decline", content: null } diff --git a/src/codex-path.ts b/src/codex-path.ts index d6f695f..9235ed7 100644 --- a/src/codex-path.ts +++ b/src/codex-path.ts @@ -3,6 +3,7 @@ import type { Host } from "./host" /** macOS app bundles first: their `codex` matches the Computer Use runtime shipped in the same bundle. */ export const BUNDLED_CODEX_PATHS: readonly string[] = [ "/Applications/ChatGPT.app/Contents/Resources/codex", + "/Applications/ChatGPT.app/Contents/Resources/codex-cli/bin/codex", "/Applications/Codex.app/Contents/Resources/codex", ] diff --git a/src/doctor.ts b/src/doctor.ts index 9d0c9f6..3798d2b 100644 --- a/src/doctor.ts +++ b/src/doctor.ts @@ -4,7 +4,7 @@ import { CUA_SERVER, type ComputerUseBridge, type McpContentBlock } from "./brid import { imageMime, textOf } from "./content" import { runOnHost, type Host, type HostInfo } from "./host" import { recognizeText } from "./ocr" -import type { Options } from "./options" +import type { AppAccess, Options } from "./options" type Status = "ok" | "warn" | "fail" | "skip" @@ -74,20 +74,38 @@ export async function runDoctor( return report(checks, options) } add("Computer Use runtime", "ok", `Codex app-server running with \`${CUA_SERVER}\``) - add("App access", ...approvalCheck(await bridge.approvalPolicy().catch(() => undefined))) + add("App access", ...approvalCheck(await bridge.approvalPolicy().catch(() => undefined), options.appAccess)) const probe = (code: string) => bridge.run(sessionKey, code) if (options.surfaces.includes("apps")) { - const apps = await probe('nodeRepl.write("APPS=" + (await cua.listApps({ emit: false })).length);') - const count = textOf(apps.content).match(/APPS=(\d+)/)?.[1] - add( - "Native apps", - count ? "ok" : "fail", - count ? `Engine reachable, ${count} apps listed` : firstLine(apps.content, "Could not list apps"), - ) + // `cua.listApps` can hang in some Codex builds on macOS, so check that one app can be attached to instead. + // Other platforms need an open window to bind an app, so keep the inventory check there. + let ready = false + if (info.platform === "darwin") { + const apps = await probe('await cua.getApp("com.apple.finder"); nodeRepl.write("APPS=1");') + const output = textOf(apps.content) + ready = /APPS=1/.test(output) + if (ready) { + add("Native apps", "ok", "Engine reachable (Finder attached)") + } else if (/not approved/i.test(output)) { + // The engine works; the App access check already explains how to allow apps. + add("Native apps", "warn", "Engine reachable, but Finder is not approved (see App access)") + } else { + add("Native apps", "fail", firstLine(apps.content, "Could not attach to Finder")) + } + } else { + const apps = await probe('nodeRepl.write("APPS=" + (await cua.listApps({ emit: false })).length);') + const count = textOf(apps.content).match(/APPS=(\d+)/)?.[1] + ready = Boolean(count) + add( + "Native apps", + ready ? "ok" : "fail", + ready ? `Engine reachable, ${count} apps listed` : firstLine(apps.content, "Could not list apps"), + ) + } - if (info.platform === "darwin" && count) { + if (info.platform === "darwin" && ready) { const shot = await probe( 'const __doctorFinder = await cua.getApp("com.apple.finder"); await __doctorFinder.getScreenshot({ emit: true }); nodeRepl.write("SHOT");', ).catch((error: unknown) => ({ @@ -144,19 +162,24 @@ function report(checks: Check[], options: Options): DoctorReport { ...checks.map((check) => `- ${ICONS[check.status]} **${check.name}**: ${check.detail}`), "", `Options: surfaces=${options.surfaces.join("+")}, screenshots=${options.screenshots}, ` + + `appAccess=${options.appAccess}, ` + `maxOutputKB=${Math.round(options.maxOutputBytes / 1024) || "unlimited"}, ` + `idleShutdownMinutes=${options.idleShutdownMs / 60_000 || "never"}, debug=${options.debugLog ?? "off"}`, ] return { ok, checks, text: lines.join("\n") } } -export function approvalCheck(policy: string | undefined): [Status, string] { +export function approvalCheck(policy: string | undefined, appAccess: AppAccess = "decline"): [Status, string] { + if (appAccess === "accept") { + return ["ok", 'Computer Use app-access prompts are accepted automatically (appAccess = "accept")'] + } if (policy === "never") return ["ok", 'Codex approves app access itself (approval_policy = "never")'] const setting = policy ? `approval_policy = "${policy}"` : "approval_policy unknown" return [ "warn", `${setting}: apps you have not approved permanently in ChatGPT/Codex will be declined, because OpenCode cannot ` + - 'show Computer Use\'s approval prompts yet. Approve apps once in ChatGPT/Codex, or set approval_policy = "never".', + 'show Computer Use\'s approval prompts yet. Approve apps once in ChatGPT/Codex, set approval_policy = "never", ' + + 'or set the appAccess option to "accept".', ] } diff --git a/src/options.ts b/src/options.ts index 5f6fa3c..3d2a5f6 100644 --- a/src/options.ts +++ b/src/options.ts @@ -9,6 +9,9 @@ export type ScreenshotMode = (typeof SCREENSHOT_MODES)[number] export const SURFACES = ["apps", "browser"] as const export type Surface = (typeof SURFACES)[number] +export const APP_ACCESS = ["decline", "accept"] as const +export type AppAccess = (typeof APP_ACCESS)[number] + export interface Options { codexPath?: string /** SSH destination of the machine that runs Codex Computer Use; undefined runs it on this machine. */ @@ -19,6 +22,8 @@ export interface Options { maxOutputBytes: number screenshots: ScreenshotMode surfaces: readonly Surface[] + /** How to answer Codex's per-app access prompts. */ + appAccess: AppAccess /** JSONL file receiving the app-server traffic, or undefined when debugging is off. */ debugLog?: string /** Messages about options that were accepted but have no effect. */ @@ -47,6 +52,7 @@ export function parseOptions(raw: PluginOptions): Options { maxOutputBytes: Math.round(nonNegative(raw.maxOutputKB, 128) * 1024), screenshots: oneOf("screenshots", raw.screenshots, SCREENSHOT_MODES, "image"), surfaces: surfaces(raw.surfaces), + appAccess: oneOf("appAccess", raw.appAccess, APP_ACCESS, "decline"), debugLog: debugLog(raw.debug), warnings, } diff --git a/test/bridge.test.ts b/test/bridge.test.ts index 1be7fdf..ded1ed2 100644 --- a/test/bridge.test.ts +++ b/test/bridge.test.ts @@ -6,6 +6,7 @@ import { delimiter, join } from "node:path" import { AppServerClient } from "../src/app-server" import { ComputerUseBridge, SetupError } from "../src/bridge" import { localHost, sshHost, type Host } from "../src/host" +import type { AppAccess } from "../src/options" const FAKE_CODEX = join(import.meta.dir, "fixtures", "fake-codex") const FAKE_SSH_DIR = join(import.meta.dir, "fixtures", "fake-ssh") @@ -49,6 +50,7 @@ function setup( host?: Host idleShutdownMs?: number codePrefix?: string + appAccess?: AppAccess onTraffic?: (direction: "send" | "receive", message: unknown) => void } = {}, ) { @@ -62,6 +64,7 @@ function setup( callTimeoutMs: 10_000, cwd: "/tmp/project", version: "test", + appAccess: input.appAccess ?? "decline", log: (message) => logs.push(message), codePrefix: input.codePrefix, onTraffic: input.onTraffic, @@ -95,6 +98,7 @@ describe("ComputerUseBridge", () => { expect(starts).toHaveLength(2) expect(starts[0].params).toEqual({ ephemeral: true, cwd: "/tmp/project" }) expect(calls("mcpServer/tool/call")[0].params).toMatchObject({ server: "cua_repl", tool: "js" }) + expect(calls("mcpServer/tool/call")[0].params.arguments).toEqual({ code: "1 + 1", timeout_ms: 10_000 }) }) test("prepends the code prefix and reports traffic", async () => { @@ -150,6 +154,14 @@ describe("ComputerUseBridge", () => { expect(result.notes[0]).toContain('approval_policy = "never"') }) + test("accepts forwarded prompts when appAccess is accept", async () => { + const { bridge, logs } = setup({ appAccess: "accept" }) + const result = await bridge.run("ses_a", "ELICIT") + expect(text(result)).toBe("elicitation:accept") + expect(result.notes).toEqual([]) + expect(logs.some((line) => line.startsWith('accepted: Allow Computer Use to use "Calculator"'))).toBe(true) + }) + test("reads the Codex approval policy", async () => { process.env.FAKE_CODEX_POLICY = "never" expect(await setup().bridge.approvalPolicy()).toBe("never") diff --git a/test/options.test.ts b/test/options.test.ts index 0374665..ed6cac5 100644 --- a/test/options.test.ts +++ b/test/options.test.ts @@ -20,6 +20,7 @@ describe("parseOptions", () => { maxOutputBytes: 128 * 1024, screenshots: "image", surfaces: ["apps", "browser"], + appAccess: "decline", debugLog: undefined, warnings: [], }) @@ -35,6 +36,7 @@ describe("parseOptions", () => { maxOutputKB: 0, screenshots: "ocr", surfaces: ["browser"], + appAccess: "accept", debug: "~/cu.jsonl", }), ).toEqual({ @@ -45,6 +47,7 @@ describe("parseOptions", () => { maxOutputBytes: 0, screenshots: "ocr", surfaces: ["browser"], + appAccess: "accept", debugLog: join(homedir(), "cu.jsonl"), warnings: [], }) @@ -56,6 +59,7 @@ describe("parseOptions", () => { expect(() => parseOptions({ screenshots: "video" })).toThrow("invalid option screenshots") expect(() => parseOptions({ surfaces: [] })).toThrow("invalid option surfaces") expect(() => parseOptions({ surfaces: ["apps", "files"] })).toThrow("invalid option surfaces") + expect(() => parseOptions({ appAccess: "ask" })).toThrow('invalid option appAccess="ask"; use decline, accept') }) test("ssh takes one SSH destination", () => { @@ -75,11 +79,19 @@ test("the removed approvals option is ignored with a warning", () => { }) describe("approvalCheck", () => { - test("only never is fully automatic", () => { + test("only never is fully automatic by default", () => { expect(approvalCheck("never")[0]).toBe("ok") expect(approvalCheck("on-request")).toEqual(["warn", expect.stringContaining('approval_policy = "on-request"')]) expect(approvalCheck(undefined)[1]).toContain("approval_policy unknown") }) + + test("reports the appAccess option", () => { + expect(approvalCheck("on-request", "accept")).toEqual([ + "ok", + 'Computer Use app-access prompts are accepted automatically (appAccess = "accept")', + ]) + expect(approvalCheck(undefined)[1]).toContain('appAccess option to "accept"') + }) }) function doctor(host: Host) { @@ -89,6 +101,7 @@ function doctor(host: Host) { callTimeoutMs: 1_000, cwd: "/", version: "test", + appAccess: "decline", log: () => {}, }) return runDoctor(bridge, parseOptions({ ssh: "my-mac" }), "doctor")