diff --git a/.github/workflows/codemode-evals.yml b/.github/workflows/codemode-evals.yml index 18c7eda..1fb2b54 100644 --- a/.github/workflows/codemode-evals.yml +++ b/.github/workflows/codemode-evals.yml @@ -26,7 +26,9 @@ jobs: # unchanged main added no signal for these offline checks. deterministic: name: Deterministic evals - runs-on: macos-latest + # Pinned, not macos-latest: the package declares a macOS 15 floor, and a + # silently-rolling runner would move that floor without a code change. + runs-on: macos-15 steps: - name: Checkout uses: actions/checkout@v4 @@ -49,6 +51,18 @@ jobs: - name: Test package run: swift test + - name: Set up Node + uses: actions/setup-node@v4 + with: + node-version: "22" + + # The generated TypeScript declarations go into model prompts and host + # type-checkers, so they are compiled for real: every platform surface and + # every per-capability fragment, under `tsc --strict`, TypeScript pinned. + # The Swift tests only catch failure shapes someone already thought of. + - name: Type-check generated TypeScript declarations + run: scripts/check-typescript-declarations.sh "$RUNNER_TEMP/codemode-dts" + - name: Build eval CLI run: swift build --package-path Tools/CodeModeEval @@ -57,7 +71,7 @@ jobs: platform-build: name: Build (${{ matrix.platform }}) - runs-on: macos-latest + runs-on: macos-15 strategy: fail-fast: false matrix: @@ -107,7 +121,7 @@ jobs: # live+compare steps with a warning, instead of pretending or failing noisily. llm-live: name: LLM eval (live regression gate) - runs-on: macos-latest + runs-on: macos-15 needs: deterministic if: github.event_name == 'workflow_dispatch' env: diff --git a/Package.swift b/Package.swift index b1b3f42..3ceeeb2 100644 --- a/Package.swift +++ b/Package.swift @@ -67,6 +67,11 @@ let package = Package( .testTarget( name: "CodeModeAuthoringTests", dependencies: [ + // Declared directly, not only via CodeModeAuthoring: these tests use + // CodeModeConfiguration/CodeModeAgentTools, and without the direct + // edge SwiftPM did not rebuild them when CodeMode's types changed + // layout — leaving stale objects that failed to link or crashed. + "CodeMode", "CodeModeAuthoring", "CodeModeMacros", .product(name: "SwiftSyntaxMacrosTestSupport", package: "swift-syntax"), diff --git a/README.md b/README.md index 29bf64f..4c19cfe 100644 --- a/README.md +++ b/README.md @@ -48,16 +48,24 @@ Then add the product to your target: ## Public API - `CodeModeAgentTools` -- `searchJavaScriptAPI(_:) async throws -> JavaScriptAPISearchResponse` -- `executeJavaScript(_:) async throws -> JavaScriptExecutionCall` -- `JavaScriptExecutionCall.events` -- `JavaScriptExecutionCall.result` -- `JavaScriptExecutionCall.cancel()` + - `searchJavaScriptAPI(_:) async throws -> JavaScriptAPISearchResponse` + - `executeJavaScript(_:) async throws -> JavaScriptExecutionCall` + - `typeDeclarations()` / `typeDeclarations(for:capabilityKeys:)` — TypeScript for the granted surface or a subset + - `capabilities()` — every reference available on this host, each with its `dts` + - `register(provider:)` / `register(providers:)` — add host providers after construction +- `JavaScriptExecutionCall.events`, `.result`, `.cancel()` - `CodeModeToolError` -- `CodeModeConfiguration` -- `CodeModeFileSystem` -- `LocalCodeModeFileSystem` -- `CodeModeAgentToolDescriptions` +- `CodeModeConfiguration`, and the policies it carries: + - `CapabilityGrant` — the host-owned capability ceiling + - `NetworkAccessPolicy` — fetch destinations, response size, credential headers + - `ExecutionLimits` — result/log/event bounds and concurrency + - `FileSystemLimits` — `fs.read`/`fs.write`/`fs.copy` byte ceilings + - `PathPolicy` / `DefaultPathPolicy` — sandbox roots +- `CodeModeFileSystem`, `LocalCodeModeFileSystem` +- `CodeModeAgentToolDescriptions` — tool names, descriptions, and parameter schemas +- `CodeModeAgentGuidance` — budgeted system-prompt guidance (`.brief`/`.standard`/`.full`) +- `TypeScriptDeclarations` — the declaration generator behind `typeDeclarations()` +- `HostConfigurationValidator` — Info.plist usage strings and configuration warnings - `CodeModeEvaluation` SwiftPM product for deterministic scenario evaluation - `SystemUIPresenter` - `UIKitSystemUIPresenter` on iOS/visionOS @@ -67,7 +75,13 @@ Then add the product to your target: ```swift import CodeMode -let tools = CodeModeAgentTools() +// The grant is the host's ceiling. Without it the model's own allowlist is the +// only one in force — fine for a trusted caller, not for tool JSON from a model. +let tools = CodeModeAgentTools( + config: CodeModeConfiguration( + capabilityGrant: .only([.fsRead, .fsWrite]) + ) +) let searchResponse = try await tools.searchJavaScriptAPI( JavaScriptAPISearchRequest( @@ -131,6 +145,8 @@ let tools = CodeModeAgentTools( `CodeModeFileSystem` receives paths after `PathPolicy` resolution, so sandbox root enforcement stays in CodeMode while the host can route reads, writes, listings, moves, copies, deletes, and stats through another backing implementation. `LocalCodeModeFileSystem` preserves the default `FileManager` behavior. +A host that supplies its own `PathPolicy` must declare `allowedRoots`. `fs.move`, `fs.copy`, and `fs.delete` refuse a sandbox root as either source or destination, and they can only recognize a root the policy names — so there is deliberately no default. + ## Network Access Policy `network.fetch` egress is governed by `CodeModeConfiguration.networkAccessPolicy`: @@ -141,6 +157,12 @@ let tools = CodeModeAgentTools( - `allowedHosts` restricts fetch to an explicit list (entries match the host and its subdomains, and deliberately allowlisted private hosts such as `localhost` are honored); `blockedHosts` refuses specific hosts; `NetworkAccessPolicy.permissive` restores unrestricted behavior. - Matching is by URL host only; DNS resolution is not performed, so a public hostname that resolves to a private address is not detected. Hosts that need stricter guarantees should set `allowedHosts`. +Script traffic also carries no ambient authority: + +- `network.fetch` runs on an isolated ephemeral `URLSession` with no cookie storage, no credential storage, and no shared cache — not `URLSession.shared`, which would let a script ride whatever the host app is already logged in to and let a `Set-Cookie` poison the app's cookie jar. +- Script-supplied `Cookie`, `Authorization`, and `Proxy-*` request headers are refused. Set `allowsCredentialHeaders: true` when scripts legitimately call an authenticated API. +- A host that passes its own `session` to `NetworkBridge` takes responsibility for this; requests still disable per-request cookie handling. + ```swift let tools = CodeModeAgentTools( config: CodeModeConfiguration( @@ -152,6 +174,88 @@ let tools = CodeModeAgentTools( ) ``` +## Agent Guidance + +The tool descriptions and the generated TypeScript tell a model *what the helpers +are called*. Neither tells it *what shape its work should take* — so an agent can +end up using `executeJavaScript` as a thin RPC, one call per operation, which is +the pattern this package exists to replace. + +`CodeModeAgentGuidance` is the missing half. Pair it with `typeDeclarations()`: + +```swift +let systemPrompt = CodeModeAgentGuidance.systemPrompt(.standard) + + "\n\n" + + tools.typeDeclarations() +``` + +Three tiers, each self-contained and additive — dropping to a smaller one loses +examples, never a rule: + +| Tier | ~Tokens | Contents | +| --- | --- | --- | +| `.brief` | ~160 | The core idea: one call runs the whole task; request only the capabilities you use. | +| `.standard` | ~580 | Adds the search → write → repair workflow and a worked multi-step example. | +| `.full` | ~1000 | Adds partial-failure handling, the sequential-native-call reality, and the anti-pattern list. | + +Or let a budget choose: + +```swift +CodeModeAgentGuidance.systemPrompt(approximateTokenBudget: 400) // -> .standard +``` + +Token counts are estimated at four characters per token — a planning bound, not a +real tokenization. A budget too small for any tier still returns `.brief`, on the +grounds that some guidance is the difference between an agent that batches its +work and one that does not. + +The `execution.*` and `fs.whole-job-one-script` eval scenarios grade this: a +transcript that splits one obvious job across several executions, or that asks +for capabilities it never calls, fails. + +## TypeScript Declarations + +The registry already knows argument names, types, optionality, enum constraints, +and hints. That metadata is emitted as TypeScript so the model writes code +against real declarations instead of a coarse type map plus prose: + +```swift +let tools = CodeModeAgentTools( + config: CodeModeConfiguration(capabilityGrant: .only([.fsRead, .fsList])) +) +print(tools.typeDeclarations()) // only what the grant permits +print(tools.typeDeclarations(for: [.keychainRead])) // an explicit subset +print(tools.capabilities()) // every reference, each carrying `dts` +``` + +`typeDeclarations()` follows the host's `CapabilityGrant`, so it never advertises +a helper the model would be denied — and size matters here: the unrestricted iOS +surface is roughly 21,000 tokens, while a grant of a few helpers is around 1,000, +most of it the shared preamble. Without a grant, prefer search-time `ref.dts` or +`typeDeclarations(for:)` over inlining everything. + +Every `JavaScriptAPIReference` carries a per-capability `dts`, so code-driven +search stays the filter and TypeScript becomes the payload: + +```javascript +async () => { + return api.references + .filter(ref => ref.tags.includes("calendar")) + .map(ref => ref.dts) + .join("\n\n"); +} +``` + +Constrained arguments become string-literal unions, dotted argument paths +(`options.timeoutMs`) become nested object types, and argument hints become doc +comments. Results are typed `CodeModeValue` — built-in bridges return untyped +JSON dictionaries, so a narrower result type would be a fiction; the prose result +summary is in the doc comment until per-capability result schemas exist. + +The Node-compatibility globals (`fetch`, `fs.promises.*`, `path`, `console`) are +declared in a hand-authored preamble, because their positional calling convention +is not something the catalog can express. + ## Search `searchJavaScriptAPI` accepts `JavaScriptAPISearchRequest`: @@ -215,11 +319,44 @@ async () => { - `code` - `allowedCapabilities` +- `allowedCapabilityKeys` - `timeoutMs` - `context` It returns a `JavaScriptExecutionCall` immediately. +### Capability allowlists are model-authored; the grant is not + +`allowedCapabilities` and `allowedCapabilityKeys` travel in the advertised tool +schema, so the *model* fills them in. They are a least-privilege declaration and +a useful audit signal, but on their own they are not a sandbox: a host that pipes +tool JSON straight into `executeJavaScript` gives the script whatever the script +asked for. + +`CodeModeConfiguration.capabilityGrant` is the host-owned ceiling. The effective +set is always `requested ∩ granted`: + +```swift +let tools = CodeModeAgentTools( + config: CodeModeConfiguration( + capabilityGrant: .only( + [.fsRead, .fsWrite, .networkFetch], + capabilityKeys: ["myapp.tasks.complete"] + ) + ) +) +``` + +It defaults to `.unrestricted` for source compatibility. Anything the request +declares and the grant withholds fails with `CAPABILITY_DENIED`, is reported to +the model as not repairable by retrying, and is recorded as a +`CAPABILITY_WITHHELD_BY_HOST` diagnostic. + +The two allowlists are strictly disjoint. Built-in `CapabilityID`s are granted +only by `allowedCapabilities`; `allowedCapabilityKeys` accepts arbitrary strings +and reaches custom providers only, so it cannot be used to spell a built-in past +a host that vets the typed field. + Cross-platform privileged helpers are installed under `apple.*`. Platform-specific helpers are installed only where supported, for example `ios.alarm.*` on iOS hosts that support AlarmKit. `apple.location.requestPermission()` is currently exposed only on iOS hosts. Other Apple platforms can expose `apple.location.*` helpers when supported, but the explicit permission-request helper is intentionally hidden outside iOS for now. @@ -242,6 +379,25 @@ System UI helpers are installed only on supported UI platforms. Shared iOS/visio - on failure it throws `CodeModeToolError` - `call.cancel()` interrupts in-flight JavaScript +### Timers and the event loop + +`setTimeout`/`clearTimeout` are real: callbacks are deferred, delays are honoured, +`clearTimeout` cancels, and an error thrown inside a callback becomes a +`TIMER_CALLBACK_ERROR` diagnostic rather than propagating to whoever called +`setTimeout`. So `await new Promise(r => setTimeout(r, 1000))` — the standard +backoff — actually waits instead of spinning. A long wait remains cancellable and +is still bounded by `timeoutMs`. + +There is no I/O event loop yet: the bridge ABI is synchronous, so native calls run +one at a time and `Promise.all` over several helpers completes them sequentially. +Because a timer is the only thing that can advance a pending program, a promise +with no resolve path and no queued timer is provably unsettleable and fails +immediately with `JS_RUNTIME_ERROR` instead of running out the clock. + +Bridge calls that fail while the script still returns successfully — typically a +forgotten `await`, which silently discards the rejection — produce a +`BRIDGE_FAILURES_NOT_SURFACED` warning diagnostic. + ### Timeout and cancellation `timeoutMs` and `cancel()` are enforced preemptively: a JavaScriptCore execution @@ -324,8 +480,12 @@ let required: Set = [ .contactsSearch, .weatherRead, ] +let config = CodeModeConfiguration(capabilityGrant: .only(required)) +// Info.plist usage strings for what the host will request, plus a warning if +// the configuration itself leaves the model as the only capability allowlist. let issues = HostConfigurationValidator.validate(requiredCapabilities: required) + + HostConfigurationValidator.validate(configuration: config) for issue in issues { print("[\(issue.severity.rawValue)] \(issue.key): \(issue.message)") } diff --git a/Sources/CodeMode/API/BridgeModels.swift b/Sources/CodeMode/API/BridgeModels.swift index e922dd9..f3432ef 100644 --- a/Sources/CodeMode/API/BridgeModels.swift +++ b/Sources/CodeMode/API/BridgeModels.swift @@ -3,7 +3,15 @@ import Foundation public struct CodeModeConfiguration: Sendable { public var pathPolicy: any PathPolicy public var networkAccessPolicy: NetworkAccessPolicy + /// Host-owned ceiling on the model-authored `allowedCapabilities` / + /// `allowedCapabilityKeys`. Defaults to `.unrestricted`; see `CapabilityGrant`. + public var capabilityGrant: CapabilityGrant public var fileSystem: any CodeModeFileSystem + /// Byte ceilings applied by the filesystem bridge to `fs.read` / `fs.write`. + public var fileSystemLimits: FileSystemLimits + /// Bounds on retained result/log/event volume per execution, and on how + /// many executions run concurrently. + public var executionLimits: ExecutionLimits public var artifactStore: any ArtifactStore public var permissionBroker: any PermissionBroker public var auditLogger: any AuditLogger @@ -25,7 +33,10 @@ public struct CodeModeConfiguration: Sendable { public init( pathPolicy: any PathPolicy = DefaultPathPolicy(), networkAccessPolicy: NetworkAccessPolicy = .standard, + capabilityGrant: CapabilityGrant = .unrestricted, fileSystem: any CodeModeFileSystem = LocalCodeModeFileSystem(), + fileSystemLimits: FileSystemLimits = .standard, + executionLimits: ExecutionLimits = .standard, artifactStore: any ArtifactStore = InMemoryArtifactStore(), permissionBroker: any PermissionBroker = SystemPermissionBroker(), auditLogger: any AuditLogger = SyncAuditLogger(), @@ -46,7 +57,10 @@ public struct CodeModeConfiguration: Sendable { ) { self.pathPolicy = pathPolicy self.networkAccessPolicy = networkAccessPolicy + self.capabilityGrant = capabilityGrant self.fileSystem = fileSystem + self.fileSystemLimits = fileSystemLimits + self.executionLimits = executionLimits self.artifactStore = artifactStore self.permissionBroker = permissionBroker self.auditLogger = auditLogger @@ -89,6 +103,10 @@ public struct JavaScriptAPIReference: Sendable, Codable, Equatable { public var argumentHints: [String: String] public var argumentConstraints: CapabilityArgumentConstraints public var resultSummary: String + /// TypeScript declaration for this helper, generated from the fields above. + /// Return it from `searchJavaScriptAPI` to give the model real types rather + /// than a coarse `argumentTypes` map plus prose. + public var dts: String public init( capability: String, @@ -103,7 +121,8 @@ public struct JavaScriptAPIReference: Sendable, Codable, Equatable { argumentTypes: [String: CapabilityArgumentType], argumentHints: [String: String], argumentConstraints: CapabilityArgumentConstraints = .none, - resultSummary: String + resultSummary: String, + dts: String = "" ) { self.capability = capability self.capabilityKey = capabilityKey ?? CodeModeCapabilityKey(rawValue: capability) @@ -118,6 +137,38 @@ public struct JavaScriptAPIReference: Sendable, Codable, Equatable { self.argumentHints = argumentHints self.argumentConstraints = argumentConstraints self.resultSummary = resultSummary + self.dts = dts + } + + private enum CodingKeys: String, CodingKey { + case capability, capabilityKey, builtInCapability, jsNames, summary, tags, example + case requiredArguments, optionalArguments, argumentTypes, argumentHints + case argumentConstraints, resultSummary, dts + } + + /// `dts` is optional on the way in. References encoded before it existed — + /// cached search results, eval fixtures — would otherwise fail to decode with + /// `keyNotFound`; when it is absent, the declaration is regenerated from the + /// decoded fields rather than left empty. + public init(from decoder: Decoder) throws { + let container = try decoder.container(keyedBy: CodingKeys.self) + self.capability = try container.decode(String.self, forKey: .capability) + self.capabilityKey = try container.decode(CodeModeCapabilityKey.self, forKey: .capabilityKey) + self.builtInCapability = try container.decodeIfPresent(CapabilityID.self, forKey: .builtInCapability) + self.jsNames = try container.decode([String].self, forKey: .jsNames) + self.summary = try container.decode(String.self, forKey: .summary) + self.tags = try container.decode([String].self, forKey: .tags) + self.example = try container.decode(String.self, forKey: .example) + self.requiredArguments = try container.decode([String].self, forKey: .requiredArguments) + self.optionalArguments = try container.decode([String].self, forKey: .optionalArguments) + self.argumentTypes = try container.decode([String: CapabilityArgumentType].self, forKey: .argumentTypes) + self.argumentHints = try container.decode([String: String].self, forKey: .argumentHints) + self.argumentConstraints = try container.decode(CapabilityArgumentConstraints.self, forKey: .argumentConstraints) + self.resultSummary = try container.decode(String.self, forKey: .resultSummary) + self.dts = try container.decodeIfPresent(String.self, forKey: .dts) ?? "" + if self.dts.isEmpty { + self.dts = TypeScriptDeclarations.declaration(for: self) + } } } @@ -132,6 +183,13 @@ public struct JavaScriptAPISearchResponse: Sendable, Codable, Equatable { } public struct JavaScriptExecutionRequest: Sendable, Codable, Equatable { + /// Bounds advertised in `executeJavaScriptParameterSchema`. Values outside + /// them are clamped rather than rejected: a model that asks for a 10-minute + /// budget should get the ceiling and a running script, not a hard failure. + public static let minimumTimeoutMs = 1 + public static let maximumTimeoutMs = 60_000 + public static let defaultTimeoutMs = 10_000 + public var code: String public var allowedCapabilities: [CapabilityID] public var allowedCapabilityKeys: [CodeModeCapabilityKey] @@ -142,16 +200,20 @@ public struct JavaScriptExecutionRequest: Sendable, Codable, Equatable { code: String, allowedCapabilities: [CapabilityID], allowedCapabilityKeys: [CodeModeCapabilityKey] = [], - timeoutMs: Int = 10_000, + timeoutMs: Int = JavaScriptExecutionRequest.defaultTimeoutMs, context: ExecutionContext = .init() ) { self.code = code self.allowedCapabilities = allowedCapabilities self.allowedCapabilityKeys = allowedCapabilityKeys - self.timeoutMs = timeoutMs + self.timeoutMs = Self.clampTimeoutMs(timeoutMs) self.context = context } + static func clampTimeoutMs(_ value: Int) -> Int { + min(max(value, minimumTimeoutMs), maximumTimeoutMs) + } + private enum CodingKeys: String, CodingKey { case code case allowedCapabilities @@ -194,10 +256,13 @@ public struct JavaScriptExecutionRequest: Sendable, Codable, Equatable { self.allowedCapabilities = capabilities self.allowedCapabilityKeys = try container.decodeIfPresent([CodeModeCapabilityKey].self, forKey: .allowedCapabilityKeys) ?? [] - self.timeoutMs = try Self.decodeTimeoutMs(from: container) ?? 10_000 + self.timeoutMs = Self.clampTimeoutMs(try Self.decodeTimeoutMs(from: container) ?? Self.defaultTimeoutMs) self.context = try container.decodeIfPresent(ExecutionContext.self, forKey: .context) ?? .init() } + // Every conversion here is total. `Int.init(_: Double)` traps on non-finite + // and out-of-range input, and this runs on model-authored JSON before any + // script does — `{"timeoutMs": 1e300}` would kill the host process in-process. private static func decodeTimeoutMs(from container: KeyedDecodingContainer) throws -> Int? { guard container.contains(.timeoutMs), try !container.decodeNil(forKey: .timeoutMs) else { return nil @@ -206,11 +271,11 @@ public struct JavaScriptExecutionRequest: Sendable, Codable, Equatable { return value } if let value = try? container.decode(Double.self, forKey: .timeoutMs) { - return Int(value) + return try requireRepresentable(value, in: container) } if let string = try? container.decode(String.self, forKey: .timeoutMs), let value = Double(string.trimmingCharacters(in: .whitespaces)) { - return Int(value) + return try requireRepresentable(value, in: container) } throw DecodingError.dataCorruptedError( forKey: .timeoutMs, @@ -218,6 +283,20 @@ public struct JavaScriptExecutionRequest: Sendable, Codable, Equatable { debugDescription: "timeoutMs must be an integer or a numeric string." ) } + + private static func requireRepresentable( + _ value: Double, + in container: KeyedDecodingContainer + ) throws -> Int { + guard let converted = JSONValue.exactInt(from: value) else { + throw DecodingError.dataCorruptedError( + forKey: .timeoutMs, + in: container, + debugDescription: "timeoutMs must be a finite number representable as an integer; received \(value)." + ) + } + return converted + } } public enum JavaScriptExecutionEvent: Sendable, Equatable { @@ -299,22 +378,32 @@ public final class JavaScriptExecutionCall: @unchecked Sendable { public let events: AsyncStream private let resultTask: Task + /// Keeps the execution alive while this handle is; the events stream holds + /// the same token. When both are gone the execution is cancelled, so a call + /// nobody can observe does not keep touching the filesystem, the network, + /// and system UI on the user's behalf. + private let observation: ExecutionObservationToken private let cancelImpl: @Sendable () -> Void init( events: AsyncStream, resultTask: Task, + observation: ExecutionObservationToken, cancelImpl: @escaping @Sendable () -> Void ) { self.events = events self.resultTask = resultTask + self.observation = observation self.cancelImpl = cancelImpl } public var result: JavaScriptExecutionResult { get async throws { let outcome = await withTaskCancellationHandler { - await waitForResultOutcome() + // `resultTask.result` already detaches the await from the calling + // task's cancellation, so the previous `Task.detached` per access + // bought nothing but an extra task. + await resultTask.result } onCancel: { self.cancel() } @@ -324,8 +413,7 @@ public final class JavaScriptExecutionCall: @unchecked Sendable { return result case let .failure(error as CodeModeToolError): throw error - case let .failure(error as CancellationError): - _ = error + case .failure(is CancellationError): throw CodeModeToolError(code: "CANCELLED", message: "Execution cancelled") case let .failure(error): throw error @@ -337,18 +425,6 @@ public final class JavaScriptExecutionCall: @unchecked Sendable { cancelImpl() resultTask.cancel() } - - private func waitForResultOutcome() async -> Result { - await withCheckedContinuation { continuation in - Task.detached { - do { - continuation.resume(returning: .success(try await self.resultTask.value)) - } catch { - continuation.resume(returning: .failure(error)) - } - } - } - } } public struct ExecutionContext: Sendable, Codable, Equatable { diff --git a/Sources/CodeMode/API/SystemAppleServiceClients.swift b/Sources/CodeMode/API/SystemAppleServiceClients.swift index 1f3db40..c501be0 100644 --- a/Sources/CodeMode/API/SystemAppleServiceClients.swift +++ b/Sources/CodeMode/API/SystemAppleServiceClients.swift @@ -819,8 +819,14 @@ enum SystemMapsMapping { private func openSystemURL(_ url: URL) -> Bool { #if canImport(UIKit) if Thread.isMainThread { - UIApplication.shared.open(url, options: [:], completionHandler: nil) - return true + // `UIApplication.shared` and `open` are main-actor-isolated. We have + // already established we are on the main thread, so state that to the + // compiler rather than hopping — a hop here would return before the open + // has been requested. + return MainActor.assumeIsolated { + UIApplication.shared.open(url, options: [:], completionHandler: nil) + return true + } } let semaphore = DispatchSemaphore(value: 0) let opened = LockedBox(false) diff --git a/Sources/CodeMode/Bridges/AlarmBridge.swift b/Sources/CodeMode/Bridges/AlarmBridge.swift index 4736f96..6931d19 100644 --- a/Sources/CodeMode/Bridges/AlarmBridge.swift +++ b/Sources/CodeMode/Bridges/AlarmBridge.swift @@ -139,13 +139,7 @@ public final class AlarmBridge: @unchecked Sendable { } private func isoDate(_ text: String?) -> Date? { - guard let text else { return nil } - let formatter = ISO8601DateFormatter() - if let date = formatter.date(from: text) { - return date - } - formatter.formatOptions = [.withInternetDateTime, .withFractionalSeconds] - return formatter.date(from: text) + CodeModeDate.parse(text) } #if canImport(AlarmKit) && canImport(SwiftUI) && os(iOS) diff --git a/Sources/CodeMode/Bridges/BigTicketAppleBridges.swift b/Sources/CodeMode/Bridges/BigTicketAppleBridges.swift index 55090e8..1160d8c 100644 --- a/Sources/CodeMode/Bridges/BigTicketAppleBridges.swift +++ b/Sources/CodeMode/Bridges/BigTicketAppleBridges.swift @@ -111,7 +111,12 @@ public final class SpeechBridge: @unchecked Sendable { try ensurePermission(.speechRecognition, context: context) let path = try requireNonEmptyString("path", in: arguments, capability: "speech.file.transcribe") let resolved = try context.pathPolicy.resolve(path: path) + // The raw, script-supplied `path` is replaced rather than accompanied: + // leaving both in the dictionary made every host adapter one + // `arguments.string("path")` away from a path-policy bypass, with nothing + // in the code saying which key was the safe one. var clientArguments = arguments + clientArguments["path"] = .string(resolved.path) clientArguments["resolvedPath"] = .string(resolved.path) try validateOptionalPositiveNumber("timeoutMs", in: arguments, capability: "speech.file.transcribe") return try client.transcribeFile(arguments: clientArguments) @@ -348,7 +353,9 @@ public final class PassKitBridge: @unchecked Sendable { public func addPass(arguments: [String: JSONValue], context: BridgeInvocationContext) throws -> JSONValue { let path = try requireNonEmptyString("path", in: arguments, capability: "passkit.pass.add") let resolved = try context.pathPolicy.resolve(path: path) + // As above: the raw path does not survive into the client dictionary. var clientArguments = arguments + clientArguments["path"] = .string(resolved.path) clientArguments["resolvedPath"] = .string(resolved.path) return try client.addPass(arguments: clientArguments) } diff --git a/Sources/CodeMode/Bridges/CapabilityRegistrations+Core.swift b/Sources/CodeMode/Bridges/CapabilityRegistrations+Core.swift index 706b4af..1c210e3 100644 --- a/Sources/CodeMode/Bridges/CapabilityRegistrations+Core.swift +++ b/Sources/CodeMode/Bridges/CapabilityRegistrations+Core.swift @@ -14,7 +14,7 @@ extension DefaultCapabilityRegistrationBuilder { descriptor: .init( id: .networkFetch, title: "Fetch HTTP resource", - summary: "Perform HTTP(S) requests through URLSession via a fetch-compatible API.", + summary: "Perform HTTP(S) requests through an isolated URLSession via a fetch-compatible API. Requests carry no app cookies or stored credentials, and credential headers (Cookie, Authorization, Proxy-*) are refused unless the host app permits them.", tags: ["network", "http", "fetch"], example: "await fetch('https://api.example.com/data').then(r => r.json())", requiredArguments: ["url"], @@ -29,7 +29,7 @@ extension DefaultCapabilityRegistrationBuilder { argumentHints: [ "url": "Absolute HTTP(S) URL string.", "options.method": "HTTP method; defaults to GET.", - "options.headers": "Object of header key/value string pairs.", + "options.headers": "Object of header key/value string pairs. Cookie, Authorization, and Proxy-* are refused unless the host app permits credential headers.", "options.body": "UTF-8 request body string.", "options.bodyBase64": "Base64-encoded request body. Mutually exclusive with options.body.", "options.timeoutMs": "Request and bridge wait timeout in milliseconds; defaults to 30000.", diff --git a/Sources/CodeMode/Bridges/DefaultCapabilityLoader.swift b/Sources/CodeMode/Bridges/DefaultCapabilityLoader.swift index 96eb81e..3985f55 100644 --- a/Sources/CodeMode/Bridges/DefaultCapabilityLoader.swift +++ b/Sources/CodeMode/Bridges/DefaultCapabilityLoader.swift @@ -3,6 +3,7 @@ import Foundation public enum DefaultCapabilityLoader { public static func loadAllRegistrations( fileSystem: any CodeModeFileSystem = LocalCodeModeFileSystem(), + fileSystemLimits: FileSystemLimits = .standard, networkAccessPolicy: NetworkAccessPolicy = .standard, eventInbox: any CodeModeEventInbox = UnavailableCodeModeEventInbox(), cloudKitClient: any CloudKitClient = UnavailableCloudKitClient(), @@ -18,6 +19,7 @@ public enum DefaultCapabilityLoader { ) -> [CapabilityRegistration] { DefaultCapabilityRegistrationBuilder( fileSystem: fileSystem, + fileSystemLimits: fileSystemLimits, networkAccessPolicy: networkAccessPolicy, eventInbox: eventInbox, cloudKitClient: cloudKitClient, @@ -64,6 +66,7 @@ struct DefaultCapabilityRegistrationBuilder { init( fileSystem: any CodeModeFileSystem, + fileSystemLimits: FileSystemLimits, networkAccessPolicy: NetworkAccessPolicy, eventInbox: any CodeModeEventInbox, cloudKitClient: any CloudKitClient, @@ -77,7 +80,7 @@ struct DefaultCapabilityRegistrationBuilder { passKitClient: any PassKitClient, storeKitClient: any StoreKitClient ) { - self.fs = FileSystemBridge(fileSystem: fileSystem) + self.fs = FileSystemBridge(fileSystem: fileSystem, limits: fileSystemLimits) self.network = NetworkBridge(policy: networkAccessPolicy) self.keychain = KeychainBridge() self.location = LocationBridge() @@ -91,7 +94,7 @@ struct DefaultCapabilityRegistrationBuilder { self.health = HealthBridge() self.home = HomeBridge() self.media = MediaBridge() - self.systemUI = SystemUIBridge() + self.systemUI = SystemUIBridge(networkAccessPolicy: networkAccessPolicy) self.eventInbox = eventInbox self.cloudKit = CloudKitBridge(client: cloudKitClient, eventInbox: eventInbox) self.remoteNotifications = RemoteNotificationsBridge(client: remoteNotificationsClient, eventInbox: eventInbox) diff --git a/Sources/CodeMode/Bridges/EventKitBridge.swift b/Sources/CodeMode/Bridges/EventKitBridge.swift index 279c31b..9750c4a 100644 --- a/Sources/CodeMode/Bridges/EventKitBridge.swift +++ b/Sources/CodeMode/Bridges/EventKitBridge.swift @@ -5,6 +5,8 @@ import EventKit #endif public final class EventKitBridge: @unchecked Sendable { + static let reminderFetchTimeoutSeconds: TimeInterval = 15 + public init() {} public func readEvents(arguments: [String: JSONValue], context: BridgeInvocationContext) throws -> JSONValue { @@ -15,8 +17,13 @@ public final class EventKitBridge: @unchecked Sendable { #if canImport(EventKit) let store = EKEventStore() - let start = isoDate(arguments.string("start")) ?? Date() - let end = isoDate(arguments.string("end")) ?? Calendar.current.date(byAdding: .day, value: 14, to: start) ?? start + // A value that is present but unparseable is an error, not a reason to + // fall back to now/+14d: that handed the script a plausible but wrong + // window with no diagnostic to notice it by. + let start = try CodeModeDate.optional(arguments.string("start"), argument: "start", capability: "calendar.read") ?? Date() + let end = try CodeModeDate.optional(arguments.string("end"), argument: "end", capability: "calendar.read") + ?? Calendar.current.date(byAdding: .day, value: 14, to: start) + ?? start let limit = arguments.int("limit") ?? 50 let calendars = try resolveCalendars( from: arguments, @@ -97,11 +104,9 @@ public final class EventKitBridge: @unchecked Sendable { #if canImport(EventKit) let store = EKEventStore() - let semaphore = DispatchSemaphore(value: 0) - var result: [JSONValue] = [] let includeCompleted = arguments.bool("includeCompleted") ?? false - let start = isoDate(arguments.string("start")) - let end = isoDate(arguments.string("end")) + let start = try CodeModeDate.optional(arguments.string("start"), argument: "start", capability: "reminders.read") + let end = try CodeModeDate.optional(arguments.string("end"), argument: "end", capability: "reminders.read") let limit = max(1, arguments.int("limit") ?? 50) let calendars = try resolveCalendars( from: arguments, @@ -113,23 +118,30 @@ public final class EventKitBridge: @unchecked Sendable { let predicate = includeCompleted ? store.predicateForReminders(in: calendars) : store.predicateForIncompleteReminders(withDueDateStarting: start, ending: end, calendars: calendars) - store.fetchReminders(matching: predicate) { reminders in - let filtered = (reminders ?? []) - .filter { reminder in - guard includeCompleted else { return true } - guard let dueDate = reminder.dueDateComponents?.date else { - return start == nil && end == nil + // Previously a `var` written from EventKit's completion queue, read after + // a semaphore wait whose timeout result was discarded — a data race with + // the late callback, and an empty array on timeout that the script could + // not tell apart from "no reminders". + let result = try CompletionWait.value( + timeout: Self.reminderFetchTimeoutSeconds, + operationName: "reminders.read" + ) { complete in + store.fetchReminders(matching: predicate) { reminders in + let filtered = (reminders ?? []) + .filter { reminder in + guard includeCompleted else { return true } + guard let dueDate = reminder.dueDateComponents?.date else { + return start == nil && end == nil + } + if let start, dueDate < start { return false } + if let end, dueDate > end { return false } + return true } - if let start, dueDate < start { return false } - if let end, dueDate > end { return false } - return true - } - .prefix(limit) - result = filtered.map { Self.reminderJSON($0) } - semaphore.signal() + .prefix(limit) + complete(filtered.map { Self.reminderJSON($0) }) + } } - _ = semaphore.wait(timeout: .now() + 15) return .array(result) #else _ = arguments @@ -191,8 +203,7 @@ public final class EventKitBridge: @unchecked Sendable { } private func isoDate(_ text: String?) -> Date? { - guard let text, text.isEmpty == false else { return nil } - return ISO8601DateFormatter().date(from: text) + CodeModeDate.parse(text) } private func eventOperation(_ arguments: [String: JSONValue]) throws -> CalendarWriteOperation { @@ -480,6 +491,17 @@ public final class EventKitBridge: @unchecked Sendable { return span.ekSpan } + /// Test seam: these serializers are the only EventKit logic reachable + /// without a live, permission-granted store, and they are where the + /// implicitly-unwrapped `calendar`/`title` crashes lived. + static func eventJSONForTesting(_ event: EKEvent) -> JSONValue { + eventJSON(event) + } + + static func reminderJSONForTesting(_ reminder: EKReminder) -> JSONValue { + reminderJSON(reminder) + } + private static func eventJSON(_ event: EKEvent) -> JSONValue { .object([ "identifier": .string(event.eventIdentifier ?? ""), @@ -487,8 +509,11 @@ public final class EventKitBridge: @unchecked Sendable { "startDate": .string(event.startDate.ISO8601Format()), "endDate": .string(event.endDate.ISO8601Format()), "notes": .string(event.notes ?? ""), - "calendarIdentifier": .string(event.calendar.calendarIdentifier), - "calendarTitle": .string(event.calendar.title), + // `EKEvent.calendar` is `EKCalendar!` and is nil for an orphaned + // event — every neighboring field here is nil-coalesced, and this one + // crashed the execution thread. + "calendarIdentifier": .string(event.calendar?.calendarIdentifier ?? ""), + "calendarTitle": .string(event.calendar?.title ?? ""), "location": .string(event.location ?? ""), "url": .string(event.url?.absoluteString ?? ""), "isAllDay": .bool(event.isAllDay), @@ -498,7 +523,8 @@ public final class EventKitBridge: @unchecked Sendable { private static func reminderJSON(_ reminder: EKReminder) -> JSONValue { var object: [String: JSONValue] = [ "identifier": .string(reminder.calendarItemIdentifier), - "title": .string(reminder.title), + // `EKReminder.title` is `String!`. + "title": .string(reminder.title ?? ""), "isCompleted": .bool(reminder.isCompleted), "dueDate": .string(reminder.dueDateComponents?.date?.ISO8601Format() ?? ""), "completionDate": .string(reminder.completionDate?.ISO8601Format() ?? ""), diff --git a/Sources/CodeMode/Bridges/FileSystemBridge.swift b/Sources/CodeMode/Bridges/FileSystemBridge.swift index a4b8be9..aba3ec1 100644 --- a/Sources/CodeMode/Bridges/FileSystemBridge.swift +++ b/Sources/CodeMode/Bridges/FileSystemBridge.swift @@ -1,14 +1,40 @@ import Foundation +/// Byte ceilings for the filesystem bridge. +/// +/// A sandbox root can hold files far larger than a script's result budget — a +/// 2 GB video in `documents:` read whole and base64-serialized is an immediate +/// jetsam on iOS — so reads and writes are bounded and fail with a diagnostic +/// instead of exhausting memory. +public struct FileSystemLimits: Sendable, Equatable { + public var maxReadBytes: Int + public var maxWriteBytes: Int + + public init(maxReadBytes: Int = 16 * 1_024 * 1_024, maxWriteBytes: Int = 16 * 1_024 * 1_024) { + self.maxReadBytes = maxReadBytes + self.maxWriteBytes = maxWriteBytes + } + + public static let standard = FileSystemLimits() + + /// No ceiling. Only appropriate when the host has its own quota. + public static let unlimited = FileSystemLimits(maxReadBytes: .max, maxWriteBytes: .max) +} + public final class FileSystemBridge: @unchecked Sendable { private let fileSystem: any CodeModeFileSystem + private let limits: FileSystemLimits - public init(fileSystem: any CodeModeFileSystem = LocalCodeModeFileSystem()) { + public init( + fileSystem: any CodeModeFileSystem = LocalCodeModeFileSystem(), + limits: FileSystemLimits = .standard + ) { self.fileSystem = fileSystem + self.limits = limits } - public convenience init(fileManager: FileManager) { - self.init(fileSystem: LocalCodeModeFileSystem(fileManager: fileManager)) + public convenience init(fileManager: FileManager, limits: FileSystemLimits = .standard) { + self.init(fileSystem: LocalCodeModeFileSystem(fileManager: fileManager), limits: limits) } public func list(arguments: [String: JSONValue], context: BridgeInvocationContext) throws -> JSONValue { @@ -36,11 +62,23 @@ public final class FileSystemBridge: @unchecked Sendable { let encoding = arguments.string("encoding") ?? "utf8" let url = try context.pathPolicy.resolve(path: path) + + // Checked against stat first: reading and then rejecting has already spent + // the memory the cap exists to protect. Re-checked after, because a + // custom CodeModeFileSystem's stat may be absent or stale. + try enforceReadLimit(try? fileSystem.attributesOfItem(at: url).size, at: url) let data = try fileSystem.readData(at: url) + try enforceReadLimit(data.count, at: url) switch encoding.lowercased() { case "utf8", "utf-8": - let text = String(data: data, encoding: .utf8) ?? "" + // Silently substituting "" for undecodable bytes tells the script the + // file is empty, which is worse than a failure it can act on. + guard let text = String(data: data, encoding: .utf8) else { + throw BridgeError.invalidArguments( + "fs.read could not decode \(url.lastPathComponent) as UTF-8. Retry with encoding: 'base64' for binary data." + ) + } return .object([ "path": .string(url.path), "text": .string(text), @@ -63,9 +101,6 @@ public final class FileSystemBridge: @unchecked Sendable { let encoding = arguments.string("encoding") ?? "utf8" let url = try context.pathPolicy.resolve(path: path) - let parent = url.deletingLastPathComponent() - try fileSystem.createDirectory(at: parent, recursive: true) - let data: Data switch encoding.lowercased() { case "utf8", "utf-8": @@ -80,6 +115,13 @@ public final class FileSystemBridge: @unchecked Sendable { throw BridgeError.invalidArguments("Unsupported encoding: \(encoding)") } + guard data.count <= limits.maxWriteBytes else { + throw Self.overLimit("fs.write", url: url, bytes: data.count, limit: limits.maxWriteBytes) + } + + // Only after the payload is known good, so a rejected write leaves no + // directories behind. + try fileSystem.createDirectory(at: url.deletingLastPathComponent(), recursive: true) try fileSystem.writeData(data, to: url) return .object([ "path": .string(url.path), @@ -94,12 +136,11 @@ public final class FileSystemBridge: @unchecked Sendable { let fromURL = try context.pathPolicy.resolve(path: from) let toURL = try context.pathPolicy.resolve(path: to) + try validateTransfer(from: fromURL, to: toURL, operation: "fs.move", arguments: arguments, context: context) - if fileSystem.itemExists(at: toURL) { - try fileSystem.removeItem(at: toURL) + try replacing(toURL, operation: "fs.move", arguments: arguments) { + try fileSystem.moveItem(at: fromURL, to: toURL) } - - try fileSystem.moveItem(at: fromURL, to: toURL) return .object([ "from": .string(fromURL.path), "to": .string(toURL.path), @@ -113,18 +154,154 @@ public final class FileSystemBridge: @unchecked Sendable { let fromURL = try context.pathPolicy.resolve(path: from) let toURL = try context.pathPolicy.resolve(path: to) + try validateTransfer(from: fromURL, to: toURL, operation: "fs.copy", arguments: arguments, context: context) - if fileSystem.itemExists(at: toURL) { - try fileSystem.removeItem(at: toURL) + // A copy writes new bytes, so it answers to the write limit — otherwise + // copying a large tree is a way around it, and fills the disk. + let bytes = try totalSize(of: fromURL, stoppingAbove: limits.maxWriteBytes) + guard bytes <= limits.maxWriteBytes else { + throw Self.overLimit("fs.copy", url: fromURL, bytes: bytes, limit: limits.maxWriteBytes) } - try fileSystem.copyItem(at: fromURL, to: toURL) + try replacing(toURL, operation: "fs.copy", arguments: arguments) { + try fileSystem.copyItem(at: fromURL, to: toURL) + } return .object([ "from": .string(fromURL.path), "to": .string(toURL.path), ]) } + private func enforceReadLimit(_ bytes: Int?, at url: URL) throws { + guard let bytes, bytes > limits.maxReadBytes else { + return + } + throw Self.overLimit("fs.read", url: url, bytes: bytes, limit: limits.maxReadBytes) + } + + private static func overLimit(_ operation: String, url: URL, bytes: Int, limit: Int) -> BridgeError { + .invalidArguments( + "\(operation) refused \(url.lastPathComponent): \(bytes) bytes exceeds the \(limit)-byte limit. Use a smaller payload, or have the host raise CodeModeConfiguration.fileSystemLimits." + ) + } + + /// Everything that must hold before a move or copy touches the disk. + /// + /// Containment alone admits a sandbox root: `documents:` resolves to the + /// documents root itself. On the destination side that let + /// `fs.move({ to: 'documents:' })` delete Documents; on the source side + /// `fs.move({ from: 'documents:', to: 'tmp:x' })` moved Documents into a + /// purgeable directory, and `fs.copy` from a root duplicated the whole tree. + /// Both sides are refused, matching `fs.delete`. + private func validateTransfer( + from fromURL: URL, + to toURL: URL, + operation: String, + arguments: [String: JSONValue], + context: BridgeInvocationContext + ) throws { + guard context.pathPolicy.isAllowedRoot(fromURL) == false else { + throw BridgeError.pathViolation( + "\(operation) refuses a sandbox root as its source. Name a path inside the root, for example 'documents:archive'." + ) + } + guard context.pathPolicy.isAllowedRoot(toURL) == false else { + throw BridgeError.pathViolation( + "\(operation) refuses a sandbox root as its destination. Name a path inside the root, for example 'documents:archive/file.txt'." + ) + } + + // Checked before anything is removed. Without it, a missing source with + // overwrite: true deleted the destination and then failed. + guard fileSystem.itemExists(at: fromURL) else { + throw BridgeError.invalidArguments("\(operation) source does not exist: \(fromURL.lastPathComponent)") + } + + // Same path, or a destination inside the source: the operation can never + // succeed, and with overwrite: true the old order removed the destination + // first — for `from == to`, that was the source itself. + let fromPath = fromURL.standardizedFileURL.path + let toPath = toURL.standardizedFileURL.path + guard toPath != fromPath, toPath.hasPrefix(fromPath + "/") == false else { + throw BridgeError.invalidArguments("\(operation) destination cannot be the source or a path inside it.") + } + + guard fileSystem.itemExists(at: toURL) else { + return + } + + guard arguments.bool("overwrite") == true else { + throw BridgeError.invalidArguments( + "\(operation) destination already exists. Pass overwrite: true to replace it, or choose a different 'to' path." + ) + } + + let isDirectory = (try? fileSystem.attributesOfItem(at: toURL))?.isDirectory ?? false + if isDirectory, arguments.bool("recursive") != true { + throw BridgeError.invalidArguments( + "\(operation) destination is a directory. Pass recursive: true along with overwrite: true to replace it and everything under it." + ) + } + } + + /// Runs `operation`, replacing an existing destination only if it succeeds. + /// + /// The destination is moved aside rather than deleted, the operation runs, + /// and only then is the old item removed; on failure it is moved back. The + /// previous remove-then-act order meant any failure after the remove — a + /// missing source, a full disk, a permissions error — lost the destination + /// along with the operation. + private func replacing( + _ toURL: URL, + operation: String, + arguments: [String: JSONValue], + _ body: () throws -> Void + ) throws { + guard fileSystem.itemExists(at: toURL) else { + try body() + return + } + + // Same directory, so the aside move is a rename, not a copy. + let aside = toURL + .deletingLastPathComponent() + .appendingPathComponent(".\(toURL.lastPathComponent).codemode-replacing-\(UUID().uuidString)") + try fileSystem.moveItem(at: toURL, to: aside) + + do { + try body() + } catch { + try? fileSystem.moveItem(at: aside, to: toURL) + throw error + } + try? fileSystem.removeItem(at: aside) + } + + /// Total bytes under `url`, stopping as soon as the running total exceeds + /// `limit` so a huge tree is not walked in full just to be refused. + private func totalSize(of url: URL, stoppingAbove limit: Int) throws -> Int { + let attributes = try fileSystem.attributesOfItem(at: url) + guard attributes.isDirectory else { + return attributes.size + } + + var total = 0 + var pending = [url] + while let directory = pending.popLast() { + for entry in try fileSystem.listDirectory(at: directory) { + if entry.isDirectory { + pending.append(URL(fileURLWithPath: entry.path)) + } else { + total += entry.size + if total > limit { + return total + } + } + } + } + return total + } + public func delete(arguments: [String: JSONValue], context: BridgeInvocationContext) throws -> JSONValue { guard let path = arguments.string("path") else { throw BridgeError.invalidArguments("fs.delete requires 'path'") @@ -137,6 +314,12 @@ public final class FileSystemBridge: @unchecked Sendable { return .object(["deleted": .bool(false), "path": .string(url.path)]) } + guard context.pathPolicy.isAllowedRoot(url) == false else { + throw BridgeError.pathViolation( + "fs.delete refuses a sandbox root. Delete entries inside it individually." + ) + } + let attrs = try fileSystem.attributesOfItem(at: url) if attrs.isDirectory, recursive == false { throw BridgeError.invalidArguments("fs.delete requires recursive=true for directories") diff --git a/Sources/CodeMode/Bridges/FileSystemCodeModeTools.swift b/Sources/CodeMode/Bridges/FileSystemCodeModeTools.swift index 650190b..ca296cb 100644 --- a/Sources/CodeMode/Bridges/FileSystemCodeModeTools.swift +++ b/Sources/CodeMode/Bridges/FileSystemCodeModeTools.swift @@ -1,7 +1,16 @@ import Foundation -// Filesystem capabilities. Flat arguments, no constrained values, so no -// CodeModeStringEnum is needed here. +// Filesystem capabilities. + +/// The encodings `fs.read`/`fs.write` accept. Declared as an enum so the catalog +/// advertises `allowedStringValues` instead of leaving the constraint in prose, +/// where the model cannot discover it. +enum FileEncoding: String, CodeModeStringEnum { + case utf8 + case base64 + + static let codeModeAliases: [String: FileEncoding] = ["utf-8": .utf8] +} @BuiltInCodeMode(.fsList, path: "apple.fs.list", aliases: ["fs.promises.readdir"]) struct FSListTool: BuiltInCodeModeTool { @@ -35,8 +44,8 @@ struct FSReadTool: BuiltInCodeModeTool { struct Arguments: Sendable { @ToolParam("File path using allowed root prefix.") var path: String - @ToolParam("utf8 (default) or base64.") - var encoding: String? + @ToolParam("utf8 (default) or base64. Binary files must use base64; a utf8 read of undecodable bytes fails rather than returning empty text.") + var encoding: FileEncoding? var raw: [String: JSONValue] } @@ -61,7 +70,7 @@ struct FSWriteTool: BuiltInCodeModeTool { @ToolParam("UTF-8 text or base64 string depending on encoding.") var data: String? @ToolParam("utf8 (default) or base64.") - var encoding: String? + var encoding: FileEncoding? var raw: [String: JSONValue] } @@ -75,7 +84,7 @@ struct FSWriteTool: BuiltInCodeModeTool { @BuiltInCodeMode(.fsMove, path: "apple.fs.move", aliases: ["fs.promises.rename"]) struct FSMoveTool: BuiltInCodeModeTool { static let codeModeTitle = "Move file" - static let codeModeSummary = "Move file/directory within allowed sandbox roots." + static let codeModeSummary = "Move file/directory within allowed sandbox roots. Fails rather than clobbering an existing destination unless overwrite is set, and never accepts a sandbox root as the destination." static let codeModeTags = ["filesystem", "io", "fs"] static let codeModeExample = "await apple.fs.move({ from: 'tmp:a.txt', to: 'tmp:b.txt' })" static let codeModeResultSummary = "Object with from/to resolved paths." @@ -83,8 +92,12 @@ struct FSMoveTool: BuiltInCodeModeTool { struct Arguments: Sendable { @ToolParam("Source sandbox path.") var from: String - @ToolParam("Destination sandbox path.") + @ToolParam("Destination sandbox path. Must name a path inside a root, not the root itself.") var to: String + @ToolParam("Required as true to replace an existing destination; default false.") + var overwrite: Bool? + @ToolParam("Required as true alongside overwrite when the existing destination is a directory.") + var recursive: Bool? var raw: [String: JSONValue] } @@ -98,7 +111,7 @@ struct FSMoveTool: BuiltInCodeModeTool { @BuiltInCodeMode(.fsCopy, path: "apple.fs.copy", aliases: ["fs.promises.copyFile"]) struct FSCopyTool: BuiltInCodeModeTool { static let codeModeTitle = "Copy file" - static let codeModeSummary = "Copy file/directory within allowed sandbox roots." + static let codeModeSummary = "Copy file/directory within allowed sandbox roots. Fails rather than clobbering an existing destination unless overwrite is set, and never accepts a sandbox root as the destination." static let codeModeTags = ["filesystem", "io", "fs"] static let codeModeExample = "await apple.fs.copy({ from: 'tmp:a.txt', to: 'tmp:b.txt' })" static let codeModeResultSummary = "Object with from/to resolved paths." @@ -106,8 +119,12 @@ struct FSCopyTool: BuiltInCodeModeTool { struct Arguments: Sendable { @ToolParam("Source sandbox path.") var from: String - @ToolParam("Destination sandbox path.") + @ToolParam("Destination sandbox path. Must name a path inside a root, not the root itself.") var to: String + @ToolParam("Required as true to replace an existing destination; default false.") + var overwrite: Bool? + @ToolParam("Required as true alongside overwrite when the existing destination is a directory.") + var recursive: Bool? var raw: [String: JSONValue] } diff --git a/Sources/CodeMode/Bridges/HealthBridge.swift b/Sources/CodeMode/Bridges/HealthBridge.swift index 0732e72..7f584ba 100644 --- a/Sources/CodeMode/Bridges/HealthBridge.swift +++ b/Sources/CodeMode/Bridges/HealthBridge.swift @@ -26,11 +26,21 @@ public final class HealthBridge: @unchecked Sendable { let writeTypes = try requestedWriteTypes(from: writeNames) try ensureAuthorization(toShare: writeTypes, read: readTypes, context: context, forceRequest: true) + // `requestAuthorization`'s `success` means the sheet flow completed, not + // that anything was granted — HealthKit deliberately refuses to disclose + // read authorization so an app cannot infer a health condition from a + // denial. Reporting `granted: true` here was a fabrication that left the + // transcript claiming access while reads returned []. Share types are the + // only ones with an answerable status, so those are the only ones + // reported. return .object([ - "status": .string(PermissionStatus.granted.rawValue), - "granted": .bool(true), + "status": .string("completed"), + "authorizationFlowCompleted": .bool(true), "readTypes": .array(readNames.map(JSONValue.string)), "writeTypes": .array(writeNames.map(JSONValue.string)), + "writeAuthorization": .object(writeAuthorizationStatuses(for: writeNames)), + "readAuthorizationDisclosed": .bool(false), + "note": .string("HealthKit does not disclose read authorization. An empty health.read result may mean the user denied that type rather than that no samples exist."), ]) #else _ = arguments @@ -150,15 +160,7 @@ public final class HealthBridge: @unchecked Sendable { } private func parseISODate(_ value: String?) -> Date? { - guard let value else { return nil } - - let formatter = ISO8601DateFormatter() - if let date = formatter.date(from: value) { - return date - } - - formatter.formatOptions = [.withInternetDateTime, .withFractionalSeconds] - return formatter.date(from: value) + CodeModeDate.parse(value) } #if canImport(HealthKit) @@ -320,6 +322,44 @@ public final class HealthBridge: @unchecked Sendable { return types } + /// Per-type share authorization, which is the only part HealthKit answers + /// honestly. Read authorization is deliberately undisclosed by the framework. + private func writeAuthorizationStatuses(for names: [String]) -> [String: JSONValue] { + var statuses: [String: JSONValue] = [:] + for name in names { + guard let spec = try? writeSpec(for: name), + let quantityType = HKObjectType.quantityType(forIdentifier: spec.identifier) + else { + continue + } + statuses[name] = .string(Self.shareStatusName(healthStore.authorizationStatus(for: quantityType))) + } + return statuses + } + + /// `.granted` only when every requested share type is actually authorized; + /// otherwise `.notDetermined`, because HealthKit will not say more. + private func resolvedShareAuthorization(for toShare: Set) -> PermissionStatus { + guard toShare.isEmpty == false else { + return .notDetermined + } + let allShared = toShare.allSatisfy { healthStore.authorizationStatus(for: $0) == .sharingAuthorized } + return allShared ? .granted : .notDetermined + } + + private static func shareStatusName(_ status: HKAuthorizationStatus) -> String { + switch status { + case .sharingAuthorized: + return PermissionStatus.granted.rawValue + case .sharingDenied: + return PermissionStatus.denied.rawValue + case .notDetermined: + return PermissionStatus.notDetermined.rawValue + @unknown default: + return PermissionStatus.unavailable.rawValue + } + } + private func ensureAuthorization( toShare: Set, read: Set, @@ -340,7 +380,12 @@ public final class HealthBridge: @unchecked Sendable { let requestStatus = try authorizationRequestStatus(toShare: toShare, read: read) switch requestStatus { case .unnecessary: - context.recordPermission(.healthKit, status: .granted) + // `.unnecessary` means "there is nothing left to ask the user", not + // "access was granted" — for read types the user may well have + // denied. Record what is actually known: granted only when every + // share type says so, otherwise leave it undetermined rather than + // writing a granted claim into the transcript. + context.recordPermission(.healthKit, status: resolvedShareAuthorization(for: toShare)) case .shouldRequest, .unknown: context.recordPermission(.healthKit, status: .notDetermined) let granted = try requestAuthorization(toShare: toShare, read: read) diff --git a/Sources/CodeMode/Bridges/KeychainBridge.swift b/Sources/CodeMode/Bridges/KeychainBridge.swift index 1668d48..3f03c72 100644 --- a/Sources/CodeMode/Bridges/KeychainBridge.swift +++ b/Sources/CodeMode/Bridges/KeychainBridge.swift @@ -8,6 +8,22 @@ public final class KeychainBridge: @unchecked Sendable { self.service = service } + /// A failure that says what went wrong, not just which OSStatus it was. + /// + /// "failed with status -34018" told neither the model nor the host + /// developer anything. -34018 in particular — `errSecMissingEntitlement` — + /// is a host configuration problem (no keychain access group; routine in an + /// unsigned test bundle), and the model should know a retry cannot fix it. + static func failure(_ operation: String, status: OSStatus) -> BridgeError { + let description = (SecCopyErrorMessageString(status, nil) as String?) ?? "OSStatus \(status)" + if status == errSecMissingEntitlement { + return .nativeFailure( + "\(operation) failed: the host app has no keychain entitlement (\(status)). This is a host configuration issue — the app needs a keychain access group — and retrying will not help." + ) + } + return .nativeFailure("\(operation) failed: \(description) (\(status))") + } + public func read(arguments: [String: JSONValue]) throws -> JSONValue { guard let key = arguments.string("key"), key.isEmpty == false else { throw BridgeError.invalidArguments("keychain.read requires 'key'") @@ -32,7 +48,7 @@ public final class KeychainBridge: @unchecked Sendable { case errSecItemNotFound: return .null default: - throw BridgeError.nativeFailure("keychain.read failed with status \(status)") + throw Self.failure("keychain.read", status: status) } } @@ -62,12 +78,12 @@ public final class KeychainBridge: @unchecked Sendable { if status == errSecDuplicateItem { let updateStatus = SecItemUpdate(baseQuery as CFDictionary, attrs as CFDictionary) guard updateStatus == errSecSuccess else { - throw BridgeError.nativeFailure("keychain.write update failed with status \(updateStatus)") + throw Self.failure("keychain.write", status: updateStatus) } return .object(["key": .string(key), "written": .bool(true)]) } - throw BridgeError.nativeFailure("keychain.write failed with status \(status)") + throw Self.failure("keychain.write", status: status) } public func delete(arguments: [String: JSONValue]) throws -> JSONValue { @@ -86,6 +102,6 @@ public final class KeychainBridge: @unchecked Sendable { return .object(["key": .string(key), "deleted": .bool(true)]) } - throw BridgeError.nativeFailure("keychain.delete failed with status \(status)") + throw Self.failure("keychain.delete", status: status) } } diff --git a/Sources/CodeMode/Bridges/NetworkBridge.swift b/Sources/CodeMode/Bridges/NetworkBridge.swift index ef33703..01234a9 100644 --- a/Sources/CodeMode/Bridges/NetworkBridge.swift +++ b/Sources/CodeMode/Bridges/NetworkBridge.swift @@ -6,7 +6,27 @@ public final class NetworkBridge: @unchecked Sendable { private let session: URLSession private let policy: NetworkAccessPolicy - public init(session: URLSession = .shared, policy: NetworkAccessPolicy = .standard) { + /// The session `network.fetch` uses unless the host supplies its own. + /// + /// Deliberately *not* `URLSession.shared`: that session reads + /// `HTTPCookieStorage.shared` and `URLCredentialStorage.shared`, so every + /// script request would ride whatever the host app is already logged in to, + /// and a `Set-Cookie` in a script-fetched response would poison the app's + /// cookie jar. The sandbox's premise is that script traffic carries no + /// ambient authority, so this session has no cookie jar, no credential + /// store, and no shared cache. + public static let isolatedSession: URLSession = { + let configuration = URLSessionConfiguration.ephemeral + configuration.httpCookieStorage = nil + configuration.urlCredentialStorage = nil + configuration.urlCache = nil + configuration.httpShouldSetCookies = false + configuration.httpCookieAcceptPolicy = .never + configuration.requestCachePolicy = .reloadIgnoringLocalCacheData + return URLSession(configuration: configuration) + }() + + public init(session: URLSession = NetworkBridge.isolatedSession, policy: NetworkAccessPolicy = .standard) { self.session = session self.policy = policy } @@ -43,12 +63,23 @@ public final class NetworkBridge: @unchecked Sendable { var request = URLRequest(url: url) request.httpMethod = options.string("method")?.uppercased() ?? "GET" request.timeoutInterval = TimeInterval(timeoutMs) / 1_000 + // Belt and braces alongside the isolated session: even a host-supplied + // session must not attach its cookie jar to script traffic. + request.httpShouldHandleCookies = false if let headers = options.object("headers") { - for (key, value) in headers { - if let string = value.stringValue { - request.setValue(string, forHTTPHeaderField: key) + for key in headers.keys.sorted() { + guard let string = headers[key]?.stringValue else { + continue } + if let reason = policy.headerViolationReason(for: key) { + context.auditLogger.log(AuditEvent( + capability: CapabilityID.networkFetch.rawValue, + message: "denied \(urlString): \(reason)" + )) + throw BridgeError.networkPolicyViolation(reason) + } + request.setValue(string, forHTTPHeaderField: key) } } diff --git a/Sources/CodeMode/Bridges/NotificationsBridge.swift b/Sources/CodeMode/Bridges/NotificationsBridge.swift index da6fc2a..178f605 100644 --- a/Sources/CodeMode/Bridges/NotificationsBridge.swift +++ b/Sources/CodeMode/Bridges/NotificationsBridge.swift @@ -228,7 +228,7 @@ public final class NotificationsBridge: @unchecked Sendable { private func isoDate(_ text: String?) -> Date? { guard let text else { return nil } - return ISO8601DateFormatter().date(from: text) + return CodeModeDate.parse(text) } #if canImport(UserNotifications) diff --git a/Sources/CodeMode/Bridges/SimpleBuiltInCodeModeProviders.swift b/Sources/CodeMode/Bridges/SimpleBuiltInCodeModeProviders.swift index 194d853..0066ce5 100644 --- a/Sources/CodeMode/Bridges/SimpleBuiltInCodeModeProviders.swift +++ b/Sources/CodeMode/Bridges/SimpleBuiltInCodeModeProviders.swift @@ -39,7 +39,7 @@ private struct KeychainReadTool: BuiltInCodeModeTool { static let codeModeTitle = "Read Keychain value" static let codeModeSummary = "Read a string value from app-scoped Keychain storage." static let codeModeTags = ["security", "token", "keychain"] - static let codeModeExample = "await apple.keychain.get('auth_token')" + static let codeModeExample = "await apple.keychain.get({ key: 'auth_token' })" static let codeModeArguments = [ BuiltInToolArgument("key", .string, hint: "Logical key for this secret value."), ] @@ -67,7 +67,7 @@ private struct KeychainWriteTool: BuiltInCodeModeTool { static let codeModeTitle = "Write Keychain value" static let codeModeSummary = "Store or update a string value in app-scoped Keychain storage." static let codeModeTags = ["security", "token", "keychain"] - static let codeModeExample = "await apple.keychain.set('auth_token', token)" + static let codeModeExample = "await apple.keychain.set({ key: 'auth_token', value: token })" static let codeModeArguments = [ BuiltInToolArgument("key", .string, hint: "Logical key for this secret value."), BuiltInToolArgument("value", .string, optional: true, hint: "Secret string value. Defaults to empty string when omitted."), @@ -102,7 +102,7 @@ private struct KeychainDeleteTool: BuiltInCodeModeTool { static let codeModeTitle = "Delete Keychain value" static let codeModeSummary = "Delete an app-scoped Keychain value." static let codeModeTags = ["security", "token", "keychain"] - static let codeModeExample = "await apple.keychain.delete('auth_token')" + static let codeModeExample = "await apple.keychain.delete({ key: 'auth_token' })" static let codeModeArguments = [ BuiltInToolArgument("key", .string, hint: "Logical key for value removal."), ] diff --git a/Sources/CodeMode/Bridges/SystemServicesCodeModeTools.swift b/Sources/CodeMode/Bridges/SystemServicesCodeModeTools.swift index e5c97d4..0258ef7 100644 --- a/Sources/CodeMode/Bridges/SystemServicesCodeModeTools.swift +++ b/Sources/CodeMode/Bridges/SystemServicesCodeModeTools.swift @@ -289,7 +289,7 @@ struct HealthPermissionRequestTool: BuiltInCodeModeTool { static let codeModeSummary = "Request HealthKit authorization for requested read/write types." static let codeModeTags = ["healthkit", "health", "permission"] static let codeModeExample = "await apple.health.requestPermission({ readTypes: ['stepCount', 'heartRate'], writeTypes: ['stepCount'] })" - static let codeModeResultSummary = "Object with status/granted fields and requested type arrays." + static let codeModeResultSummary = "Object with status ('completed'), per-type writeAuthorization, and the requested type arrays. Read authorization is never disclosed by HealthKit, so an empty health.read may mean denial." struct Arguments: Sendable { @ToolParam("Optional array of type names to read, e.g. stepCount, heartRate, activeEnergyBurned.") diff --git a/Sources/CodeMode/Bridges/SystemUIBridge.swift b/Sources/CodeMode/Bridges/SystemUIBridge.swift index a5b59bb..1d63159 100644 --- a/Sources/CodeMode/Bridges/SystemUIBridge.swift +++ b/Sources/CodeMode/Bridges/SystemUIBridge.swift @@ -1,7 +1,11 @@ import Foundation public final class SystemUIBridge: @unchecked Sendable { - public init() {} + private let networkAccessPolicy: NetworkAccessPolicy + + public init(networkAccessPolicy: NetworkAccessPolicy = .standard) { + self.networkAccessPolicy = networkAccessPolicy + } public func pickCalendar(arguments: [String: JSONValue], context: BridgeInvocationContext) throws -> JSONValue { try validateCalendarPickerArguments(arguments) @@ -188,12 +192,11 @@ public final class SystemUIBridge: @unchecked Sendable { } private func validateCalendarEventArguments(_ arguments: [String: JSONValue]) throws { - let formatter = ISO8601DateFormatter() for key in ["start", "end"] { guard let value = arguments.string(key) else { continue } - guard formatter.date(from: value) != nil else { + guard CodeModeDate.parse(value) != nil else { throw BridgeError.invalidArguments("calendar.ui.presentNewEvent requires \(key) to be an ISO8601 timestamp when provided") } } @@ -447,6 +450,13 @@ public final class SystemUIBridge: @unchecked Sendable { else { throw BridgeError.invalidArguments("\(capability) \(key) must be an absolute HTTP(S) URL") } + + // Destinations opened in a browser were exempt from the policy that + // governs fetch, so a script blocked from reaching an origin could still + // put it in front of the user — or start an OAuth flow against it. + if let reason = networkAccessPolicy.violationReason(for: url) { + throw BridgeError.networkPolicyViolation("\(capability) \(key): \(reason)") + } } private func validateNonemptyString(_ arguments: [String: JSONValue], key: String, capability: String) throws { diff --git a/Sources/CodeMode/Bridges/SystemUICodeModeTools.swift b/Sources/CodeMode/Bridges/SystemUICodeModeTools.swift index b4d47d7..398af2d 100644 --- a/Sources/CodeMode/Bridges/SystemUICodeModeTools.swift +++ b/Sources/CodeMode/Bridges/SystemUICodeModeTools.swift @@ -327,7 +327,7 @@ struct WebUIPresentTool: BuiltInCodeModeTool { @BuiltInCodeMode(.authUIWebAuthenticate, path: "apple.auth.webAuthenticate") struct AuthUIWebAuthenticateTool: BuiltInCodeModeTool { static let codeModeTitle = "Authenticate with system web UI" - static let codeModeSummary = "Start an ASWebAuthenticationSession for OAuth-style browser authentication." + static let codeModeSummary = "Start an ASWebAuthenticationSession for OAuth-style browser authentication. Runs in a private browser session by default and the URL must satisfy the host's network access policy." static let codeModeTags = ["auth", "oauth", "web", "browser", "system-ui"] static let codeModeExample = "await apple.auth.webAuthenticate({ url: 'https://example.com/oauth', callbackURLScheme: 'myapp' })" static let codeModeResultSummary = "Object with action callback/cancelled and callbackURL when available." @@ -337,7 +337,7 @@ struct AuthUIWebAuthenticateTool: BuiltInCodeModeTool { var url: String @ToolParam("Optional custom URL scheme that completes the session.") var callbackURLScheme: String? - @ToolParam("Whether to prefer a private browser session; default false.") + @ToolParam("Whether to use a private browser session; default true. Set false only when the flow must reuse the user's existing browser session.") var prefersEphemeralSession: Bool? @ToolParam("Optional timeout for waiting on callback/cancellation.") var timeoutMs: Double? diff --git a/Sources/CodeMode/Catalog/BridgeCatalog.swift b/Sources/CodeMode/Catalog/BridgeCatalog.swift index 9e55914..8ec69f1 100644 --- a/Sources/CodeMode/Catalog/BridgeCatalog.swift +++ b/Sources/CodeMode/Catalog/BridgeCatalog.swift @@ -1,25 +1,54 @@ import Foundation -struct BridgeCatalog: Sendable { +/// The model-facing view of the registry. +/// +/// Tracks the registry rather than snapshotting it once at init. The snapshot +/// used to be frozen in `CodeModeAgentTools.init`, which made +/// `CapabilityRegistry`'s public post-init `register(...)` methods either +/// unreachable in the supported flow or — if reached — a way to desync what +/// search advertises from what execution can actually invoke. +final class BridgeCatalog: @unchecked Sendable { private struct SearchCatalogPayload: Sendable, Codable { var references: [JavaScriptAPIReference] var byCapability: [String: JavaScriptAPIReference] var byJSName: [String: JavaScriptAPIReference] } - private let references: [JavaScriptAPIReference] - private let referencesByCapability: [CodeModeCapabilityKey: JavaScriptAPIReference] - private let searchCatalog: JSONValue - private let allJavaScriptNames: [String] + private struct Snapshot { + var generation: Int + var references: [JavaScriptAPIReference] + var referencesByCapability: [CodeModeCapabilityKey: JavaScriptAPIReference] + var searchCatalog: JSONValue + var allJavaScriptNames: [String] + } + + private let registry: CapabilityRegistry + private let lock = NSLock() + private var snapshot: Snapshot init(registry: CapabilityRegistry) { + self.registry = registry + self.snapshot = Self.makeSnapshot(registry: registry) + } + + /// Rebuilds if the registry has changed since the last read. Rebuilding is + /// the whole catalog, but it only happens when a provider is actually + /// registered — not on every search. + private func current() -> Snapshot { + lock.lock() + defer { lock.unlock() } + let generation = registry.generation + if snapshot.generation != generation { + snapshot = Self.makeSnapshot(registry: registry) + } + return snapshot + } + + private static func makeSnapshot(registry: CapabilityRegistry) -> Snapshot { + let generation = registry.generation let functions = registry.allRegisteredFunctions().sorted { $0.catalogCapability < $1.catalogCapability } let references = functions.map(Self.reference(from:)) - self.references = references - self.referencesByCapability = Dictionary(uniqueKeysWithValues: references.map { ($0.capabilityKey, $0) }) - self.allJavaScriptNames = Array(Set(references.flatMap(\.jsNames))).sorted() - var byJSName: [String: JavaScriptAPIReference] = [:] for reference in references { for jsName in reference.jsNames { @@ -27,29 +56,41 @@ struct BridgeCatalog: Sendable { } } - self.searchCatalog = Self.jsonValue( - from: SearchCatalogPayload( - references: references, - byCapability: Dictionary(uniqueKeysWithValues: references.map { ($0.capability, $0) }), - byJSName: byJSName - ) + return Snapshot( + generation: generation, + references: references, + referencesByCapability: Dictionary(references.map { ($0.capabilityKey, $0) }, uniquingKeysWith: { first, _ in first }), + searchCatalog: Self.jsonValue( + from: SearchCatalogPayload( + references: references, + byCapability: Dictionary(references.map { ($0.capability, $0) }, uniquingKeysWith: { first, _ in first }), + byJSName: byJSName + ) + ), + allJavaScriptNames: Array(Set(references.flatMap(\.jsNames))).sorted() ) } func reference(for capability: CapabilityID) -> JavaScriptAPIReference? { - referencesByCapability[capability.codeModeKey] + current().referencesByCapability[capability.codeModeKey] } func reference(for capabilityKey: CodeModeCapabilityKey) -> JavaScriptAPIReference? { - referencesByCapability[capabilityKey] + current().referencesByCapability[capabilityKey] } func allReferences() -> [JavaScriptAPIReference] { - references + current().references } func searchCatalogValue() -> JSONValue { - searchCatalog + current().searchCatalog + } + + /// TypeScript declarations for the platform-filtered references that pass + /// `isIncluded`, for a host to drop into its system prompt. + func typeDeclarations(where isIncluded: (JavaScriptAPIReference) -> Bool) -> String { + TypeScriptDeclarations.surface(for: current().references.filter(isIncluded)) } func closestFunctionNames(to candidate: String, limit: Int = 3) -> [String] { @@ -62,7 +103,7 @@ struct BridgeCatalog: Sendable { let distanceThreshold = max(2, min(8, normalizedCandidate.count / 3 + 1)) - let ranked = allJavaScriptNames.map { name in + let ranked = current().allJavaScriptNames.map { name in let normalizedName = name.lowercased() let distance = Self.levenshtein(normalizedCandidate, normalizedName) let prefixBonus = normalizedName.hasPrefix(normalizedCandidate) || normalizedCandidate.hasPrefix(normalizedName) ? -2 : 0 @@ -99,7 +140,7 @@ struct BridgeCatalog: Sendable { } private static func reference(from function: RegisteredCodeModeFunction) -> JavaScriptAPIReference { - return JavaScriptAPIReference( + let reference = JavaScriptAPIReference( capability: function.catalogCapability, capabilityKey: function.capabilityKey, builtInCapability: function.builtInCapability, @@ -114,6 +155,9 @@ struct BridgeCatalog: Sendable { argumentConstraints: function.argumentConstraints, resultSummary: function.resultSummary ) + var withTypes = reference + withTypes.dts = TypeScriptDeclarations.declaration(for: reference) + return withTypes } private static func levenshtein(_ lhs: String, _ rhs: String) -> Int { diff --git a/Sources/CodeMode/Catalog/TypeScriptDeclarations.swift b/Sources/CodeMode/Catalog/TypeScriptDeclarations.swift new file mode 100644 index 0000000..29cfe77 --- /dev/null +++ b/Sources/CodeMode/Catalog/TypeScriptDeclarations.swift @@ -0,0 +1,368 @@ +import Foundation + +/// Generates TypeScript declarations for the CodeMode JavaScript surface. +/// +/// The registry already holds names, required/optional arguments, types, enum +/// constraints, and hints; before this, the model saw that as a flat +/// `argumentTypes` map of six coarse cases and a prose `resultSummary`. Models +/// write markedly better code against real declarations, so the same metadata is +/// emitted as `.d.ts` — per capability for search results, and whole-surface for +/// a system prompt. +/// +/// Result types are `CodeModeValue` for now: built-in tools return untyped +/// `JSONValue` dictionaries, so a typed result would be a fiction. The prose +/// `resultSummary` is carried in the doc comment instead, and a per-capability +/// result schema can tighten this later without changing the shape here. +public enum TypeScriptDeclarations { + /// Shared types and the runtime globals that are not catalog capabilities. + /// + /// Hand-authored deliberately: these are the Node-compatibility shims in + /// `RuntimeJavaScript.bootstrap`, and unlike the namespaced helpers they take + /// *positional* arguments, which the catalog has no way to express. Keeping + /// them here means the declarations tell the truth about both conventions. + public static let preamble = """ + /** Any JSON-representable value. Bridge results are JSON, not class instances. */ + type CodeModeValue = + | null + | boolean + | number + | string + | CodeModeValue[] + | { [key: string]: CodeModeValue }; + + declare const console: { + log(...values: unknown[]): void; + info(...values: unknown[]): void; + warn(...values: unknown[]): void; + error(...values: unknown[]): void; + }; + + interface CodeModeResponse { + ok: boolean; + status: number; + statusText: string; + headers: { get(name: string): string | null }; + text(): Promise; + json(): Promise; + } + + /** + * HTTP(S) fetch. Runs on an isolated session: no app cookies, no stored + * credentials. Cookie/Authorization/Proxy-* request headers are refused + * unless the host app permits them. + */ + declare function fetch( + url: string, + options?: { + method?: string; + headers?: { [name: string]: string }; + body?: string; + bodyBase64?: string; + timeoutMs?: number; + responseEncoding?: "text" | "base64"; + } + ): Promise; + + /** + * Node-style filesystem aliases. Unlike the apple.* helpers these take + * POSITIONAL arguments and return Node-like values. + */ + declare const fs: { + promises: { + readFile(path: string, encoding?: "utf8" | "base64"): Promise; + writeFile(path: string, data: string, encoding?: "utf8" | "base64"): Promise; + readdir(path: string): Promise; + stat(path: string): Promise; + access(path: string): Promise; + mkdir(path: string, options?: { recursive?: boolean }): Promise; + rm(path: string, options?: { recursive?: boolean }): Promise; + rename(from: string, to: string, options?: { overwrite?: boolean; recursive?: boolean }): Promise; + copyFile(from: string, to: string, options?: { overwrite?: boolean; recursive?: boolean }): Promise; + }; + }; + + declare const path: { + join(...parts: string[]): string; + }; + """ + + /// The declaration for one capability, suitable for a search result payload. + /// + /// Declared at its real call path — `declare const apple: { fs: { read(…) } }` + /// — so what the model reads is exactly what it can call. + /// + /// A fragment type-checks alongside `preamble`, which defines `CodeModeValue`; + /// on its own that name is unresolved. Fragments also do not merge with each + /// other — two for the same root each declare it — which is harmless when a + /// model reads joined search results, but for one compilation unit covering + /// several capabilities use `surface(for:)` (or + /// `CodeModeAgentTools.typeDeclarations(for:)`), which merges them under a + /// single declaration per root. Verified with `tsc --strict`: every fragment + /// plus the preamble, and every platform's whole surface, compile cleanly. + public static func declaration(for reference: JavaScriptAPIReference) -> String { + guard let canonical = canonicalName(for: reference) else { + return "" + } + // A bare global such as `fetch` has no namespace to sit in. The whole + // surface covers it from the hand-authored preamble; a per-reference + // declaration has to say it directly. + guard canonical.contains(".") else { + return "\(documentation(for: reference, canonicalName: canonical))\ndeclare function \(canonical)\(functionSignature(for: reference));" + } + return renderNamespaces(for: [reference]) + } + + /// The preamble plus the given capabilities, grouped by canonical dotted + /// JavaScript name under one declaration per root — valid as a single + /// compilation unit. + public static func surface(for references: [JavaScriptAPIReference]) -> String { + """ + // CodeMode JavaScript API — generated from the capability registry. + // Only helpers supported on the current host platform appear here. + + \(preamble) + + \(renderNamespaces(for: references)) + """ + } + + private static func renderNamespaces(for references: [JavaScriptAPIReference]) -> String { + let root = Namespace(name: "") + for reference in references.sorted(by: { $0.capability < $1.capability }) { + guard let canonical = canonicalName(for: reference) else { + continue + } + var components = canonical.split(separator: ".").map(String.init) + guard components.count >= 2 else { + // A bare global such as `fetch` is covered by the preamble. + continue + } + let functionName = components.removeLast() + root.insert( + Member( + name: functionName, + documentation: documentation(for: reference, canonicalName: canonical), + signature: functionSignature(for: reference) + ), + at: components + ) + } + + return root.children + .sorted { $0.key < $1.key } + .map { render(namespace: $0.value, indent: "", isTopLevel: true) } + .joined(separator: "\n\n") + } + + // MARK: - Declaration pieces + + /// The namespaced helper name, preferred over Node-style aliases: aliases use + /// positional arguments the catalog cannot describe, and are documented in + /// the preamble instead. + private static func canonicalName(for reference: JavaScriptAPIReference) -> String? { + reference.jsNames.first { $0.contains(".") && $0.hasPrefix("fs.promises.") == false } + ?? reference.jsNames.first + } + + private static func documentation(for reference: JavaScriptAPIReference, canonicalName: String) -> String { + var lines = ["/**"] + lines.append(" * \(reference.summary)") + lines.append(" *") + lines.append(" * Capability: \(reference.capability)") + lines.append(" * Returns: \(reference.resultSummary)") + + let aliases = reference.jsNames.filter { $0 != canonicalName } + if aliases.isEmpty == false { + // Only the fs.promises.* shims are positional; other aliases are + // additional namespaced spellings of the same object-argument call. + let positional = aliases.filter { $0.hasPrefix("fs.promises.") || $0 == "fetch" } + let note = positional.isEmpty ? "" : " — \(positional.joined(separator: ", ")) take positional arguments" + lines.append(" * Aliases: \(aliases.joined(separator: ", "))\(note)") + } + if reference.example.isEmpty == false { + lines.append(" *") + lines.append(" * @example \(reference.example)") + } + lines.append(" */") + return lines.joined(separator: "\n") + } + + private static func functionSignature(for reference: JavaScriptAPIReference) -> String { + let tree = ArgumentTree(reference: reference) + guard tree.isEmpty == false else { + return "(): Promise" + } + // `args` is optional only when nothing in it is required. + let optionalMarker = reference.requiredArguments.isEmpty ? "?" : "" + return "(args\(optionalMarker): \(tree.render(indent: ""))): Promise" + } + + // MARK: - Argument tree + // + // Arguments are declared as dotted paths (`options.timeoutMs`), so the object + // literal has to be rebuilt from them. + + private struct ArgumentTree { + private var children: [String: ArgumentTree] = [:] + private var leaves: [Leaf] = [] + + private struct Leaf { + var name: String + var type: String + var optional: Bool + var hint: String? + } + + init(reference: JavaScriptAPIReference) { + let allPaths = Array(Set( + reference.requiredArguments + reference.optionalArguments + Array(reference.argumentTypes.keys) + )).sorted() + let required = Set(reference.requiredArguments) + + for path in allPaths { + let components = path.split(separator: ".").map(String.init) + insert( + components: components, + leaf: Leaf( + name: components.last ?? path, + type: TypeScriptDeclarations.tsType( + for: reference.argumentTypes[path] ?? .any, + allowedValues: reference.argumentConstraints.allowedStringValues[path] + ), + // A nested path is optional unless the exact path is + // listed as required. + optional: required.contains(path) == false, + hint: reference.argumentHints[path] + ) + ) + } + } + + private init() {} + + var isEmpty: Bool { + children.isEmpty && leaves.isEmpty + } + + private mutating func insert(components: [String], leaf: Leaf) { + guard components.count > 1 else { + // A declared parent (`options: object`) whose children also appear + // is represented by the nested literal, not by a duplicate leaf. + if children[leaf.name] == nil, leaves.contains(where: { $0.name == leaf.name }) == false { + leaves.append(leaf) + } + return + } + let head = components[0] + leaves.removeAll { $0.name == head } + var child = children[head] ?? ArgumentTree() + child.insert(components: Array(components.dropFirst()), leaf: leaf) + children[head] = child + } + + func render(indent: String) -> String { + let inner = indent + " " + var lines: [String] = ["{"] + + for leaf in leaves.sorted(by: { $0.name < $1.name }) { + if let hint = leaf.hint, hint.isEmpty == false { + lines.append("\(inner)/** \(hint) */") + } + lines.append("\(inner)\(leaf.name)\(leaf.optional ? "?" : ""): \(leaf.type);") + } + + for (name, child) in children.sorted(by: { $0.key < $1.key }) { + lines.append("\(inner)\(name)?: \(child.render(indent: inner));") + } + + lines.append("\(indent)}") + return lines.joined(separator: "\n") + } + } + + private static func tsType(for type: CapabilityArgumentType, allowedValues: [String]?) -> String { + if let allowedValues, allowedValues.isEmpty == false { + // The constraint is the type: a union beats `string` plus prose the + // model has to notice. + return allowedValues.map { "\"\($0)\"" }.joined(separator: " | ") + } + + switch type { + case .string: + return "string" + case .number: + return "number" + case .bool: + return "boolean" + case .object: + return "{ [key: string]: CodeModeValue }" + case .array: + return "CodeModeValue[]" + case .any: + return "CodeModeValue" + } + } + + // MARK: - Namespace rendering + + private struct Member { + var name: String + var documentation: String + var signature: String + } + + private final class Namespace { + let name: String + var children: [String: Namespace] = [:] + var members: [Member] = [] + + init(name: String) { + self.name = name + } + + func insert(_ member: Member, at path: [String]) { + guard let head = path.first else { + members.append(member) + return + } + let child = children[head] ?? Namespace(name: head) + children[head] = child + child.insert(member, at: Array(path.dropFirst())) + } + } + + /// Renders as an object type — `declare const apple: { keychain: { + /// delete(args): …; }; };` — not as `declare namespace`. + /// + /// Namespace members are declarations and need identifiers, so helpers named + /// with reserved words (`apple.keychain.delete`, `apple.photos.export`) + /// rendered as `function delete(…)`: not valid TypeScript, which broke any + /// host that type-checks the output. Object type members may be any property + /// name. + private static func render(namespace: Namespace, indent: String, isTopLevel: Bool) -> String { + let inner = indent + " " + var lines = [isTopLevel ? "\(indent)declare const \(namespace.name): {" : "\(indent)\(namespace.name): {"] + + for member in namespace.members.sorted(by: { $0.name < $1.name }) { + lines.append(indented(member.documentation, by: inner)) + lines.append("\(inner)\(member.name)\(indented(member.signature, by: inner, skipFirstLine: true));") + } + + for (_, child) in namespace.children.sorted(by: { $0.key < $1.key }) { + lines.append(render(namespace: child, indent: inner, isTopLevel: false)) + } + + lines.append("\(indent)};") + return lines.joined(separator: "\n") + } + + private static func indented(_ text: String, by indent: String, skipFirstLine: Bool = false) -> String { + text + .split(separator: "\n", omittingEmptySubsequences: false) + .enumerated() + .map { index, line in + index == 0 && skipFirstLine ? String(line) : indent + line + } + .joined(separator: "\n") + } +} diff --git a/Sources/CodeMode/Host/CodeModeAgentGuidance.swift b/Sources/CodeMode/Host/CodeModeAgentGuidance.swift new file mode 100644 index 0000000..129cd70 --- /dev/null +++ b/Sources/CodeMode/Host/CodeModeAgentGuidance.swift @@ -0,0 +1,188 @@ +import Foundation + +/// System-prompt guidance that teaches an agent *why* it is writing a program +/// rather than calling tools one at a time. +/// +/// The tool descriptions and the generated TypeScript answer "what is this helper +/// called and what does it take". Neither answers "what shape should my work +/// take", and an agent that never learns the second question will use +/// `executeJavaScript` as a thin RPC — one call per operation — which is the +/// pattern this package exists to replace. +/// +/// Pair it with `CodeModeAgentTools.typeDeclarations()`, which supplies the +/// vocabulary this supplies the strategy for. Those declarations follow the +/// host's `CapabilityGrant`, so a host with a ceiling pays only for what it +/// grants — the unrestricted iOS surface is roughly 21,000 tokens: +/// +/// ```swift +/// let prompt = CodeModeAgentGuidance.systemPrompt(.standard) +/// + "\n\n" +/// + tools.typeDeclarations() +/// ``` +public enum CodeModeAgentGuidance { + /// How much prompt budget to spend on the guidance. + /// + /// Every tier is self-contained and states the core idea; the larger ones add + /// worked examples and failure modes rather than qualifying what came before, + /// so a host can drop down a tier without losing a rule. + public enum Length: String, Sendable, CaseIterable { + // Declared smallest to largest; `length(forApproximateTokenBudget:)` + // relies on that order. + /// One paragraph: the core idea and nothing else. + case brief + /// The idea, the workflow, and one worked example. The default. + case standard + /// Adds fan-out and error-handling examples plus the anti-pattern list. + /// Worth it when the agent will do open-ended multi-step work. + case full + + /// Rough size of this tier's text, for budgeting against a prompt. + /// + /// Estimated at four characters per token — the usual English-prose rule + /// of thumb, not a real tokenization. Treat it as a bound to plan with, + /// not a number to assert on. + public var approximateTokenCount: Int { + CodeModeAgentGuidance.systemPrompt(self).count / 4 + } + } + + /// The guidance text at the requested length. + public static func systemPrompt(_ length: Length = .standard) -> String { + switch length { + case .brief: + return core + case .standard: + return [core, workflow, workedExample].joined(separator: "\n\n") + case .full: + return [core, workflow, workedExample, fanOutExample, antiPatterns].joined(separator: "\n\n") + } + } + + /// The largest tier that fits the budget, or `.brief` when nothing does. + /// + /// `.brief` is returned even when it exceeds the budget: some guidance is the + /// difference between an agent that batches and one that does not, so the + /// floor is deliberate rather than an empty string. + public static func systemPrompt(approximateTokenBudget: Int) -> String { + systemPrompt(length(forApproximateTokenBudget: approximateTokenBudget)) + } + + /// The tier `systemPrompt(approximateTokenBudget:)` would choose. + public static func length(forApproximateTokenBudget budget: Int) -> Length { + Length.allCases + .reversed() + .first { $0.approximateTokenCount <= budget } + ?? .brief + } + + // MARK: - Sections + + private static let core = """ + ## Writing code instead of calling tools + + You have a JavaScript runtime with direct access to this device's APIs. Use it \ + to run whole tasks, not single operations. One `executeJavaScript` call should \ + do the entire job — read, loop, branch, filter, transform, write — and return \ + only the finished answer. Ten helper calls inside one script is normal and \ + fast; ten `executeJavaScript` calls that each make one helper call is the \ + mistake this runtime exists to avoid, because every round trip costs a model \ + turn and pushes intermediate data you do not need through your context. \ + Request only the capabilities your script actually calls. + """ + + private static let workflow = """ + ## How to work + + 1. Call `searchJavaScriptAPI` first when you are unsure of helper names, \ + arguments, or result shapes. Return `ref.dts` — the TypeScript declaration — \ + rather than assembling the metadata fields yourself. + 2. Write one script that completes the task. Reach for ordinary control flow: \ + loops over collections, `if` for branching, `try`/`catch` around anything that \ + may fail per item. Before writing a loop of single calls, check the declaration \ + for an argument that takes a collection — `apple.contacts.list({ identifiers })` \ + is one call, not one per contact. + 3. Do the filtering and aggregation *in the script*. Return the graded answer — \ + a count, a summary, the three matching records — not the raw collection you \ + read. A large return value is truncated, and the data you did not need cost \ + you context on the way through. + 4. Repair from the structured error. It names the capability, the line and \ + column, and what to change; the suggestions tell you when a retry cannot help. + """ + + private static let workedExample = """ + ## The shape to aim for + + Task: "how much did I spend on the trip, and file the receipts?" + + ```javascript + // ONE call. Reads, filters, sums, writes, and returns just the total. + const entries = await apple.fs.list({ path: 'documents:receipts' }); + let total = 0; + const filed = []; + + for (const entry of entries) { + if (entry.isDirectory || !entry.name.endsWith('.json')) continue; + const { text } = await apple.fs.read({ path: entry.path }); + const receipt = JSON.parse(text); + if (receipt.trip !== 'lisbon') continue; + total += receipt.amount; + await apple.fs.move({ + from: entry.path, + to: `documents:receipts/lisbon/${entry.name}`, + overwrite: true + }); + filed.push(entry.name); + } + + return { total, filedCount: filed.length }; + ``` + + `allowedCapabilities: ["fs.list", "fs.read", "fs.move"]` — exactly the three \ + helpers the script calls, and nothing else. + """ + + private static let fanOutExample = """ + ## Handling many items and partial failure + + Keep going when one item fails, and report what happened rather than throwing \ + the whole run away: + + ```javascript + const results = []; + for (const city of ['lisbon', 'porto', 'faro']) { + try { + const response = await fetch(`https://api.example.com/weather/${city}`); + if (!response.ok) { results.push({ city, error: response.status }); continue; } + const data = await response.json(); + results.push({ city, tempC: data.current.temp_c }); + } catch (error) { + results.push({ city, error: String(error) }); + } + } + return results; + ``` + + Native calls run one at a time, so `Promise.all` does not make these overlap — \ + a loop is just as fast here and reads better. Use \ + `await new Promise(r => setTimeout(r, 1000))` to back off between retries; \ + delays are real. + """ + + private static let antiPatterns = """ + ## Anti-patterns + + - **One call per operation.** If your second `executeJavaScript` starts from \ + data the first one returned, both should have been one script. + - **Returning the raw collection.** Reading 500 records to answer "how many \ + are overdue" should return a number, not 500 records. + - **Asking for capabilities you do not call.** Request the exact set your \ + script uses. A wider list grants nothing useful, and it is the wrong thing to \ + ask a user to approve. + - **Forgetting `await`.** An un-awaited helper's failure is silently \ + discarded and you will be told the run succeeded. A \ + `BRIDGE_FAILURES_NOT_SURFACED` diagnostic means exactly this happened. + - **Re-running a call the error told you not to retry.** \ + `NETWORK_POLICY_VIOLATION`, `UI_PRESENTER_UNAVAILABLE`, and a host-withheld \ + `CAPABILITY_DENIED` will not succeed on a second attempt. + """ +} diff --git a/Sources/CodeMode/Host/CodeModeAgentToolDescriptions.swift b/Sources/CodeMode/Host/CodeModeAgentToolDescriptions.swift index 62cace0..84e53b3 100644 --- a/Sources/CodeMode/Host/CodeModeAgentToolDescriptions.swift +++ b/Sources/CodeMode/Host/CodeModeAgentToolDescriptions.swift @@ -42,7 +42,7 @@ public enum CodeModeAgentToolDescriptions { "items": .object([ "type": .string("string"), ]), - "description": .string("Custom provider capability keys required by the JavaScript. Built-in capability IDs may also be accepted here by hosts that expose one allowlist field."), + "description": .string("Custom provider capability keys required by the JavaScript, for example myapp.tasks.complete. Built-in capability IDs do not belong here and are ignored — list those in allowedCapabilities."), ]), "timeoutMs": .object([ "type": .string("integer"), @@ -75,6 +75,7 @@ public enum CodeModeAgentToolDescriptions { argumentHints: Record; argumentConstraints: { allowedStringValues: Record }; resultSummary: string; + dts: string; } declare const api: { @@ -86,6 +87,8 @@ public enum CodeModeAgentToolDescriptions { Your code must evaluate to an async function and return JSON-serializable output. Search has a 2s budget, so slice broad result sets before returning them. + Prefer returning ref.dts: it is the TypeScript declaration for the helper, with argument names, types, string-literal unions for constrained values, optionality, and the result summary. It is more precise and usually shorter than assembling argumentTypes, argumentHints, and argumentConstraints yourself. + Examples: async () => { return api.references @@ -106,7 +109,14 @@ public enum CodeModeAgentToolDescriptions { } async () => { - return api.byJSName["apple.fs.read"]; + return api.byJSName["apple.fs.read"].dts; + } + + async () => { + return api.references + .filter(ref => ref.tags.includes("calendar")) + .map(ref => ref.dts) + .join("\n\n"); } """ ) @@ -114,21 +124,29 @@ public enum CodeModeAgentToolDescriptions { public static let executeJavaScript = CodeModeAgentToolDescription( name: "executeJavaScript", description: """ - Execute JavaScript against the CodeMode runtime. Prefer searchJavaScriptAPI first when choosing helpers or arguments. Cross-platform helpers live under apple.* and platform-specific helpers live under platform namespaces such as ios.alarm.*; custom host providers may expose additional namespaces. System UI helpers such as apple.ui.presentAlert, apple.calendar.presentNewEvent, apple.photos.pick, apple.contacts.pick, apple.documents.pick, apple.share.present, apple.quicklook.preview, apple.web.present, and apple.auth.webAuthenticate require a host-provided SystemUIPresenter; camera, document scan, mail, and message compose helpers are iOS-only. Only helpers supported on the current host platform are installed. + Execute JavaScript against the CodeMode runtime. Use one call to run a whole task — read, loop, branch, filter, transform, write — and return only the finished answer. Many helper calls inside one script is the intended shape; many executeJavaScript calls that each make one helper call is not, because every round trip costs a turn and pushes intermediate data through your context. Prefer searchJavaScriptAPI first when choosing helpers or arguments. Cross-platform helpers live under apple.* and platform-specific helpers live under platform namespaces such as ios.alarm.*; custom host providers may expose additional namespaces. System UI helpers such as apple.ui.presentAlert, apple.calendar.presentNewEvent, apple.photos.pick, apple.contacts.pick, apple.documents.pick, apple.share.present, apple.quicklook.preview, apple.web.present, and apple.auth.webAuthenticate require a host-provided SystemUIPresenter; camera, document scan, mail, and message compose helpers are iOS-only. Only helpers supported on the current host platform are installed. + + Calling conventions: apple.* and other namespaced helpers take one object argument, for example apple.fs.read({ path: "tmp:file.txt" }) and often return structured objects such as { text, base64 }. Node-style filesystem aliases under fs.promises.* use positional arguments, for example fs.promises.readFile("tmp:file.txt", "utf8"), and return Node-like values such as a string for readFile. When in doubt, use the ref.dts from searchJavaScriptAPI — it states the exact signature. fetch(url, options) is a global helper and returns a Response-like object with text(), json(), headers.get(), status, and ok. - Calling conventions: apple.* helpers take one object argument, for example apple.fs.read({ path: "tmp:file.txt" }) and often return structured objects such as { text, base64 }. Node-style filesystem aliases use positional arguments, for example fs.promises.readFile("tmp:file.txt", "utf8"), and return Node-like values such as a string for readFile. fetch(url, options) is a global helper and returns a Response-like object with text(), json(), headers.get(), status, and ok. + Return semantics: the runtime wraps your code in an async function. For multi-statement code, return the final graded value with an explicit top-level return statement. A script that is only a bare final top-level await expression, such as await apple.fs.read({ path: "tmp:file.txt" }), returns that awaited value. Do not use an unreturned async IIFE as the final expression. - Return semantics: the runtime wraps your code in an async function. For multi-statement code, return the final graded value with an explicit top-level return statement. A script that is only a bare final top-level await expression, such as await apple.fs.read({ path: "tmp:file.txt" }), returns that awaited value. Do not use an unreturned async IIFE as the final expression. setTimeout callbacks fire synchronously in this runtime. + Concurrency: setTimeout and clearTimeout work normally — callbacks are deferred and delays are honoured, so `await new Promise(r => setTimeout(r, 1000))` really waits. There is no I/O event loop, though: native calls run one at a time, so Promise.all over several helpers completes them sequentially rather than concurrently. A promise with no resolve path and no pending timer can never settle and fails fast with JS_RUNTIME_ERROR rather than running out the clock. - Allowlisting: include only the required built-in capabilities in allowedCapabilities and custom provider keys in allowedCapabilityKeys. Execution defaults to a 10000ms timeout and returns structured CodeModeToolError failures for syntax errors, missing JS helpers, runtime throws, validation failures, permission denials, timeouts, cancellation, and internal errors. + Allowlisting: request the exact set of capabilities your script calls and nothing more — a capability you list but never call widens the grant for nothing. Built-in capability IDs go in allowedCapabilities; custom provider keys go in allowedCapabilityKeys. The two fields are not interchangeable: a built-in capability listed in allowedCapabilityKeys is ignored. These fields declare what your script needs; the host app applies its own ceiling on top, so a capability you list may still be withheld. Execution defaults to a 10000ms timeout and returns structured CodeModeToolError failures for syntax errors, missing JS helpers, runtime throws, validation failures, permission denials, timeouts, cancellation, and internal errors. Error repair guide: JS_API_NOT_FOUND: use the suggested JS helper names or searchJavaScriptAPI. - CAPABILITY_DENIED: add the exact capability to allowedCapabilities or allowedCapabilityKeys and retry. + CAPABILITY_DENIED: add the exact capability to allowedCapabilities (built-ins) or allowedCapabilityKeys (custom provider keys) and retry — unless the error suggestions say the host withheld it, in which case retrying will not help. PERMISSION_DENIED: the capability is allowlisted but the OS, host, or custom provider denied permission; request permission if a helper exists, otherwise tell the user or host. UI_PRESENTER_UNAVAILABLE: host configuration issue; do not retry the same call. INVALID_ARGUMENTS: use the catalog requiredArguments, optionalArguments, argumentHints, and example. NETWORK_POLICY_VIOLATION: the host's network access policy refused the destination; do not retry the same URL and do not add capabilities to work around it. + JS_RUNTIME_ERROR "can never settle": a promise in your script has no resolve path; await every promise and make sure each one resolves. + + Warning diagnostics worth acting on: + BRIDGE_FAILURES_NOT_SURFACED: a helper failed but your script still returned successfully — usually a missing await, which discards the rejection. + RESULT_TRUNCATED: the return value exceeded the size limit; return a summary or write the payload to the sandbox and return the path. + TIMER_CALLBACK_ERROR: a setTimeout callback threw; its error cannot reach the caller, so handle it inside the callback. """ ) diff --git a/Sources/CodeMode/Host/CodeModeAgentTools.swift b/Sources/CodeMode/Host/CodeModeAgentTools.swift index 1959dfb..4c6356f 100644 --- a/Sources/CodeMode/Host/CodeModeAgentTools.swift +++ b/Sources/CodeMode/Host/CodeModeAgentTools.swift @@ -4,10 +4,18 @@ public final class CodeModeAgentTools: @unchecked Sendable { private let registry: CapabilityRegistry private let catalog: BridgeCatalog private let runtime: BridgeRuntime + private let capabilityGrant: CapabilityGrant - public init(config: CodeModeConfiguration = .init()) { + public convenience init(config: CodeModeConfiguration = .init()) { + self.init(config: config, clock: RealClock()) + } + + /// Test seam: a virtual clock makes timer-driven executions instant and + /// exact. Not public — production time is not a host decision. + init(config: CodeModeConfiguration, clock: any RuntimeClock) { let allDefaultRegistrations = DefaultCapabilityLoader.loadAllRegistrations( fileSystem: config.fileSystem, + fileSystemLimits: config.fileSystemLimits, networkAccessPolicy: config.networkAccessPolicy, eventInbox: config.eventInbox, cloudKitClient: config.cloudKitClient, @@ -32,12 +40,14 @@ public final class CodeModeAgentTools: @unchecked Sendable { let providerRegistrations = config.codeModeProviders.flatMap { $0.codeModeRegistrations() } let registry = CapabilityRegistry(registrations: registrations, codeModeRegistrations: providerRegistrations) self.registry = registry + self.capabilityGrant = config.capabilityGrant self.catalog = BridgeCatalog(registry: registry) self.runtime = BridgeRuntime( registry: registry, catalog: self.catalog, config: config, - unsupportedBuiltInJavaScriptNames: unsupportedBuiltInJavaScriptNames + unsupportedBuiltInJavaScriptNames: unsupportedBuiltInJavaScriptNames, + clock: clock ) } @@ -45,7 +55,65 @@ public final class CodeModeAgentTools: @unchecked Sendable { try await runtime.searchAsync(request) } + /// Declared `async throws` for a body that is currently synchronous and + /// non-throwing. That is deliberate: making the bridge ABI asynchronous is + /// planned, and this signature is the one that will not have to change. public func executeJavaScript(_ request: JavaScriptExecutionRequest) async throws -> JavaScriptExecutionCall { runtime.makeExecutionCall(request) } + + /// TypeScript declarations for what this host actually permits: the + /// platform-filtered surface, narrowed to the configured `CapabilityGrant`. + /// + /// Models write markedly better code against real types than against prose, + /// but the unrestricted iOS surface is roughly 21,000 tokens. A host with a + /// grant pays only for what it grants — and never advertises a helper the + /// model would be denied. Searches return a per-capability `dts` for the + /// cases in between. + public func typeDeclarations() -> String { + catalog.typeDeclarations { reference in + if let builtIn = reference.builtInCapability { + return capabilityGrant.permits(builtIn) + } + return capabilityGrant.permits(key: reference.capabilityKey) + } + } + + /// Declarations for an explicit subset — for example, the capabilities a + /// particular agent task needs, when even the grant is too broad to inline. + /// Identifiers not available on this host are skipped. + public func typeDeclarations( + for capabilities: Set, + capabilityKeys: Set = [] + ) -> String { + catalog.typeDeclarations { reference in + if let builtIn = reference.builtInCapability { + return capabilities.contains(builtIn) + } + return capabilityKeys.contains(reference.capabilityKey) + } + } + + /// Every capability available on this host, for consent UI and for hosts + /// building their own `CapabilityGrant`. + public func capabilities() -> [JavaScriptAPIReference] { + catalog.allReferences() + } + + /// Adds host providers after construction. + /// + /// Providers no longer have to be known at init: the catalog tracks the + /// registry, so search starts advertising these immediately and the JS + /// bindings are installed for the next execution. A host whose domain APIs + /// depend on runtime state — a signed-in account, a loaded document — can + /// register them when that state arrives. + /// + /// Registering a key that already exists replaces it. + public func register(providers: [any CodeModeProvider]) { + registry.register(providers.flatMap { $0.codeModeRegistrations() }) + } + + public func register(provider: any CodeModeProvider) { + register(providers: [provider]) + } } diff --git a/Sources/CodeMode/Host/HostConfigurationValidator.swift b/Sources/CodeMode/Host/HostConfigurationValidator.swift index d9eb7f6..5796cb0 100644 --- a/Sources/CodeMode/Host/HostConfigurationValidator.swift +++ b/Sources/CodeMode/Host/HostConfigurationValidator.swift @@ -119,6 +119,26 @@ public enum HostConfigurationValidator { validate(requiredCapabilities: requiredCapabilities, infoPlist: bundle.infoDictionary ?? [:]) } + /// Checks the runtime configuration itself, independent of Info.plist. + /// + /// The one thing worth flagging today is an unrestricted `capabilityGrant`. + /// It is the default for source compatibility, but it means the *model* + /// decides what a script may reach — the allowlist in the tool call is + /// model-authored. A host that pipes tool JSON straight into + /// `executeJavaScript` with this default has no capability boundary at all. + public static func validate(configuration: CodeModeConfiguration) -> [HostConfigurationIssue] { + guard configuration.capabilityGrant == .unrestricted else { + return [] + } + return [ + HostConfigurationIssue( + severity: .warning, + key: "capabilityGrant", + message: "CodeModeConfiguration.capabilityGrant is .unrestricted: the model-authored allowedCapabilities is the only allowlist in force. Set CapabilityGrant.only(...) to the capabilities this host actually permits." + ) + ] + } + public static func validate(requiredCapabilities: Set, infoPlist: [String: Any]) -> [HostConfigurationIssue] { var issues: [HostConfigurationIssue] = [] let keys = requiredInfoPlistKeys(for: requiredCapabilities) diff --git a/Sources/CodeMode/Registry/CapabilityRegistry.swift b/Sources/CodeMode/Registry/CapabilityRegistry.swift index b50f76d..e5bb049 100644 --- a/Sources/CodeMode/Registry/CapabilityRegistry.swift +++ b/Sources/CodeMode/Registry/CapabilityRegistry.swift @@ -16,7 +16,10 @@ public enum CapabilityArgumentType: String, Sendable, Codable, Equatable { if case .string = value { return true } return false case .number: - if case .number = value { return true } + // Non-finite is not a usable number for any bridge: it cannot be + // converted to an Int, cannot round-trip through JSON, and reaches + // here from `"inf"`/`"nan"` coercion or a JS `Infinity`. + if case let .number(number) = value { return number.isFinite } return false case .bool: if case .bool = value { return true } @@ -432,6 +435,11 @@ public final class CapabilityRegistry: @unchecked Sendable { private let lock = NSLock() private var registrations: [CapabilityID: CapabilityRegistration] = [:] private var codeModeRegistrations: [CodeModeCapabilityKey: CodeModeRegistration] = [:] + /// Bumped on every mutation so `BridgeCatalog` can tell whether its snapshot + /// is still current. Without it the catalog was frozen at init, which made + /// the public `register(...)` methods below either unreachable or a way to + /// desync what search advertises from what execution can actually invoke. + private var generationValue = 0 public init(registrations: [CapabilityRegistration] = [], codeModeRegistrations: [CodeModeRegistration] = []) { for registration in registrations { @@ -442,9 +450,17 @@ public final class CapabilityRegistry: @unchecked Sendable { } } + /// Monotonic version of the registry's contents. + public var generation: Int { + lock.lock() + defer { lock.unlock() } + return generationValue + } + public func register(_ registration: CapabilityRegistration) { lock.lock() registrations[registration.descriptor.id] = registration + generationValue += 1 lock.unlock() } @@ -453,12 +469,14 @@ public final class CapabilityRegistry: @unchecked Sendable { for registration in registrations { self.registrations[registration.descriptor.id] = registration } + generationValue += 1 lock.unlock() } public func register(_ registration: CodeModeRegistration) { lock.lock() codeModeRegistrations[registration.capabilityKey] = registration + generationValue += 1 lock.unlock() } @@ -467,6 +485,7 @@ public final class CapabilityRegistry: @unchecked Sendable { for registration in registrations { self.codeModeRegistrations[registration.capabilityKey] = registration } + generationValue += 1 lock.unlock() } @@ -506,14 +525,13 @@ public final class CapabilityRegistry: @unchecked Sendable { return registrations[capability].map(RegisteredCodeModeFunction.init) } + /// Resolves a *custom provider* key only. Built-ins are deliberately not + /// reachable here: they are dispatched by `CapabilityID` and gated by + /// `allowedCapabilities`, so resolving them through the loosely-typed key + /// namespace would let `allowedCapabilityKeys` alias a built-in. func registeredFunction(for capabilityKey: CodeModeCapabilityKey) -> RegisteredCodeModeFunction? { lock.lock() defer { lock.unlock() } - if let builtIn = CapabilityID(rawValue: capabilityKey.rawValue), - let registration = registrations[builtIn] - { - return RegisteredCodeModeFunction(registration) - } return codeModeRegistrations[capabilityKey].map(RegisteredCodeModeFunction.init) } @@ -535,7 +553,11 @@ public final class CapabilityRegistry: @unchecked Sendable { } private func invokeBuiltIn(_ capability: CapabilityID, arguments: [String: JSONValue], context: BridgeInvocationContext) throws -> JSONValue { - guard context.allowedCapabilities.contains(capability) || context.allowedCapabilityKeys.contains(capability.codeModeKey) else { + // Built-ins are gated by `allowedCapabilities` alone. `allowedCapabilityKeys` + // accepts arbitrary strings and is not validated against `CapabilityID` at + // decode time, so honouring it here would bypass any host that vets only + // the strictly-typed field. + guard context.allowedCapabilities.contains(capability) else { throw BridgeError.capabilityDenied(capability) } @@ -547,6 +569,13 @@ public final class CapabilityRegistry: @unchecked Sendable { } private func invokeCodeMode(_ capabilityKey: CodeModeCapabilityKey, arguments: [String: JSONValue], context: BridgeInvocationContext) throws -> JSONValue { + // Defense in depth: `invoke` already routes anything that parses as a + // built-in to `invokeBuiltIn`, but a key that spells a built-in must never + // reach a handler through the provider path. + if let builtIn = CapabilityID(rawValue: capabilityKey.rawValue) { + throw BridgeError.capabilityDenied(builtIn) + } + guard context.allowedCapabilityKeys.contains(capabilityKey) else { throw BridgeError.capabilityKeyDenied(capabilityKey) } @@ -594,7 +623,10 @@ public final class CapabilityRegistry: @unchecked Sendable { let trimmed = raw.trimmingCharacters(in: .whitespaces) switch type { case .number: - guard let number = Double(trimmed) else { return nil } + // `Double.init(String)` accepts "inf", "-infinity", and "nan", so a + // declared-number argument sent as a string is a way to smuggle a + // non-finite value past the type check. + guard let number = Double(trimmed), number.isFinite else { return nil } return .number(number) case .bool: switch trimmed.lowercased() { diff --git a/Sources/CodeMode/Runtime/BridgeInvocationContext.swift b/Sources/CodeMode/Runtime/BridgeInvocationContext.swift index 4a64acf..2148721 100644 --- a/Sources/CodeMode/Runtime/BridgeInvocationContext.swift +++ b/Sources/CodeMode/Runtime/BridgeInvocationContext.swift @@ -2,8 +2,17 @@ import Foundation public final class BridgeInvocationContext: @unchecked Sendable { public let executionContext: ExecutionContext + /// Built-in capabilities this execution may reach: the model's declaration + /// intersected with the host's `CapabilityGrant`. public let allowedCapabilities: Set + /// Custom provider keys this execution may reach. Deliberately disjoint from + /// `allowedCapabilities` — a built-in capability is never granted by a key, or + /// `allowedCapabilityKeys` would be a way around a host that vets only the + /// strictly-typed `allowedCapabilities`. public let allowedCapabilityKeys: Set + /// Identifiers the request asked for and the host's grant withheld. Used to + /// tell the model a denial is not repairable by widening `allowedCapabilities`. + let hostWithheldCapabilityIdentifiers: Set public let pathPolicy: any PathPolicy public let artifactStore: any ArtifactStore public let permissionBroker: any PermissionBroker @@ -12,6 +21,7 @@ public final class BridgeInvocationContext: @unchecked Sendable { private let lock = NSLock() private var validatedPermissions: Set = [] + private var failedCapabilityInvocations: [(capability: String, code: String)] = [] private let transcript: ExecutionTranscript private let cancellationController: ExecutionCancellationController @@ -19,6 +29,7 @@ public final class BridgeInvocationContext: @unchecked Sendable { executionContext: ExecutionContext, allowedCapabilities: Set, allowedCapabilityKeys: Set = [], + hostWithheldCapabilityIdentifiers: Set = [], pathPolicy: any PathPolicy, artifactStore: any ArtifactStore, permissionBroker: any PermissionBroker, @@ -29,7 +40,8 @@ public final class BridgeInvocationContext: @unchecked Sendable { ) { self.executionContext = executionContext self.allowedCapabilities = allowedCapabilities - self.allowedCapabilityKeys = allowedCapabilityKeys.union(allowedCapabilities.map(\.codeModeKey)) + self.allowedCapabilityKeys = allowedCapabilityKeys + self.hostWithheldCapabilityIdentifiers = hostWithheldCapabilityIdentifiers self.pathPolicy = pathPolicy self.artifactStore = artifactStore self.permissionBroker = permissionBroker @@ -97,12 +109,64 @@ public final class BridgeInvocationContext: @unchecked Sendable { return resolved } + /// True when the request declared this identifier but the host's + /// `CapabilityGrant` withheld it — a denial the model cannot repair. + func isWithheldByHost(_ identifier: String) -> Bool { + hostWithheldCapabilityIdentifiers.contains(identifier) + } + + /// Builds a `CodeModeToolError` carrying this execution's transcript. + /// + /// Lives here rather than on the runtime because the transcript fields are + /// this type's; three separate call sites were otherwise assembling the same + /// three accessors by hand. + func toolError( + code: String, + message: String, + functionName: String? = nil, + capability: CapabilityID? = nil, + capabilityKey: CodeModeCapabilityKey? = nil, + suggestions: [String] = [] + ) -> CodeModeToolError { + CodeModeToolError( + code: code, + message: message, + functionName: functionName, + capability: capability, + capabilityKey: capabilityKey, + suggestions: suggestions, + diagnostics: allDiagnostics(), + logs: allLogs(), + permissionEvents: allPermissionEvents() + ) + } + func recordDiagnostic(_ diagnostic: ToolDiagnostic) { transcript.record(diagnostic: diagnostic) } + /// Notes a capability call that failed, so a script that finishes + /// *successfully* despite failed bridge calls can be flagged. Nothing installs + /// an unhandled-rejection hook — a forgotten `await` is the most common LLM + /// JavaScript mistake, and it silently discards a `CAPABILITY_DENIED` while + /// the agent is told the run succeeded. + func recordCapabilityFailure(capability: String, code: String) { + lock.lock() + failedCapabilityInvocations.append((capability, code)) + lock.unlock() + } + + func capabilityFailures() -> [(capability: String, code: String)] { + lock.lock() + defer { lock.unlock() } + return failedCapabilityInvocations + } + func checkCancellation() throws { - if cancellationController.isCancelled || Task.isCancelled { + // Bridge handlers run on a GCD worker, not inside a Task, so + // `Task.isCancelled` was always false here. The controller is the signal + // `JavaScriptExecutionCall.cancel()` actually sets. + if cancellationController.isCancelled { throw BridgeError.cancelled } } diff --git a/Sources/CodeMode/Runtime/BridgeRuntime.swift b/Sources/CodeMode/Runtime/BridgeRuntime.swift index 4eec3c2..416d6eb 100644 --- a/Sources/CodeMode/Runtime/BridgeRuntime.swift +++ b/Sources/CodeMode/Runtime/BridgeRuntime.swift @@ -18,6 +18,11 @@ final class BridgeRuntime: @unchecked Sendable { var suggestions: [String] } + /// Ceiling on the re-armed watchdog budget for result serialization. Bounded + /// independently of the execution timeout so a long-running script cannot buy + /// an equally long serialization window. + private static let serializationBudgetMs = 1_000 + private let registry: CapabilityRegistry private let catalog: BridgeCatalog private let config: CodeModeConfiguration @@ -27,17 +32,36 @@ final class BridgeRuntime: @unchecked Sendable { qos: .userInitiated, attributes: .concurrent ) + /// Caps how many executions hold a thread and a live `JSContext` at once. + /// + /// The queue is concurrent and each execution blocks its thread for the whole + /// run, so thirty parallel `executeJavaScript` calls meant thirty blocked GCD + /// threads and thirty JavaScriptCore VMs. Excess executions park as suspended + /// tasks — not blocked threads — until a slot frees; `timeoutMs` still + /// measures the run itself, not the wait. + private let executionSlots: ExecutionSlots + /// Search has its own slots. It is cheap, capability-free, and bounded to 2s, + /// and sharing the execution slots meant API discovery stalled behind long + /// scripts — exactly when an agent is trying to work out its next step. + private let searchSlots: ExecutionSlots + /// Real in production; virtual under test so timer-driven tests are exact + /// and instant. See RuntimeClock. + private let clock: any RuntimeClock init( registry: CapabilityRegistry, catalog: BridgeCatalog, config: CodeModeConfiguration, - unsupportedBuiltInJavaScriptNames: [String] = [] + unsupportedBuiltInJavaScriptNames: [String] = [], + clock: any RuntimeClock = RealClock() ) { self.registry = registry self.catalog = catalog self.config = config self.unsupportedBuiltInJavaScriptNames = unsupportedBuiltInJavaScriptNames + self.clock = clock + self.executionSlots = ExecutionSlots(count: config.executionLimits.maxConcurrentExecutions) + self.searchSlots = ExecutionSlots(count: config.executionLimits.maxConcurrentExecutions) } func search(_ request: JavaScriptAPISearchRequest) throws -> JavaScriptAPISearchResponse { @@ -49,7 +73,7 @@ final class BridgeRuntime: @unchecked Sendable { ) } - let transcript = ExecutionTranscript() + let transcript = ExecutionTranscript(limits: config.executionLimits) let cancellationController = ExecutionCancellationController() let invocationContext = BridgeInvocationContext( executionContext: .init(), @@ -96,7 +120,7 @@ final class BridgeRuntime: @unchecked Sendable { } func searchAsync(_ request: JavaScriptAPISearchRequest) async throws -> JavaScriptAPISearchResponse { - try await runOnExecutionQueue { + try await runOnExecutionQueue(slots: searchSlots) { try self.search(request) } } @@ -104,10 +128,28 @@ final class BridgeRuntime: @unchecked Sendable { func makeExecutionCall(_ request: JavaScriptExecutionRequest) -> JavaScriptExecutionCall { let cancellationController = ExecutionCancellationController() let continuationBox = LockedBox.Continuation?>(nil) - let events = AsyncStream { continuation in + let resultTaskBox = LockedBox?>(nil) + + // Shared by the call and the stream; the execution is cancelled when + // the last of the two goes away. See ExecutionObservationToken. + let observation = ExecutionObservationToken { + cancellationController.cancel() + resultTaskBox.get()?.cancel() + } + + // Bounded: the default policy buffers without limit, so a chatty script + // whose events the host is not draining grows the buffer until the app + // dies. Dropping the oldest keeps the terminal events — which is what + // the host actually needs — and the full transcript is on the result. + let events = AsyncStream( + bufferingPolicy: .bufferingNewest(config.executionLimits.maxBufferedEvents) + ) { continuation in continuationBox.set(continuation) + // The stream's storage retains this closure until termination, which + // is what keeps the token alive for as long as someone is iterating. + continuation.onTermination = { _ in withExtendedLifetime(observation) {} } } - let transcript = ExecutionTranscript { event in + let transcript = ExecutionTranscript(limits: config.executionLimits) { event in continuationBox.get()?.yield(event) } @@ -154,9 +196,12 @@ final class BridgeRuntime: @unchecked Sendable { } } + resultTaskBox.set(resultTask) + return JavaScriptExecutionCall( events: events, resultTask: resultTask, + observation: observation, cancelImpl: { cancellationController.cancel() } @@ -164,9 +209,17 @@ final class BridgeRuntime: @unchecked Sendable { } private func runOnExecutionQueue( + slots: ExecutionSlots? = nil, _ operation: @escaping @Sendable () throws -> Output ) async throws -> Output { - try await withCheckedThrowingContinuation { continuation in + // The slot is taken *here*, in the async context, so an execution beyond + // the limit suspends without touching the thread pool. A cancel while + // parked throws before anything is dispatched. + let slots = slots ?? executionSlots + try await slots.acquire() + defer { Task { await slots.release() } } + + return try await withCheckedThrowingContinuation { continuation in executionQueue.async { do { continuation.resume(returning: try operation()) @@ -182,10 +235,35 @@ final class BridgeRuntime: @unchecked Sendable { transcript: ExecutionTranscript, cancellationController: ExecutionCancellationController ) throws -> JavaScriptExecutionResult { + // `allowedCapabilities` / `allowedCapabilityKeys` are model-authored, so they + // are a declaration of intent, not a boundary. The host's grant is the + // boundary: the effective set is always requested ∩ granted. + let grant = config.capabilityGrant.resolve( + requestedCapabilities: Set(request.allowedCapabilities), + requestedCapabilityKeys: Set(request.allowedCapabilityKeys) + ) + + let withheld = grant.withheld + if withheld.isEmpty == false { + transcript.record( + diagnostic: ToolDiagnostic( + severity: .warning, + code: "CAPABILITY_WITHHELD_BY_HOST", + message: "The host's capability grant withheld: \(withheld.joined(separator: ", ")). Calls needing these fail with CAPABILITY_DENIED and cannot be repaired from the script.", + category: "security", + suggestions: [ + "Do not retry with a wider allowedCapabilities; the host decides this set.", + "Complete what you can with the granted capabilities, or report the missing access to the user.", + ] + ) + ) + } + let invocationContext = BridgeInvocationContext( executionContext: request.context, - allowedCapabilities: Set(request.allowedCapabilities), - allowedCapabilityKeys: Set(request.allowedCapabilityKeys), + allowedCapabilities: grant.capabilities, + allowedCapabilityKeys: grant.capabilityKeys, + hostWithheldCapabilityIdentifiers: Set(withheld), pathPolicy: config.pathPolicy, artifactStore: config.artifactStore, permissionBroker: config.permissionBroker, @@ -256,8 +334,10 @@ final class BridgeRuntime: @unchecked Sendable { let errorPayload = self.bridgeFailurePayload( for: bridgeError, capability: capabilityID, - capabilityKey: capabilityKey + capabilityKey: capabilityKey, + invocationContext: invocationContext ) + invocationContext.recordCapabilityFailure(capability: capability, code: errorPayload.code) invocationContext.log(.error, message: "Capability failed \(capability): \(errorPayload.message)") invocationContext.auditLogger.log(AuditEvent(capability: capability, message: "failed: \(errorPayload.message)")) @@ -436,7 +516,7 @@ final class BridgeRuntime: @unchecked Sendable { }); """ - let watchdog = ExecutionWatchdog(timeoutMs: timeoutMs, cancellationController: cancellationController) + let watchdog = ExecutionWatchdog(timeoutMs: timeoutMs, cancellationController: cancellationController, clock: clock) watchdog.install(on: context) defer { watchdog.uninstall(from: context) } @@ -445,16 +525,14 @@ final class BridgeRuntime: @unchecked Sendable { switch watchdog.termination { case .timedOut: - throw toolError( + throw invocationContext.toolError( code: "EXECUTION_TIMEOUT", - message: "Execution timed out after \(timeoutMs)ms", - transcript: invocationContext + message: "Execution timed out after \(timeoutMs)ms" ) case .cancelled: - throw toolError( + throw invocationContext.toolError( code: "CANCELLED", - message: "Execution cancelled", - transcript: invocationContext + message: "Execution cancelled" ) case nil: break @@ -467,22 +545,31 @@ final class BridgeRuntime: @unchecked Sendable { let settlement = try waitForSettlement( context: context, - watchdog: watchdog, - cancellationController: cancellationController, - invocationContext: invocationContext, - timeoutCode: "EXECUTION_TIMEOUT", - timeoutMessage: "Execution timed out after \(timeoutMs)ms", - cancelMessage: "Execution cancelled" + settlement: SettlementParameters( + watchdog: watchdog, + cancellationController: cancellationController, + invocationContext: invocationContext, + timeoutCode: "EXECUTION_TIMEOUT", + timeoutMessage: "Execution timed out after \(timeoutMs)ms", + cancelMessage: "Execution cancelled" + ) ) // Give result serialization its own bounded budget so a script that // settled near the deadline still serializes, while a runaway getter or - // toJSON is terminated instead of hanging the execution thread. - watchdog.rearm(timeoutMs: max(timeoutMs, 1_000)) + // toJSON is terminated instead of hanging the execution thread. `min`, + // not `max`: the point is a *bounded* budget, and `max` handed a 60s + // execution another 60s just to run JSON.stringify. + watchdog.rearm(timeoutMs: min(timeoutMs, Self.serializationBudgetMs)) switch settlement { case .fulfilled: - let output = try decodeOutput(from: context) + recordSwallowedFailures(invocationContext: invocationContext) + let output = try decodeOutput( + from: context, + watchdog: watchdog, + invocationContext: invocationContext + ) if output == nil { invocationContext.recordDiagnostic(Self.noReturnValueDiagnostic(for: code)) } @@ -498,25 +585,36 @@ final class BridgeRuntime: @unchecked Sendable { case rejected } - /// Polls the wrapped promise's settled state until it resolves, is - /// cancelled, or the watchdog deadline passes. The settled state is checked - /// before the deadline on every iteration, so a script that has already - /// fulfilled — the common case under the synchronous bridge model — returns - /// its result even if evaluation finished a hair past the deadline, instead - /// of being discarded with a spurious timeout. + /// Runs the runtime's event loop until the wrapped promise settles, the + /// deadline passes, or the program is provably stuck. + /// + /// The settled state is checked before the deadline on every iteration, so a + /// script that has already fulfilled returns its result even if evaluation + /// finished a hair past the deadline, instead of being discarded with a + /// spurious timeout. + /// + /// When the promise is still pending, the *only* thing that can advance it + /// under the synchronous bridge model is a queued timer — microtasks have + /// already drained inside `evaluateScript`, and no native call is + /// outstanding. So this either sleeps until the next timer is due and fires + /// it, or, when no timer is queued, reports immediately that the promise can + /// never settle instead of sleeping a thread to the deadline for nothing. private func waitForSettlement( context: JSContext, - watchdog: ExecutionWatchdog, - cancellationController: ExecutionCancellationController, - invocationContext: BridgeInvocationContext, - timeoutCode: String, - timeoutMessage: String, - cancelMessage: String + settlement: SettlementParameters ) throws -> SettlementState { while true { - if cancellationController.isCancelled || Task.isCancelled { - cancellationController.cancel() - throw toolError(code: "CANCELLED", message: cancelMessage, transcript: invocationContext) + // Cancellation only, deliberately: the deadline is checked *after* + // the settled state below, so a script that fulfilled a hair past it + // still returns its result instead of a spurious timeout. + // + // `Task.isCancelled` is not checked at all — this runs on a plain GCD + // worker with no surrounding Task, so it was always false, dead code + // that read as a second layer of cancellation support. The + // controller, which `JavaScriptExecutionCall.cancel()` sets, is the + // real signal. + if settlement.cancellationController.isCancelled { + throw settlement.error(code: "CANCELLED", message: settlement.cancelMessage) } let state = context.evaluateScript("globalThis.__codemode.state")?.toString() ?? "unknown" @@ -526,22 +624,135 @@ final class BridgeRuntime: @unchecked Sendable { case "rejected": return .rejected default: - if let termination = watchdog.termination { - switch termination { - case .cancelled: - throw toolError(code: "CANCELLED", message: cancelMessage, transcript: invocationContext) - case .timedOut: - throw toolError(code: timeoutCode, message: timeoutMessage, transcript: invocationContext) - } - } - if Date() >= watchdog.deadline { - throw toolError(code: timeoutCode, message: timeoutMessage, transcript: invocationContext) + try checkInterrupted(settlement) + + let nextTimerDelay = context + .evaluateScript("globalThis.__codemode.nextTimerDelay()")? + .toNumber()? + .doubleValue ?? -1 + + guard nextTimerDelay >= 0 else { + throw settlement.error( + code: "JS_RUNTIME_ERROR", + message: "Script finished with a promise that can never settle: nothing is left to resolve it. This runtime has no I/O event loop, so only a pending setTimeout can advance a program after the synchronous work is done.", + suggestions: [ + "Await every promise the script creates, and make sure each one has a resolve path.", + "A `new Promise(...)` whose executor never calls resolve or reject will hang forever here.", + ] + ) } - Thread.sleep(forTimeInterval: 0.01) + + try runDueTimers(delayMs: nextTimerDelay, context: context, settlement: settlement) } } } + /// Everything the settlement loop needs to report a cancellation or timeout. + /// The execution and search paths differ only in these message strings. + private struct SettlementParameters { + var watchdog: ExecutionWatchdog + var cancellationController: ExecutionCancellationController + var invocationContext: BridgeInvocationContext + var timeoutCode: String + var timeoutMessage: String + var cancelMessage: String + + func error(code: String, message: String, suggestions: [String] = []) -> CodeModeToolError { + invocationContext.toolError(code: code, message: message, suggestions: suggestions) + } + } + + /// Throws if the run was cancelled or has run out of time — the check every + /// waiting loop in this file has to make before sleeping again. + private func checkInterrupted(_ settlement: SettlementParameters) throws { + if settlement.cancellationController.isCancelled { + throw settlement.error(code: "CANCELLED", message: settlement.cancelMessage) + } + switch settlement.watchdog.termination { + case .cancelled: + throw settlement.error(code: "CANCELLED", message: settlement.cancelMessage) + case .timedOut: + throw settlement.error(code: settlement.timeoutCode, message: settlement.timeoutMessage) + case nil: + break + } + if settlement.watchdog.hasPassedDeadline { + throw settlement.error(code: settlement.timeoutCode, message: settlement.timeoutMessage) + } + } + + /// Sleeps out a timer's delay — bounded by the deadline and responsive to + /// cancellation — then fires everything that has come due. + private func runDueTimers( + delayMs: Double, + context: JSContext, + settlement: SettlementParameters + ) throws { + let start = clock.now + let target = start.advanced(by: .milliseconds(Int(delayMs.rounded(.up)))) + + while clock.now < target { + try checkInterrupted(settlement) + // One interruptible wait until the timer is due or the deadline + // passes — a cancel wakes it immediately, and the loop re-checks + // both against the clock on waking. + clock.sleep(until: min(target, settlement.watchdog.deadline), cancellation: settlement.cancellationController) + } + + // Advance by the time that actually passed, so timers that came due while + // we slept fire together. The call returns the callbacks' uncaught errors, + // which cannot propagate to whoever called `setTimeout` and so surface as + // diagnostics the way an uncaught task error does in a real event loop. + let elapsed = (clock.now - start).milliseconds + let errors = Self.stringArray( + from: context, + evaluating: "globalThis.__codemode.advanceTimers(\(elapsed))" + ) + + for message in errors { + settlement.invocationContext.recordDiagnostic( + ToolDiagnostic( + severity: .warning, + code: "TIMER_CALLBACK_ERROR", + message: "A setTimeout callback threw and the error was discarded: \(message)", + category: "execution", + suggestions: ["Handle errors inside timer callbacks; they cannot propagate to the caller of setTimeout."] + ) + ) + } + } + + /// Flags bridge calls that failed while the script still reported success. + /// + /// The runtime installs no unhandled-rejection hook, so a forgotten `await` — + /// the most common LLM JavaScript mistake — discards a `CAPABILITY_DENIED` + /// and the agent is told the run completed. This does not prove the failure + /// was unhandled (a deliberate try/catch looks the same), so it is a warning + /// rather than an error, but it makes the silent case visible. + private func recordSwallowedFailures(invocationContext: BridgeInvocationContext) { + let failures = invocationContext.capabilityFailures() + guard failures.isEmpty == false else { + return + } + + let summary = failures + .map { "\($0.capability) (\($0.code))" } + .joined(separator: ", ") + + invocationContext.recordDiagnostic( + ToolDiagnostic( + severity: .warning, + code: "BRIDGE_FAILURES_NOT_SURFACED", + message: "The script completed successfully, but \(failures.count) bridge call(s) failed and the failure did not reach the result: \(summary).", + category: "execution", + suggestions: [ + "If that was not deliberate, check for a missing await — an un-awaited helper's rejection is silently discarded here.", + "If it was deliberate, ignore this; the result is what the script returned.", + ] + ) + ) + } + private static func noReturnValueDiagnostic(for code: String) -> ToolDiagnostic { let mentionsAsyncIIFE = code.contains("(async") && (code.contains("})()") || code.contains("})();")) let message: String @@ -611,7 +822,7 @@ final class BridgeRuntime: @unchecked Sendable { }); """ - let watchdog = ExecutionWatchdog(timeoutMs: timeoutMs, cancellationController: cancellationController) + let watchdog = ExecutionWatchdog(timeoutMs: timeoutMs, cancellationController: cancellationController, clock: clock) watchdog.install(on: context) defer { watchdog.uninstall(from: context) } @@ -620,16 +831,14 @@ final class BridgeRuntime: @unchecked Sendable { switch watchdog.termination { case .timedOut: - throw toolError( + throw invocationContext.toolError( code: "SEARCH_TIMEOUT", - message: "Search timed out after \(timeoutMs)ms", - transcript: invocationContext + message: "Search timed out after \(timeoutMs)ms" ) case .cancelled: - throw toolError( + throw invocationContext.toolError( code: "CANCELLED", - message: "Search cancelled", - transcript: invocationContext + message: "Search cancelled" ) case nil: break @@ -642,20 +851,24 @@ final class BridgeRuntime: @unchecked Sendable { let settlement = try waitForSettlement( context: context, - watchdog: watchdog, - cancellationController: cancellationController, - invocationContext: invocationContext, - timeoutCode: "SEARCH_TIMEOUT", - timeoutMessage: "Search timed out after \(timeoutMs)ms", - cancelMessage: "Search cancelled" + settlement: SettlementParameters( + watchdog: watchdog, + cancellationController: cancellationController, + invocationContext: invocationContext, + timeoutCode: "SEARCH_TIMEOUT", + timeoutMessage: "Search timed out after \(timeoutMs)ms", + cancelMessage: "Search cancelled" + ) ) - watchdog.rearm(timeoutMs: max(timeoutMs, 1_000)) + watchdog.rearm(timeoutMs: min(timeoutMs, Self.serializationBudgetMs)) switch settlement { case .fulfilled: return try decodeOutput( from: context, + watchdog: watchdog, + invocationContext: invocationContext, errorCode: "INVALID_SEARCH_RESULT", errorMessagePrefix: "Search result must be JSON-serializable" ) @@ -667,6 +880,8 @@ final class BridgeRuntime: @unchecked Sendable { private func decodeOutput( from context: JSContext, + watchdog: ExecutionWatchdog? = nil, + invocationContext: BridgeInvocationContext? = nil, errorCode: String = "INVALID_RESULT", errorMessagePrefix: String = "Execution result must be JSON-serializable" ) throws -> JSONValue? { @@ -674,17 +889,46 @@ final class BridgeRuntime: @unchecked Sendable { return nil } - guard let serialization = context.evaluateScript( + let limits = config.executionLimits + // Size is measured and the result is replaced *inside* JS: bringing a + // gigabyte of JSON across into a Swift String and then rejecting it has + // already spent the memory the cap exists to protect. The replacement is + // still valid JSON, so the model gets a parseable value plus a + // diagnostic rather than a truncated fragment it cannot read. + let serialization = context.evaluateScript( """ (function(){ try { - return { ok: true, json: JSON.stringify(globalThis.__codemode.result) }; + var json = JSON.stringify(globalThis.__codemode.result); + if (typeof json === 'string' && json.length > \(limits.maxResultCharacters)) { + return { + ok: true, + truncated: true, + size: json.length, + json: JSON.stringify({ + __codeModeTruncated: true, + characterCount: json.length, + limit: \(limits.maxResultCharacters), + preview: json.slice(0, \(limits.truncatedResultPreviewCharacters)) + }) + }; + } + return { ok: true, json: json }; } catch (error) { return { ok: false, message: String(error) }; } })() """ - ) else { + ) + + // Checked before anything else: a watchdog termination during + // serialization is a timeout, not a serialization problem, and reporting + // `INVALID_RESULT: must be JSON-serializable` for a runaway getter + // steered the model to fix the wrong thing. + if let termination = watchdog?.termination { + throw terminationError(termination, invocationContext: invocationContext, during: "result serialization") + } + guard let serialization else { throw CodeModeToolError(code: errorCode, message: errorMessagePrefix) } @@ -693,6 +937,22 @@ final class BridgeRuntime: @unchecked Sendable { throw CodeModeToolError(code: errorCode, message: "\(errorMessagePrefix): \(message)") } + if serialization.forProperty("truncated")?.toBool() == true { + let size = serialization.forProperty("size")?.toNumber()?.intValue ?? limits.maxResultCharacters + invocationContext?.recordDiagnostic( + ToolDiagnostic( + severity: .warning, + code: "RESULT_TRUNCATED", + message: "The result serialized to \(size) characters, over the \(limits.maxResultCharacters)-character limit. It was replaced with a descriptor object carrying a \(limits.truncatedResultPreviewCharacters)-character preview.", + category: "execution", + suggestions: [ + "Return a summary, a count, or a slice instead of the whole collection.", + "Write large payloads to the sandbox with apple.fs.write and return the path.", + ] + ) + ) + } + guard let jsonValue = serialization.forProperty("json"), jsonValue.isUndefined == false, jsonValue.isNull == false, let serialized = jsonValue.toString() else { @@ -706,12 +966,37 @@ final class BridgeRuntime: @unchecked Sendable { return try JSONDecoder.codeModeBridge.decode(JSONValue.self, from: data) } + /// Reports a watchdog termination that happened outside the script body — so + /// far, during result serialization — as the timeout or cancellation it + /// actually is. + private func terminationError( + _ termination: ExecutionWatchdog.Termination, + invocationContext: BridgeInvocationContext?, + during phase: String + ) -> CodeModeToolError { + let (code, message): (String, String) = switch termination { + case .timedOut: + ("EXECUTION_TIMEOUT", "Execution timed out during \(phase). A getter or toJSON on the returned value did not finish within the serialization budget.") + case .cancelled: + ("CANCELLED", "Execution cancelled during \(phase)") + } + + return invocationContext?.toolError(code: code, message: message) + ?? CodeModeToolError(code: code, message: message) + } + private func bridgeFailurePayload( for error: BridgeError, capability: CapabilityID?, - capabilityKey: CodeModeCapabilityKey? + capabilityKey: CodeModeCapabilityKey?, + invocationContext: BridgeInvocationContext? = nil ) -> CodeModeToolError { - let (message, suggestions) = enrichedBridgeFailure(for: error, capability: capability, capabilityKey: capabilityKey) + let (message, suggestions) = enrichedBridgeFailure( + for: error, + capability: capability, + capabilityKey: capabilityKey, + invocationContext: invocationContext + ) return CodeModeToolError( code: error.diagnosticCode, message: message, @@ -833,13 +1118,19 @@ final class BridgeRuntime: @unchecked Sendable { } private func rejectedSuggestions(from context: JSContext) -> [String] { - guard let json = context.evaluateScript("JSON.stringify(globalThis.__codemode.error?.suggestions ?? [])")?.toString(), + Self.stringArray(from: context, evaluating: "globalThis.__codemode.error?.suggestions ?? []") + } + + /// Reads a JavaScript array of strings across the bridge. Returns an empty + /// array if the expression is missing or not string-shaped. + private static func stringArray(from context: JSContext, evaluating expression: String) -> [String] { + guard let json = context.evaluateScript("JSON.stringify(\(expression))")?.toString(), let data = json.data(using: .utf8), - let suggestions = try? JSONDecoder.codeModeBridge.decode([String].self, from: data) + let values = try? JSONDecoder.codeModeBridge.decode([String].self, from: data) else { return [] } - return suggestions + return values } private func event(for error: CodeModeToolError) -> JavaScriptExecutionEvent { @@ -855,28 +1146,6 @@ final class BridgeRuntime: @unchecked Sendable { } } - private func toolError( - code: String, - message: String, - transcript: BridgeInvocationContext, - functionName: String? = nil, - capability: CapabilityID? = nil, - capabilityKey: CodeModeCapabilityKey? = nil, - suggestions: [String] = [] - ) -> CodeModeToolError { - CodeModeToolError( - code: code, - message: message, - functionName: functionName, - capability: capability, - capabilityKey: capabilityKey, - suggestions: suggestions, - diagnostics: transcript.allDiagnostics(), - logs: transcript.allLogs(), - permissionEvents: transcript.allPermissionEvents() - ) - } - private func bridgeSuggestions(for capability: CapabilityID?) -> [String] { bridgeSuggestions(for: capability, capabilityKey: capability?.codeModeKey) } @@ -908,14 +1177,34 @@ final class BridgeRuntime: @unchecked Sendable { private func enrichedBridgeFailure( for error: BridgeError, capability: CapabilityID?, - capabilityKey: CodeModeCapabilityKey? + capabilityKey: CodeModeCapabilityKey?, + invocationContext: BridgeInvocationContext? = nil ) -> (String, [String]) { + // A host-withheld denial reads the same whichever allowlist it came from, + // and is the one denial the model cannot repair — so it short-circuits + // both cases below. + let deniedIdentifier: String? = switch error { + case let .capabilityDenied(capability): capability.rawValue + case let .capabilityKeyDenied(capabilityKey): capabilityKey.rawValue + default: nil + } + if let deniedIdentifier, invocationContext?.isWithheldByHost(deniedIdentifier) == true { + return ( + error.localizedDescription, + [ + "The host app's capability grant does not include \"\(deniedIdentifier)\".", + "This is not repaired by adding it to allowedCapabilities or allowedCapabilityKeys; the host owns this decision.", + "Continue with the capabilities you do have, or report the missing access to the user.", + ] + ) + } + let suggestions: [String] switch error { case let .capabilityDenied(capability): suggestions = [ "Add \"\(capability.rawValue)\" to allowedCapabilities and retry.", - "If your host uses a unified capability-key allowlist, add \"\(capability.rawValue)\" to allowedCapabilityKeys.", + "Built-in capabilities are granted only by allowedCapabilities; listing one in allowedCapabilityKeys has no effect.", ] + bridgeSuggestions(for: capability, capabilityKey: capability.codeModeKey) case let .capabilityKeyDenied(capabilityKey): suggestions = [ diff --git a/Sources/CodeMode/Runtime/ExecutionLimits.swift b/Sources/CodeMode/Runtime/ExecutionLimits.swift new file mode 100644 index 0000000..f5529fb --- /dev/null +++ b/Sources/CodeMode/Runtime/ExecutionLimits.swift @@ -0,0 +1,66 @@ +import Foundation + +/// Bounds on how much an execution may accumulate, plus how many may run at once. +/// +/// Nothing about a script's *output* was previously capped: the whole result was +/// stringified, copied into a Swift `String`, and re-decoded into a `JSONValue`; +/// every `console.log` was retained and then copied again into any +/// `CodeModeToolError`; and the events `AsyncStream` used unbounded buffering, so +/// a chatty script that the host was not draining buffered without limit. A +/// single script could jetsam an iOS host. +/// +/// For an LLM consumer, a truncated result carrying a diagnostic is strictly +/// better than an out-of-memory kill, so every limit here degrades rather than +/// fails. +public struct ExecutionLimits: Sendable, Equatable { + /// Maximum size of the serialized result, in JSON characters. A larger + /// result is replaced by a descriptor object carrying a prefix of the JSON. + public var maxResultCharacters: Int + + /// How much of the oversized JSON to include in that descriptor. + public var truncatedResultPreviewCharacters: Int + + /// Maximum retained `console.log` entries. + public var maxLogEntries: Int + + /// Maximum characters retained per log message; longer messages are elided + /// in the middle, which keeps both the prefix and the tail readable. + public var maxLogMessageCharacters: Int + + /// Maximum retained diagnostics. + public var maxDiagnostics: Int + + /// Maximum retained permission events. + public var maxPermissionEvents: Int + + /// How many executions may hold a thread and a live JavaScriptCore context + /// at once. Excess executions queue rather than exhausting the GCD thread + /// pool; the per-execution `timeoutMs` measures the run, not the wait. + public var maxConcurrentExecutions: Int + + /// Depth of the events `AsyncStream` buffer. A host that stops draining + /// drops the oldest events rather than growing without bound. + public var maxBufferedEvents: Int + + public init( + maxResultCharacters: Int = 1_000_000, + truncatedResultPreviewCharacters: Int = 4_000, + maxLogEntries: Int = 1_000, + maxLogMessageCharacters: Int = 8_000, + maxDiagnostics: Int = 200, + maxPermissionEvents: Int = 200, + maxBufferedEvents: Int = 1_000, + maxConcurrentExecutions: Int = 8 + ) { + self.maxResultCharacters = maxResultCharacters + self.truncatedResultPreviewCharacters = truncatedResultPreviewCharacters + self.maxLogEntries = maxLogEntries + self.maxLogMessageCharacters = maxLogMessageCharacters + self.maxDiagnostics = maxDiagnostics + self.maxPermissionEvents = maxPermissionEvents + self.maxBufferedEvents = maxBufferedEvents + self.maxConcurrentExecutions = maxConcurrentExecutions + } + + public static let standard = ExecutionLimits() +} diff --git a/Sources/CodeMode/Runtime/ExecutionSupport.swift b/Sources/CodeMode/Runtime/ExecutionSupport.swift index 1b20c1f..ac6f1d2 100644 --- a/Sources/CodeMode/Runtime/ExecutionSupport.swift +++ b/Sources/CodeMode/Runtime/ExecutionSupport.swift @@ -1,56 +1,201 @@ import Foundation final class ExecutionCancellationController: @unchecked Sendable { - private let lock = NSLock() + private let condition = NSCondition() private var cancelled = false func cancel() { - lock.lock() + condition.lock() cancelled = true - lock.unlock() + // Wakes anything parked in `sleep(until:)`, so a cancel lands + // immediately instead of at the next poll. + condition.broadcast() + condition.unlock() } var isCancelled: Bool { - lock.lock() - defer { lock.unlock() } + condition.lock() + defer { condition.unlock() } return cancelled } + + /// Blocks until `date` or until cancelled, whichever is first. The timer + /// loop used to poll `Thread.sleep` in 5ms slices — 200 wakeups a second per + /// sleeping execution, and up to 5ms of cancellation latency — where one + /// interruptible wait does both jobs. + func sleep(until date: Date) { + condition.lock() + defer { condition.unlock() } + while cancelled == false, Date() < date { + condition.wait(until: date) + } + } +} + +/// Bounds how many executions run at once, parking the excess as suspended +/// tasks rather than blocked threads. +/// +/// A `DispatchSemaphore` acquired on the GCD worker got this exactly backwards: +/// every execution beyond the limit held a thread while waiting for a slot, which +/// is the resource the limit exists to protect. Waiters here cost nothing until +/// a slot frees, and a cancelled waiter leaves the queue without ever running. +actor ExecutionSlots { + private var available: Int + private var waiters: [(id: UUID, continuation: CheckedContinuation)] = [] + + init(count: Int) { + available = max(1, count) + } + + func acquire() async throws { + try Task.checkCancellation() + if available > 0 { + available -= 1 + return + } + + let id = UUID() + try await withTaskCancellationHandler { + // Runs synchronously on the actor, so the waiter is always parked + // before a `withdraw` from the cancellation handler can be scheduled + // here — even when the task was cancelled before this point. + try await withCheckedThrowingContinuation { (continuation: CheckedContinuation) in + waiters.append((id, continuation)) + } + } onCancel: { + Task { await self.withdraw(id) } + } + } + + func release() { + // Hand the slot straight to the next waiter; `available` only grows when + // nobody is waiting for it. + if waiters.isEmpty == false { + waiters.removeFirst().continuation.resume() + } else { + available += 1 + } + } + + /// Removes a cancelled waiter. Not finding it means `release()` already + /// handed it a slot — the cancel arrived too late to matter, and the + /// execution observes it through its cancellation controller instead. An + /// earlier version recorded such ids "for later" and never removed them. + private func withdraw(_ id: UUID) { + if let index = waiters.firstIndex(where: { $0.id == id }) { + waiters.remove(at: index).continuation.resume(throwing: CancellationError()) + } + } +} + +/// Cancels an execution once nothing can observe its outcome. +/// +/// Held by both the `JavaScriptExecutionCall` and its events stream, so the +/// execution keeps running while *either* is reachable — a host iterating +/// `call.events` after its last use of `call` is still observing. Cancelling in +/// the call's own `deinit` cut that host off mid-stream: Swift does not keep a +/// local alive to the end of its scope, and an optimized build released `call` as +/// soon as `.events` had been read. +final class ExecutionObservationToken: @unchecked Sendable { + private let onDrop: @Sendable () -> Void + + init(onDrop: @escaping @Sendable () -> Void) { + self.onDrop = onDrop + } + + deinit { + onDrop() + } } final class ExecutionTranscript: @unchecked Sendable { private let lock = NSLock() + private let limits: ExecutionLimits private var logs: [ExecutionLog] = [] private var diagnostics: [ToolDiagnostic] = [] private var permissionEvents: [PermissionEvent] = [] + private var noticedLogOverflow = false private let emitEvent: @Sendable (JavaScriptExecutionEvent) -> Void - init(emitEvent: @escaping @Sendable (JavaScriptExecutionEvent) -> Void = { _ in }) { + init( + limits: ExecutionLimits = .standard, + emitEvent: @escaping @Sendable (JavaScriptExecutionEvent) -> Void = { _ in } + ) { + self.limits = limits self.emitEvent = emitEvent } func record(log: ExecutionLog) { + var entry = log + entry.message = Self.elide(entry.message, to: limits.maxLogMessageCharacters) + + // Keep the first N: the beginning of a runaway log is where the cause is, + // and the tail is the same line a million times. lock.lock() - logs.append(log) + let accepted = logs.count < limits.maxLogEntries + let isFirstOverflow = accepted == false && noticedLogOverflow == false + if accepted { + logs.append(entry) + } else { + noticedLogOverflow = true + } lock.unlock() - emitEvent(.log(log)) + + if accepted { + // Streaming stays live even after the retained transcript is full; + // it is the retained copy — re-copied into every CodeModeToolError — + // that is the memory risk. + emitEvent(.log(entry)) + } else if isFirstOverflow { + record( + diagnostic: ToolDiagnostic( + severity: .warning, + code: "LOG_LIMIT_REACHED", + message: "Kept the first \(limits.maxLogEntries) console.log entries; later entries are dropped.", + category: "execution", + suggestions: ["Log summaries rather than per-item lines, or return the data instead of logging it."] + ) + ) + } } func record(diagnostic: ToolDiagnostic) { lock.lock() - diagnostics.append(diagnostic) + let accepted = diagnostics.count < limits.maxDiagnostics + if accepted { + diagnostics.append(diagnostic) + } lock.unlock() - if diagnostic.severity != .error { + // Error-severity diagnostics used to be recorded but never emitted, so a + // host watching the stream saw every diagnostic *except* the ones that + // mattered most. They are on the result either way; the stream is the + // live view and should not be the incomplete one. + if accepted { emitEvent(.diagnostic(diagnostic)) } } func record(permissionEvent: PermissionEvent) { lock.lock() - permissionEvents.append(permissionEvent) + if permissionEvents.count < limits.maxPermissionEvents { + permissionEvents.append(permissionEvent) + } lock.unlock() } + /// Elides the middle of an over-long message, keeping both ends: a stack + /// trace's origin and its outcome are usually at opposite ends. + private static func elide(_ message: String, to limit: Int) -> String { + guard limit > 0, message.count > limit else { + return message + } + let keep = max(1, (limit - 1) / 2) + let head = message.prefix(keep) + let tail = message.suffix(keep) + return "\(head)… [\(message.count - 2 * keep) characters elided] …\(tail)" + } + func snapshot(output: JSONValue? = nil) -> JavaScriptExecutionResult { lock.lock() defer { lock.unlock() } diff --git a/Sources/CodeMode/Runtime/ExecutionWatchdog.swift b/Sources/CodeMode/Runtime/ExecutionWatchdog.swift index a9b8ea4..c70c1b4 100644 --- a/Sources/CodeMode/Runtime/ExecutionWatchdog.swift +++ b/Sources/CodeMode/Runtime/ExecutionWatchdog.swift @@ -40,21 +40,30 @@ final class ExecutionWatchdog: @unchecked Sendable { fileprivate static let checkInterval: TimeInterval = 0.05 private let lock = NSLock() - private var deadlineValue: Date + /// `ContinuousClock`, not `Date`: a wall-clock deadline moves when NTP + /// adjusts the clock, which can push a timeout arbitrarily far out (or fire it + /// immediately) while a script is running. + private var deadlineValue: ContinuousClock.Instant private var terminationValue: Termination? private let cancellationController: ExecutionCancellationController + let clock: any RuntimeClock - init(timeoutMs: Int, cancellationController: ExecutionCancellationController) { - self.deadlineValue = Date().addingTimeInterval(Double(timeoutMs) / 1_000) + init(timeoutMs: Int, cancellationController: ExecutionCancellationController, clock: any RuntimeClock = RealClock()) { + self.clock = clock + self.deadlineValue = clock.now.advanced(by: .milliseconds(timeoutMs)) self.cancellationController = cancellationController } - var deadline: Date { + var deadline: ContinuousClock.Instant { lock.lock() defer { lock.unlock() } return deadlineValue } + var hasPassedDeadline: Bool { + clock.now >= deadline + } + var termination: Termination? { lock.lock() defer { lock.unlock() } @@ -67,7 +76,7 @@ final class ExecutionWatchdog: @unchecked Sendable { /// user-defined getters/`toJSON`), while a runaway getter is still terminated. func rearm(timeoutMs: Int) { lock.lock() - deadlineValue = Date().addingTimeInterval(Double(timeoutMs) / 1_000) + deadlineValue = clock.now.advanced(by: .milliseconds(timeoutMs)) lock.unlock() } @@ -100,7 +109,7 @@ final class ExecutionWatchdog: @unchecked Sendable { terminationValue = .cancelled return true } - if Date() >= deadlineValue { + if clock.now >= deadlineValue { terminationValue = .timedOut return true } diff --git a/Sources/CodeMode/Runtime/RuntimeClock.swift b/Sources/CodeMode/Runtime/RuntimeClock.swift new file mode 100644 index 0000000..1ff11fe --- /dev/null +++ b/Sources/CodeMode/Runtime/RuntimeClock.swift @@ -0,0 +1,83 @@ +import Foundation + +/// The runtime's source of time, so the parts of execution that wait — the +/// timer loop and the watchdog deadline — can be driven by a virtual clock under +/// test instead of by real elapsed time. +/// +/// Production always uses `RealClock`. The seam exists because timer tests that +/// really slept were the least reliable tests in the suite: their assertions were +/// wall-clock bounds, and a loaded CI runner turned a 150ms wait into seconds. +/// Under a `VirtualClock` a sleep is a jump, so those tests are instant and +/// exact, and they can assert on how long the runtime *asked* to wait rather +/// than on how long the machine took. +/// +/// Watchdog termination of a CPU-bound script stays real-time regardless: it is +/// driven by JavaScriptCore's own execution-time callback, which virtual time +/// cannot advance. Tests of that path use `RealClock`. +protocol RuntimeClock: Sendable { + var now: ContinuousClock.Instant { get } + + /// Blocks until `deadline` or until `cancellation` fires, whichever is + /// first. A virtual clock jumps instead of blocking. + func sleep(until deadline: ContinuousClock.Instant, cancellation: ExecutionCancellationController) +} + +struct RealClock: RuntimeClock { + var now: ContinuousClock.Instant { ContinuousClock.now } + + func sleep(until deadline: ContinuousClock.Instant, cancellation: ExecutionCancellationController) { + // The wall-clock `Date` is only the wake-up hint; callers re-check + // against `now`, so an NTP adjustment cannot move a deadline. + let seconds = max(0, (deadline - ContinuousClock.now).milliseconds / 1_000) + cancellation.sleep(until: Date(timeIntervalSinceNow: seconds)) + } +} + +/// A clock that only moves when the runtime sleeps. +/// +/// `sleep(until:)` advances `now` to the deadline — plus `overshoot`, which +/// models a slow machine waking late. That is the one condition a real clock +/// cannot reproduce on demand, and it is exactly the condition that exposed the +/// timer-ordering bug: two timers with different delays both coming due in one +/// tick. +final class VirtualClock: RuntimeClock, @unchecked Sendable { + private let base = ContinuousClock.now + private let lock = NSLock() + private var offset: Duration = .zero + let overshoot: Duration + + init(overshoot: Duration = .zero) { + self.overshoot = overshoot + } + + var now: ContinuousClock.Instant { + base.advanced(by: elapsed) + } + + /// Total virtual time the runtime has slept through. + var elapsed: Duration { + lock.lock() + defer { lock.unlock() } + return offset + } + + func sleep(until deadline: ContinuousClock.Instant, cancellation: ExecutionCancellationController) { + // A cancelled sleep returns without advancing, as a real one would wake + // early; the caller's interrupt check then throws. + guard cancellation.isCancelled == false else { + return + } + lock.lock() + offset = max(offset, (deadline - base) + overshoot) + lock.unlock() + } +} + +extension Duration { + /// Whole and fractional milliseconds. `components` splits into seconds plus + /// attoseconds, which is otherwise an awkward two-term conversion. + var milliseconds: Double { + let parts = components + return Double(parts.seconds) * 1_000 + Double(parts.attoseconds) / 1e15 + } +} diff --git a/Sources/CodeMode/Runtime/RuntimeJavaScript.swift b/Sources/CodeMode/Runtime/RuntimeJavaScript.swift index f10ef84..03b88ba 100644 --- a/Sources/CodeMode/Runtime/RuntimeJavaScript.swift +++ b/Sources/CodeMode/Runtime/RuntimeJavaScript.swift @@ -1,6 +1,83 @@ import Foundation enum RuntimeJavaScript { + /// Shared by the execution and search runtimes. + private static let timerRuntime = """ + // A real timer queue, drained by the host between JavaScript turns. + // + // `setTimeout` used to invoke its callback synchronously and ignore the + // delay, which made three things wrong at once: `clearTimeout` could never + // cancel, an error thrown by the callback propagated to setTimeout's *caller* + // rather than being an uncaught task error, and + // `await new Promise(r => setTimeout(r, 2000))` — the standard backoff — was + // a hot loop that hammered remote APIs through `fetch`. + globalThis.__codemode.timers = { nextID: 1, entries: [] }; + + globalThis.setTimeout = function(fn, delay) { + if (typeof fn !== 'function') { + return 0; + } + const timers = globalThis.__codemode.timers; + const id = timers.nextID++; + timers.entries.push({ + id: id, + fn: fn, + args: Array.prototype.slice.call(arguments, 2), + dueIn: Math.max(0, Number(delay) || 0) + }); + return id; + }; + + globalThis.clearTimeout = function(id) { + const timers = globalThis.__codemode.timers; + timers.entries = timers.entries.filter(function(entry){ return entry.id !== id; }); + }; + + // Milliseconds until the earliest pending timer, or -1 when none are queued. + // -1 means nothing can advance the program: under this runtime's synchronous + // bridge there is no other source of asynchrony. + globalThis.__codemode.nextTimerDelay = function() { + const entries = globalThis.__codemode.timers.entries; + if (entries.length === 0) { + return -1; + } + return entries.reduce(function(min, entry){ return Math.min(min, entry.dueIn); }, Infinity); + }; + + globalThis.__codemode.advanceTimers = function(elapsedMs) { + const timers = globalThis.__codemode.timers; + const due = []; + const remaining = []; + timers.entries.forEach(function(entry){ + entry.dueIn -= elapsedMs; + if (entry.dueIn <= 0) { due.push(entry); } else { remaining.push(entry); } + }); + timers.entries = remaining; + // Due time first, registration order only to break ties — the order a + // real event loop fires in. Sorting by id alone was wrong whenever the + // host's sleep overshot far enough for two timers with *different* + // delays to come due in the same tick: the later one fired first purely + // because it was registered first. `dueIn` is now negative for anything + // overdue, so ascending puts the most overdue first. + due.sort(function(a, b){ + return a.dueIn === b.dueIn ? a.id - b.id : a.dueIn - b.dueIn; + }); + const errors = []; + due.forEach(function(entry){ + try { + entry.fn.apply(null, entry.args); + } catch (error) { + // An uncaught error in a task cannot propagate to whoever called + // setTimeout; the host reports it as a diagnostic instead. + errors.push(String(error)); + } + }); + // Returned rather than buffered, so the host needs one round trip per + // tick instead of two. + return errors; + }; + """ + static func pruningScript(removingJavaScriptNames names: some Sequence) -> String { let bindingsToRemove = Array(Set(names)).sorted() @@ -123,13 +200,7 @@ enum RuntimeJavaScript { error: function(){ __searchConsole('error', Array.from(arguments).map(function(v){ return String(v); }).join(' ')); } }; - globalThis.setTimeout = function(fn, delay) { - if (typeof fn === 'function') { - fn(); - } - return 0; - }; - globalThis.clearTimeout = function(_) {}; + \(timerRuntime) """ static let bootstrap = """ @@ -200,13 +271,7 @@ enum RuntimeJavaScript { error: function(){ __nativeConsoleLog(Array.from(arguments).map(function(v){ return String(v); }).join(' ')); } }; - globalThis.setTimeout = function(fn, delay) { - if (typeof fn === 'function') { - fn(); - } - return 0; - }; - globalThis.clearTimeout = function(_) {}; + \(timerRuntime) if (typeof URLSearchParams === 'undefined') { globalThis.URLSearchParams = function(initial){ @@ -258,16 +323,32 @@ enum RuntimeJavaScript { }; globalThis.apple = globalThis.apple || {}; + // These accept either the object form the catalog advertises — + // apple.keychain.get({ key }) — or the positional form the examples have + // always shown. The catalog described object arguments while the wrapper took + // a positional string, so code written from the metadata sent "[object + // Object]" as the key. + function __keychainArgs(first, value) { + if (first && typeof first === 'object') { + const args = { key: String(first.key) }; + if (first.value !== undefined) args.value = String(first.value); + return args; + } + const args = { key: String(first) }; + if (value !== undefined) args.value = String(value); + return args; + } + globalThis.apple.keychain = { - get: function(key) { return __invokeAsync('keychain.read', { key: String(key) }); }, - set: function(key, value) { return __invokeAsync('keychain.write', { key: String(key), value: String(value) }); }, - delete: function(key) { return __invokeAsync('keychain.delete', { key: String(key) }); } + get: function(key) { return __invokeAsync('keychain.read', __keychainArgs(key)); }, + set: function(key, value) { return __invokeAsync('keychain.write', __keychainArgs(key, value)); }, + delete: function(key) { return __invokeAsync('keychain.delete', __keychainArgs(key)); } }; globalThis.apple.location = { - getPermissionStatus: function() { return __invokeAsync('location.read', { mode: 'permissionStatus' }); }, + getPermissionStatus: function(args) { return __invokeAsync('location.read', Object.assign({ mode: 'permissionStatus' }, args || {})); }, requestPermission: function() { return __invokeAsync('location.permission.request', {}); }, - getCurrentPosition: function() { return __invokeAsync('location.read', { mode: 'current' }); } + getCurrentPosition: function(args) { return __invokeAsync('location.read', Object.assign({ mode: 'current' }, args || {})); } }; globalThis.apple.calendar = { @@ -353,6 +434,17 @@ enum RuntimeJavaScript { open: function(args) { return __invokeAsync('settings.ui.open', args || {}); } }; + // fs.move / fs.copy refuse an existing destination without an explicit + // overwrite, so the Node-style aliases need a way to pass one through. + function __fsDestinationArgs(from, to, options) { + const args = { from: String(from), to: String(to) }; + if (options && typeof options === 'object') { + if (options.overwrite !== undefined) args.overwrite = !!options.overwrite; + if (options.recursive !== undefined) args.recursive = !!options.recursive; + } + return args; + } + globalThis.fs = { promises: { readFile: function(path, options) { @@ -378,8 +470,12 @@ enum RuntimeJavaScript { rm: function(path, options) { return __invokeAsync('fs.delete', { path: String(path), recursive: !!(options && options.recursive) }); }, - rename: function(from, to) { return __invokeAsync('fs.move', { from: String(from), to: String(to) }); }, - copyFile: function(from, to) { return __invokeAsync('fs.copy', { from: String(from), to: String(to) }); } + rename: function(from, to, options) { + return __invokeAsync('fs.move', __fsDestinationArgs(from, to, options)); + }, + copyFile: function(from, to, options) { + return __invokeAsync('fs.copy', __fsDestinationArgs(from, to, options)); + } } }; diff --git a/Sources/CodeMode/Security/CapabilityGrant.swift b/Sources/CodeMode/Security/CapabilityGrant.swift new file mode 100644 index 0000000..d9315e9 --- /dev/null +++ b/Sources/CodeMode/Security/CapabilityGrant.swift @@ -0,0 +1,85 @@ +import Foundation + +/// The host-owned ceiling on what an execution may reach. +/// +/// `JavaScriptExecutionRequest.allowedCapabilities` / `allowedCapabilityKeys` are +/// *model-authored* — they travel in the advertised tool schema, so a script +/// declares them for itself. They are a least-privilege declaration of intent and +/// a useful audit signal, but they are not a sandbox: on their own, a script that +/// asks for `keychain.read` gets it. +/// +/// `CapabilityGrant` is the half the host owns. It is configured on +/// `CodeModeConfiguration` and intersected with the request before the invocation +/// context is built, so the effective set is always +/// `requested ∩ granted`. Anything the model asks for that the grant withholds +/// fails with `CAPABILITY_DENIED` and is reported to the model as *not* +/// repairable by retrying with a wider `allowedCapabilities`. +public struct CapabilityGrant: Sendable, Equatable { + /// Built-in capabilities the host permits, or `nil` for "no host ceiling". + public var capabilities: Set? + /// Custom provider capability keys the host permits, or `nil` for "no host ceiling". + public var capabilityKeys: Set? + + public init(capabilities: Set?, capabilityKeys: Set?) { + self.capabilities = capabilities + self.capabilityKeys = capabilityKeys + } + + /// No host ceiling: whatever the request declares is what the script gets. + /// + /// This is the default for source compatibility, and it is the right choice + /// only when the host has already vetted the request itself. Hosts that pipe + /// model-authored tool JSON straight into `executeJavaScript` should set an + /// explicit grant. + public static let unrestricted = CapabilityGrant(capabilities: nil, capabilityKeys: nil) + + /// Grants nothing; every capability request is withheld. + public static let none = CapabilityGrant(capabilities: [], capabilityKeys: []) + + /// Grants exactly the listed built-in capabilities and provider keys. + public static func only( + _ capabilities: Set, + capabilityKeys: Set = [] + ) -> CapabilityGrant { + CapabilityGrant(capabilities: capabilities, capabilityKeys: capabilityKeys) + } + + /// Whether the grant admits this capability. Used to keep what the host + /// advertises consistent with what it will actually allow. + public func permits(_ capability: CapabilityID) -> Bool { + capabilities?.contains(capability) ?? true + } + + public func permits(key: CodeModeCapabilityKey) -> Bool { + capabilityKeys?.contains(key) ?? true + } + + /// Applies the ceiling to a model-authored request. + /// + /// Returns the effective sets alongside everything that was asked for and + /// withheld, so the caller can tell the model why a retry will not help. + public func resolve( + requestedCapabilities: Set, + requestedCapabilityKeys: Set + ) -> Resolution { + let effectiveCapabilities = capabilities.map { requestedCapabilities.intersection($0) } ?? requestedCapabilities + let effectiveKeys = capabilityKeys.map { requestedCapabilityKeys.intersection($0) } ?? requestedCapabilityKeys + + return Resolution( + capabilities: effectiveCapabilities, + capabilityKeys: effectiveKeys, + withheld: ( + requestedCapabilities.subtracting(effectiveCapabilities).map(\.rawValue) + + requestedCapabilityKeys.subtracting(effectiveKeys).map(\.rawValue) + ).sorted() + ) + } + + public struct Resolution: Sendable, Equatable { + public var capabilities: Set + public var capabilityKeys: Set + /// Identifiers the request declared and the grant refused, sorted. Both + /// namespaces together, because every consumer reports them as one list. + public var withheld: [String] + } +} diff --git a/Sources/CodeMode/Security/NetworkAccessPolicy.swift b/Sources/CodeMode/Security/NetworkAccessPolicy.swift index 3dd10f8..6bb8e80 100644 --- a/Sources/CodeMode/Security/NetworkAccessPolicy.swift +++ b/Sources/CodeMode/Security/NetworkAccessPolicy.swift @@ -5,8 +5,11 @@ import Foundation /// /// The default (`.standard`) blocks loopback, private-range, link-local, and /// other non-public destinations so agent-authored JavaScript cannot reach -/// `127.0.0.1`, cloud metadata endpoints such as `169.254.169.254`, or hosts on -/// the local network, and caps buffered response bodies at 10 MB. +/// `127.0.0.1`, cloud metadata endpoints such as `169.254.169.254`, or names that +/// resolve onto the local network, and caps buffered response bodies at 10 MB. +/// "Names that resolve onto the local network" covers `localhost`, +/// `*.local`/`*.localhost`/`*.internal`, and bare single-label hosts such as +/// `router` or `intranet`, which reach the LAN through the DHCP search domain. /// /// Host matching is by URL host only; DNS resolution is not performed, so a /// public hostname that resolves to a private address is not detected. Hosts @@ -31,28 +34,64 @@ public struct NetworkAccessPolicy: Sendable, Equatable { /// that exceed this are cancelled mid-transfer. public var maxResponseBytes: Int + /// Whether script-supplied credential headers reach the wire. + /// + /// `Cookie`, `Authorization`, and the `Proxy-*` pair are how a request + /// carries authority. Script-authored code should not be minting them + /// against arbitrary origins, so they are refused by default. Hosts whose + /// scripts legitimately call an authenticated API — a bearer token the + /// script fetched from the keychain, say — set this to true. + public var allowsCredentialHeaders: Bool + + /// Request headers refused when `allowsCredentialHeaders` is false. + /// Matched case-insensitively; `proxy-` is matched as a prefix. + static let credentialHeaderNames: Set = [ + "cookie", + "cookie2", + "authorization", + ] + public init( allowedHosts: [String]? = nil, blockedHosts: [String] = [], blocksPrivateNetworks: Bool = true, - maxResponseBytes: Int = 10_485_760 + maxResponseBytes: Int = 10_485_760, + allowsCredentialHeaders: Bool = false ) { self.allowedHosts = allowedHosts self.blockedHosts = blockedHosts self.blocksPrivateNetworks = blocksPrivateNetworks self.maxResponseBytes = maxResponseBytes + self.allowsCredentialHeaders = allowsCredentialHeaders } - /// Secure default: private networks blocked, 10 MB response cap. + /// Secure default: private networks blocked, 10 MB response cap, no + /// script-supplied credential headers. public static let standard = NetworkAccessPolicy() - /// No destination restrictions and no response size cap. Matches the - /// pre-policy behavior of `network.fetch`; opt in deliberately. + /// No destination restrictions, no response size cap, and script-supplied + /// credential headers permitted. Matches the pre-policy behavior of + /// `network.fetch`; opt in deliberately. public static let permissive = NetworkAccessPolicy( blocksPrivateNetworks: false, - maxResponseBytes: .max + maxResponseBytes: .max, + allowsCredentialHeaders: true ) + /// Returns a reason the header is refused, or nil when it may be sent. + func headerViolationReason(for name: String) -> String? { + guard allowsCredentialHeaders == false else { + return nil + } + + let normalized = name.lowercased() + guard Self.credentialHeaderNames.contains(normalized) || normalized.hasPrefix("proxy-") else { + return nil + } + + return "the \"\(name)\" request header carries authority and is refused by the host app's network access policy" + } + /// Returns a human-readable reason the URL is refused, or nil when allowed. func violationReason(for url: URL) -> String? { guard let rawHost = url.host, rawHost.isEmpty == false else { @@ -119,6 +158,14 @@ public struct NetworkAccessPolicy: Sendable, Equatable { return isPrivateOrLocal(ipv6: ipv6) } + // A bare single-label name — `router`, `intranet`, `nas`, `wpad` — is not + // a public destination. It resolves through the DHCP-supplied search + // domain, so it lands on the local network exactly like `*.local` does, + // and it previously fell straight through to "allowed". + if host.contains(".") == false { + return true + } + return false } diff --git a/Sources/CodeMode/Security/PathPolicy.swift b/Sources/CodeMode/Security/PathPolicy.swift index 6ba31f8..35d0154 100644 --- a/Sources/CodeMode/Security/PathPolicy.swift +++ b/Sources/CodeMode/Security/PathPolicy.swift @@ -23,6 +23,26 @@ public struct PathPolicyConfig: Sendable { public protocol PathPolicy: Sendable { func resolve(path: String) throws -> URL + + /// The roots this policy admits. + /// + /// `resolve` accepts a root itself — `documents:` resolves to the documents + /// root — so destructive filesystem operations need to recognize one and + /// refuse it. Deliberately has no default: an earlier default of `[]` meant a + /// host with its own policy silently got none of the root protection on + /// `fs.move`, `fs.copy`, and `fs.delete`. A policy that genuinely has no + /// roots to protect can return `[]` explicitly. + var allowedRoots: [URL] { get } +} + +public extension PathPolicy { + /// True when `url` *is* one of the allowed roots rather than something inside one. + func isAllowedRoot(_ url: URL) -> Bool { + let candidate = url.standardizedFileURL.resolvingSymlinksInPath().standardizedFileURL.path + return allowedRoots.contains { root in + root.standardizedFileURL.resolvingSymlinksInPath().standardizedFileURL.path == candidate + } + } } public struct DefaultPathPolicy: PathPolicy { @@ -32,6 +52,10 @@ public struct DefaultPathPolicy: PathPolicy { self.config = config } + public var allowedRoots: [URL] { + [config.tmpRoot, config.cachesRoot, config.documentsRoot, config.appGroupRoot].compactMap { $0 } + } + public func resolve(path: String) throws -> URL { let cleaned = path.trimmingCharacters(in: .whitespacesAndNewlines) guard cleaned.isEmpty == false else { @@ -98,15 +122,54 @@ public struct DefaultPathPolicy: PathPolicy { var missingComponents: [String] = [] let fileManager = FileManager.default - while fileManager.fileExists(atPath: existingAncestor.path) == false, + // `fileExists` follows symlinks, so a symlink pointing at a *nonexistent* + // out-of-root target reported "missing" and was treated as a component to + // re-append rather than a link to resolve — admitting it. Checking for the + // link itself keeps a dangling symlink in the resolution path, where + // `resolvingSymlinksInPath` sends containment to the real target. + while Self.isMissingComponent(existingAncestor, fileManager: fileManager), existingAncestor.path != existingAncestor.deletingLastPathComponent().path { missingComponents.insert(existingAncestor.lastPathComponent, at: 0) existingAncestor.deleteLastPathComponent() } - let resolvedAncestor = existingAncestor.resolvingSymlinksInPath().standardizedFileURL + let resolvedAncestor = Self.followingDanglingSymlink( + existingAncestor.resolvingSymlinksInPath().standardizedFileURL, + fileManager: fileManager + ) return missingComponents.reduce(resolvedAncestor) { partial, component in partial.appendingPathComponent(component) }.standardizedFileURL } + + /// True when nothing exists at this path — not even a broken symlink. + /// `FileManager.fileExists` follows links and so answers false for one. + private static func isMissingComponent(_ url: URL, fileManager: FileManager) -> Bool { + if fileManager.fileExists(atPath: url.path) { + return false + } + // `attributesOfItem` does not traverse the final symlink, so a dangling + // link still has attributes here. + return (try? fileManager.attributesOfItem(atPath: url.path)) == nil + } + + /// Resolves a symlink whose target does not exist. + /// + /// `URL.resolvingSymlinksInPath()` only rewrites links it can follow, so a + /// link to a nonexistent out-of-root target came back unchanged and passed + /// containment. Containment must be judged on where the link *points*. + private static func followingDanglingSymlink(_ url: URL, fileManager: FileManager) -> URL { + var current = url + // Bounded: a symlink cycle would otherwise spin here. + for _ in 0..<16 { + guard let destination = try? fileManager.destinationOfSymbolicLink(atPath: current.path) else { + return current + } + let next = destination.hasPrefix("/") + ? URL(fileURLWithPath: destination) + : current.deletingLastPathComponent().appendingPathComponent(destination) + current = next.standardizedFileURL.resolvingSymlinksInPath().standardizedFileURL + } + return current + } } diff --git a/Sources/CodeMode/Security/SystemPermissionBroker.swift b/Sources/CodeMode/Security/SystemPermissionBroker.swift index 27f4451..27fd3f1 100644 --- a/Sources/CodeMode/Security/SystemPermissionBroker.swift +++ b/Sources/CodeMode/Security/SystemPermissionBroker.swift @@ -45,6 +45,18 @@ import HomeKit #endif public final class SystemPermissionBroker: PermissionBroker, @unchecked Sendable { + /// How long a `request(for:)` waits on the system's TCC dialog before giving + /// up and re-reading the status. This is a human-latency budget, and it is + /// short: a user who takes longer than this leaves the caller reading the + /// pre-prompt status. + static let permissionPromptTimeoutSeconds: TimeInterval = 10 + + /// Every TCC request in this type waits that same budget, so the timeout is + /// not repeated at six call sites where it could drift apart. + private static func awaitingPrompt(_ request: (@escaping @Sendable () -> Void) -> Void) { + CompletionWait.completion(timeout: permissionPromptTimeoutSeconds, request) + } + public init() {} public func status(for permission: PermissionKind) -> PermissionStatus { @@ -111,8 +123,15 @@ public final class SystemPermissionBroker: PermissionBroker, @unchecked Sendable private func locationStatus() -> PermissionStatus { #if canImport(CoreLocation) - let manager = CLLocationManager() - switch manager.authorizationStatus { + return Self.mapLocationAuthorization(CLLocationManager().authorizationStatus) + #else + return .unavailable + #endif + } + + #if canImport(CoreLocation) + static func mapLocationAuthorization(_ status: CLAuthorizationStatus) -> PermissionStatus { + switch status { case .authorizedWhenInUse, .authorizedAlways: return .granted case .denied: @@ -124,10 +143,8 @@ public final class SystemPermissionBroker: PermissionBroker, @unchecked Sendable @unknown default: return .unavailable } - #else - return .unavailable - #endif } + #endif private func contactsStatus() -> PermissionStatus { #if canImport(Contacts) @@ -320,9 +337,47 @@ public final class SystemPermissionBroker: PermissionBroker, @unchecked Sendable #endif } + /// Reads HomeKit authorization *without* constructing a manager. + /// + /// Constructing `HMHomeManager` is itself what triggers the HomeKit TCC + /// prompt, so the previous implementation put a user-facing dialog inside a + /// status check — and `requestHomeKitPermission` called it twice, then built a + /// third manager. A status query now answers from the shared manager if one + /// already exists and otherwise fails closed with `.notDetermined`, which + /// routes the caller through the request path where a prompt belongs. private func homeKitStatus() -> PermissionStatus { #if canImport(HomeKit) - let status = HMHomeManager().authorizationStatus + guard let manager = Self.existingHomeManager() else { + return .notDetermined + } + return Self.mapHomeKitAuthorization(manager.authorizationStatus) + #else + return .unavailable + #endif + } + + #if canImport(HomeKit) + /// Process-wide so repeated permission work reuses one manager instead of + /// re-triggering the prompt. + private static let homeManagerBox = LockedBox(nil) + + private static func existingHomeManager() -> HMHomeManager? { + homeManagerBox.get() + } + + /// Creates the shared manager if needed. This is the call that prompts. + private static func makeHomeManager(delegate: HMHomeManagerDelegate) -> HMHomeManager { + if let existing = homeManagerBox.get() { + existing.delegate = delegate + return existing + } + let manager = HMHomeManager() + manager.delegate = delegate + homeManagerBox.set(manager) + return manager + } + + private static func mapHomeKitAuthorization(_ status: HMHomeManagerAuthorizationStatus) -> PermissionStatus { if status.contains(.authorized) { return .granted } @@ -333,10 +388,8 @@ public final class SystemPermissionBroker: PermissionBroker, @unchecked Sendable return .notDetermined } return .unavailable - #else - return .unavailable - #endif } + #endif private func speechRecognitionStatus() -> PermissionStatus { #if canImport(Speech) @@ -419,24 +472,41 @@ public final class SystemPermissionBroker: PermissionBroker, @unchecked Sendable private func requestLocationPermission() -> PermissionStatus { #if canImport(CoreLocation) && os(iOS) - let manager = CLLocationManager() let delegate = LocationPermissionDelegate() - manager.delegate = delegate if Thread.isMainThread { + let manager = CLLocationManager() + manager.delegate = delegate + delegate.hold(manager) manager.requestWhenInUseAuthorization() + // Cannot block main for the callback; the caller sees the pre-prompt + // status and a later call observes the resolved one. return locationStatus() } - // async, not sync: execution runs on a background queue, and the host may be - // blocking the main thread waiting on the result — main.sync would deadlock. - // If main never runs the request, the delegate wait below times out instead. + // `CLLocationManager` delivers delegate callbacks on the run loop of the + // thread that *created* it, and the bridge's GCD workers have none — so a + // manager constructed here never called back, the wait below always burned + // its full 10s, and the result was reported from a status re-read that + // races the TCC write. Creating the manager, assigning the delegate, and + // requesting all happen on main. + // + // async, not sync: the host may be blocking main waiting on our result, so + // main.sync would deadlock. If main never runs this, the wait times out. DispatchQueue.main.async { + let manager = CLLocationManager() + manager.delegate = delegate + delegate.hold(manager) manager.requestWhenInUseAuthorization() } - _ = delegate.wait(timeout: 10) - return locationStatus() + guard delegate.wait(timeout: 10) == .success else { + return locationStatus() + } + + // Prefer the status the delegate callback carried over a fresh read, + // which can still observe the pre-prompt value. + return delegate.observedStatus().map(Self.mapLocationAuthorization) ?? locationStatus() #else return .unavailable #endif @@ -444,12 +514,10 @@ public final class SystemPermissionBroker: PermissionBroker, @unchecked Sendable private func requestContactsPermission() -> PermissionStatus { #if canImport(Contacts) - let semaphore = DispatchSemaphore(value: 0) let store = CNContactStore() - store.requestAccess(for: .contacts) { _, _ in - semaphore.signal() + Self.awaitingPrompt { complete in + store.requestAccess(for: .contacts) { _, _ in complete() } } - _ = semaphore.wait(timeout: .now() + 10) return contactsStatus() #else return .unavailable @@ -459,19 +527,13 @@ public final class SystemPermissionBroker: PermissionBroker, @unchecked Sendable private func requestCalendarPermission() -> PermissionStatus { #if canImport(EventKit) let store = EKEventStore() - let semaphore = DispatchSemaphore(value: 0) - - if #available(iOS 17.0, macOS 14.0, *) { - store.requestFullAccessToEvents { _, _ in - semaphore.signal() - } - } else { - store.requestAccess(to: .event) { _, _ in - semaphore.signal() + Self.awaitingPrompt { complete in + if #available(iOS 17.0, macOS 14.0, *) { + store.requestFullAccessToEvents { _, _ in complete() } + } else { + store.requestAccess(to: .event) { _, _ in complete() } } } - - _ = semaphore.wait(timeout: .now() + 10) return calendarReadStatus() #else return .unavailable @@ -481,19 +543,13 @@ public final class SystemPermissionBroker: PermissionBroker, @unchecked Sendable private func requestCalendarWritePermission() -> PermissionStatus { #if canImport(EventKit) let store = EKEventStore() - let semaphore = DispatchSemaphore(value: 0) - - if #available(iOS 17.0, macOS 14.0, *) { - store.requestWriteOnlyAccessToEvents { _, _ in - semaphore.signal() - } - } else { - store.requestAccess(to: .event) { _, _ in - semaphore.signal() + Self.awaitingPrompt { complete in + if #available(iOS 17.0, macOS 14.0, *) { + store.requestWriteOnlyAccessToEvents { _, _ in complete() } + } else { + store.requestAccess(to: .event) { _, _ in complete() } } } - - _ = semaphore.wait(timeout: .now() + 10) return calendarWriteStatus() #else return .unavailable @@ -503,19 +559,13 @@ public final class SystemPermissionBroker: PermissionBroker, @unchecked Sendable private func requestRemindersPermission() -> PermissionStatus { #if canImport(EventKit) let store = EKEventStore() - let semaphore = DispatchSemaphore(value: 0) - - if #available(iOS 17.0, macOS 14.0, *) { - store.requestFullAccessToReminders { _, _ in - semaphore.signal() - } - } else { - store.requestAccess(to: .reminder) { _, _ in - semaphore.signal() + Self.awaitingPrompt { complete in + if #available(iOS 17.0, macOS 14.0, *) { + store.requestFullAccessToReminders { _, _ in complete() } + } else { + store.requestAccess(to: .reminder) { _, _ in complete() } } } - - _ = semaphore.wait(timeout: .now() + 10) return remindersStatus() #else return .unavailable @@ -524,19 +574,13 @@ public final class SystemPermissionBroker: PermissionBroker, @unchecked Sendable private func requestPhotoLibraryPermission() -> PermissionStatus { #if canImport(Photos) - let semaphore = DispatchSemaphore(value: 0) - - if #available(iOS 14.0, macOS 11.0, *) { - PHPhotoLibrary.requestAuthorization(for: .readWrite) { _ in - semaphore.signal() - } - } else { - PHPhotoLibrary.requestAuthorization { _ in - semaphore.signal() + Self.awaitingPrompt { complete in + if #available(iOS 14.0, macOS 11.0, *) { + PHPhotoLibrary.requestAuthorization(for: .readWrite) { _ in complete() } + } else { + PHPhotoLibrary.requestAuthorization { _ in complete() } } } - - _ = semaphore.wait(timeout: .now() + 10) return photoLibraryStatus() #else return .unavailable @@ -545,13 +589,11 @@ public final class SystemPermissionBroker: PermissionBroker, @unchecked Sendable private func requestNotificationsPermission() -> PermissionStatus { #if canImport(UserNotifications) - let semaphore = DispatchSemaphore(value: 0) - - UNUserNotificationCenter.current().requestAuthorization(options: [.alert, .badge, .sound]) { _, _ in - semaphore.signal() + Self.awaitingPrompt { complete in + UNUserNotificationCenter.current().requestAuthorization(options: [.alert, .badge, .sound]) { _, _ in + complete() + } } - - _ = semaphore.wait(timeout: .now() + 10) return notificationsStatus() #else return .unavailable @@ -560,15 +602,17 @@ public final class SystemPermissionBroker: PermissionBroker, @unchecked Sendable private func requestHomeKitPermission() -> PermissionStatus { #if canImport(HomeKit) - if homeKitStatus() != .notDetermined { - return homeKitStatus() + // One status read, not two, and one manager, not three: each construction + // is a potential prompt. + let current = homeKitStatus() + if current != .notDetermined, Self.existingHomeManager() != nil { + return current } let delegate = HomeKitPermissionDelegate() - let manager = HMHomeManager() - manager.delegate = delegate - _ = delegate.wait(timeout: 10) - return homeKitStatus() + let manager = Self.makeHomeManager(delegate: delegate) + _ = delegate.wait(timeout: Self.permissionPromptTimeoutSeconds) + return Self.mapHomeKitAuthorization(manager.authorizationStatus) #else return .unavailable #endif @@ -667,13 +711,40 @@ public final class SystemPermissionBroker: PermissionBroker, @unchecked Sendable } #if canImport(CoreLocation) && os(iOS) -private final class LocationPermissionDelegate: NSObject, CLLocationManagerDelegate { +/// `@unchecked Sendable` because it is genuinely thread-safe and has to cross an +/// isolation boundary: the manager is created inside a `DispatchQueue.main.async` +/// block — main-actor-isolated on iOS — while `wait`/`observedStatus` are called +/// from the requesting thread. All mutable state is lock-guarded and the handoff +/// is a semaphore. +private final class LocationPermissionDelegate: NSObject, CLLocationManagerDelegate, @unchecked Sendable { private let semaphore = DispatchSemaphore(value: 0) + private let lock = NSLock() + private var status: CLAuthorizationStatus? + /// The manager is created inside a `DispatchQueue.main.async` block and would + /// otherwise be released as soon as that block returns — before the callback + /// it exists to receive. `CLLocationManager` does not retain its delegate, so + /// the ownership runs the other way here. + private var manager: CLLocationManager? + + func hold(_ manager: CLLocationManager) { + lock.lock() + self.manager = manager + lock.unlock() + } func locationManagerDidChangeAuthorization(_ manager: CLLocationManager) { + lock.lock() + status = manager.authorizationStatus + lock.unlock() semaphore.signal() } + func observedStatus() -> CLAuthorizationStatus? { + lock.lock() + defer { lock.unlock() } + return status + } + func wait(timeout: TimeInterval) -> DispatchTimeoutResult { semaphore.wait(timeout: .now() + timeout) } diff --git a/Sources/CodeMode/Support/CodeModeDate.swift b/Sources/CodeMode/Support/CodeModeDate.swift new file mode 100644 index 0000000..e3c89d7 --- /dev/null +++ b/Sources/CodeMode/Support/CodeModeDate.swift @@ -0,0 +1,58 @@ +import Foundation + +/// The single ISO 8601 parser for every bridge. +/// +/// Two spellings were previously in circulation: Health and Alarm tried +/// `.withFractionalSeconds` as a fallback, while EventKit, Notifications, and the +/// system-UI validators used a bare `ISO8601DateFormatter`. So +/// `2026-07-24T10:00:00.000Z` — a spelling models emit constantly, and what +/// `Date.toISOString()` produces in JavaScript — parsed in some bridges and not +/// others. +enum CodeModeDate { + /// Parses an ISO 8601 timestamp with or without fractional seconds. + static func parse(_ text: String?) -> Date? { + guard let text else { + return nil + } + + let trimmed = text.trimmingCharacters(in: .whitespacesAndNewlines) + guard trimmed.isEmpty == false else { + return nil + } + + let formatter = ISO8601DateFormatter() + if let date = formatter.date(from: trimmed) { + return date + } + + formatter.formatOptions = [.withInternetDateTime, .withFractionalSeconds] + return formatter.date(from: trimmed) + } + + /// Parses a timestamp that the caller requires, throwing a repairable error + /// rather than substituting a default. + /// + /// Silently falling back to "now" (or "now + 14 days") gave the script a + /// plausible but wrong window with no diagnostic — the worst failure mode + /// available, because the model has no way to notice it happened. + static func require(_ text: String?, argument: String, capability: String) throws -> Date { + guard let text, text.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty == false else { + throw BridgeError.invalidArguments("\(capability) requires '\(argument)'") + } + guard let date = parse(text) else { + throw BridgeError.invalidArguments( + "\(capability) could not parse '\(argument)' as an ISO 8601 timestamp: \"\(text)\". Use a form like 2026-07-24T10:00:00Z." + ) + } + return date + } + + /// Parses an optional timestamp, throwing when a value is present but + /// unparseable. A missing value returns nil and the caller's default applies. + static func optional(_ text: String?, argument: String, capability: String) throws -> Date? { + guard let text, text.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty == false else { + return nil + } + return try require(text, argument: argument, capability: capability) + } +} diff --git a/Sources/CodeMode/Support/CompletionWait.swift b/Sources/CodeMode/Support/CompletionWait.swift new file mode 100644 index 0000000..38f54f4 --- /dev/null +++ b/Sources/CodeMode/Support/CompletionWait.swift @@ -0,0 +1,55 @@ +import Foundation + +/// Bridges a callback-style system API into the bridges' synchronous world. +/// +/// The pattern this replaces — a `var` written from the framework's completion +/// queue, read after `_ = semaphore.wait(...)` with the timeout result +/// discarded — is a data race on timeout: the late callback writes while the +/// caller reads, and the caller cannot tell "timed out" from "no results". Every +/// wait here either produces the value the callback delivered or throws +/// `BridgeError.timeout`; the box stays alive for a late callback to write into +/// harmlessly. +enum CompletionWait { + /// Runs `operation`, which must call its `complete` argument exactly once, + /// and returns the delivered value. + /// + /// - Throws: `BridgeError.timeout` if `complete` is not called in time. + static func value( + timeout: TimeInterval, + operationName: String, + _ operation: (@escaping @Sendable (Value) -> Void) -> Void + ) throws -> Value { + let box = LockedBox(nil) + let semaphore = DispatchSemaphore(value: 0) + + operation { value in + box.set(value) + semaphore.signal() + } + + guard semaphore.wait(timeout: .now() + timeout) == .success else { + throw BridgeError.timeout(milliseconds: Int(timeout * 1_000)) + } + + guard let value = box.get() else { + throw BridgeError.nativeFailure("\(operationName) completed without a result") + } + + return value + } + + /// Waits for a completion that carries no value, for APIs whose result is + /// read back from the store afterwards. + /// + /// - Returns: true when the callback arrived, false on timeout. Callers that + /// must distinguish the two should use `value(timeout:operationName:_:)`. + @discardableResult + static func completion( + timeout: TimeInterval, + _ operation: (@escaping @Sendable () -> Void) -> Void + ) -> Bool { + let semaphore = DispatchSemaphore(value: 0) + operation { semaphore.signal() } + return semaphore.wait(timeout: .now() + timeout) == .success + } +} diff --git a/Sources/CodeMode/Support/JSONValue.swift b/Sources/CodeMode/Support/JSONValue.swift index d96eec3..0442e13 100644 --- a/Sources/CodeMode/Support/JSONValue.swift +++ b/Sources/CodeMode/Support/JSONValue.swift @@ -131,13 +131,34 @@ public extension JSONValue { return nil } + /// Integer view of a JSON number, or `nil` when the value cannot be + /// represented. + /// + /// `Int.init(_: Double)` *traps* on NaN, infinity, and anything outside + /// `Int`'s range, and every value here came from adversarial JSON — a model + /// tool call or a script's `fetch(url, { timeoutMs: 1e300 })`. The conversion + /// has to be total or a single number kills the host process. var intValue: Int? { if case let .number(value) = self { - return Int(value) + return JSONValue.exactInt(from: value) } return nil } + /// Total `Double` → `Int` conversion. Rejects non-finite values and anything + /// the conversion cannot represent; truncates toward zero otherwise, matching + /// the previous behavior for in-range input. + static func exactInt(from value: Double) -> Int? { + guard value.isFinite else { return nil } + let truncated = value.rounded(.towardZero) + guard truncated >= -9_223_372_036_854_775_808.0, + truncated < 9_223_372_036_854_775_808.0 + else { + return nil + } + return Int(truncated) + } + var doubleValue: Double? { if case let .number(value) = self { return value diff --git a/Sources/CodeMode/Support/UIKitSystemUIPresenter+Calendar.swift b/Sources/CodeMode/Support/UIKitSystemUIPresenter+Calendar.swift index c3e95b3..f4d8633 100644 --- a/Sources/CodeMode/Support/UIKitSystemUIPresenter+Calendar.swift +++ b/Sources/CodeMode/Support/UIKitSystemUIPresenter+Calendar.swift @@ -96,7 +96,6 @@ extension UIKitSystemUIPresenter { public func presentNewCalendarEvent(arguments: [String: JSONValue], context: BridgeInvocationContext) throws -> JSONValue { let timeoutMs = arguments.int("timeoutMs") ?? Self.defaultTimeoutMs - let formatter = ISO8601DateFormatter() let token = UUID() return try runUIOperation(timeoutMs: timeoutMs, onTimeout: { self.releaseCoordinator(token) }) { complete in @@ -108,9 +107,9 @@ extension UIKitSystemUIPresenter { event.notes = arguments.string("notes") event.location = arguments.string("location") - let startDate = arguments.string("start").flatMap { formatter.date(from: $0) } ?? Date() + let startDate = CodeModeDate.parse(arguments.string("start")) ?? Date() event.startDate = startDate - event.endDate = arguments.string("end").flatMap { formatter.date(from: $0) } + event.endDate = CodeModeDate.parse(arguments.string("end")) ?? Calendar.current.date(byAdding: .hour, value: 1, to: startDate) ?? startDate event.calendar = store.defaultCalendarForNewEvents diff --git a/Sources/CodeMode/Support/UIKitSystemUIPresenter+CommunicationWeb.swift b/Sources/CodeMode/Support/UIKitSystemUIPresenter+CommunicationWeb.swift index 8fd4a7c..e3a5f8b 100644 --- a/Sources/CodeMode/Support/UIKitSystemUIPresenter+CommunicationWeb.swift +++ b/Sources/CodeMode/Support/UIKitSystemUIPresenter+CommunicationWeb.swift @@ -365,7 +365,12 @@ extension UIKitSystemUIPresenter { public func presentWeb(arguments: [String: JSONValue], context: BridgeInvocationContext) throws -> JSONValue { let timeoutMs = arguments.int("timeoutMs") ?? Self.defaultTimeoutMs - let url = URL(string: arguments.string("url") ?? "")! + // The bridge parses the URL first, but UIKitSystemUIPresenter is public + // API and a host may call it directly — a force unwrap here crashes the + // app on a malformed string. + guard let url = URL(string: arguments.string("url") ?? "") else { + throw BridgeError.invalidArguments("web.ui.present requires an absolute 'url'") + } let token = UUID() #if os(iOS) @@ -402,7 +407,9 @@ extension UIKitSystemUIPresenter { public func authenticateWeb(arguments: [String: JSONValue], context: BridgeInvocationContext) throws -> JSONValue { let timeoutMs = arguments.int("timeoutMs") ?? Self.defaultTimeoutMs - let url = URL(string: arguments.string("url") ?? "")! + guard let url = URL(string: arguments.string("url") ?? "") else { + throw BridgeError.invalidArguments("auth.ui.webAuthenticate requires an absolute 'url'") + } let callbackURLScheme = arguments.string("callbackURLScheme") let token = UUID() @@ -446,7 +453,11 @@ extension UIKitSystemUIPresenter { } } session.presentationContextProvider = coordinator - session.prefersEphemeralWebBrowserSession = arguments.bool("prefersEphemeralSession") ?? false + // Ephemeral by default: a script-initiated OAuth flow inside the + // user's live Safari session starts out already signed in as the + // user and hands the callback URL — with its code or token — to + // the script. Opting out is a deliberate host/script choice. + session.prefersEphemeralWebBrowserSession = arguments.bool("prefersEphemeralSession") ?? true coordinator.session = session self.retainCoordinator(coordinator, token: token) diff --git a/Sources/CodeModeAuthoring/README.md b/Sources/CodeModeAuthoring/README.md index 57069fa..7a992e9 100644 --- a/Sources/CodeModeAuthoring/README.md +++ b/Sources/CodeModeAuthoring/README.md @@ -69,6 +69,33 @@ let call = try await tools.executeJavaScript( ) ``` +The two allowlists are strictly disjoint: a built-in capability ID listed in +`allowedCapabilityKeys` is ignored, and a provider key listed in +`allowedCapabilities` will not decode. Both fields are model-authored, so treat +them as a declaration of intent — set `CodeModeConfiguration.capabilityGrant` for +the host-owned ceiling that actually enforces access. + Macro v1 maps one type to one JavaScript function. `Arguments` must be a nested struct, and no-arg tools use an empty `Arguments` struct. `call(arguments:)` must be `throws` or `async throws`, and it can return `Void` or a nested `Result` struct. -Supported `Arguments` and `Result` property shapes are JSON primitives, `JSONValue`, arrays/dictionaries of JSON-shaped values, and optional forms. Throw `CodeModeFunctionError` for structured failures that should surface as CodeMode bridge errors. +Supported `Arguments` and `Result` property shapes are JSON primitives, `JSONValue`, arrays/dictionaries of JSON-shaped values, optional forms, and `CodeModeStringEnum` types. Throw `CodeModeFunctionError` for structured failures that should surface as CodeMode bridge errors. + +An enum-typed argument advertises its accepted values, so the catalog, the +generated TypeScript declaration, and argument validation all agree: + +```swift +enum Priority: String, CodeModeStringEnum { + case low, normal, high +} + +struct Arguments { + @CodeModeParam("Task identifier") + var id: String + @CodeModeParam("low, normal (default), or high") + var priority: Priority? +} +``` + +Providers do not have to be known at construction. `CodeModeAgentTools.register(provider:)` +adds them later — search advertises them immediately and the JavaScript bindings +are installed for the next execution — which is what a host needs when its domain +APIs depend on runtime state such as a signed-in account. diff --git a/Sources/CodeModeEvaluation/EvalModels.swift b/Sources/CodeModeEvaluation/EvalModels.swift index b8ae289..5511a4b 100644 --- a/Sources/CodeModeEvaluation/EvalModels.swift +++ b/Sources/CodeModeEvaluation/EvalModels.swift @@ -54,15 +54,39 @@ public struct CodeModeEvalExecuteStep: Codable, Sendable, Equatable { public var code: String public var allowedCapabilities: [CapabilityID] public var timeoutMs: Int? + /// Whether this step is *supposed* to fail. + /// + /// Repair scenarios deliberately call a helper wrong and then fix it from the + /// structured error, so their first step failing is the point. Every other + /// step failing should stop the run — otherwise a later step masks it and the + /// scenario passes on a broken transcript. + public var expectsFailure: Bool public init( code: String, allowedCapabilities: [CapabilityID] = [], - timeoutMs: Int? = nil + timeoutMs: Int? = nil, + expectsFailure: Bool = false ) { self.code = code self.allowedCapabilities = allowedCapabilities self.timeoutMs = timeoutMs + self.expectsFailure = expectsFailure + } + + private enum CodingKeys: String, CodingKey { + case code + case allowedCapabilities + case timeoutMs + case expectsFailure + } + + public init(from decoder: Decoder) throws { + let container = try decoder.container(keyedBy: CodingKeys.self) + self.code = try container.decode(String.self, forKey: .code) + self.allowedCapabilities = try container.decodeIfPresent([CapabilityID].self, forKey: .allowedCapabilities) ?? [] + self.timeoutMs = try container.decodeIfPresent(Int.self, forKey: .timeoutMs) + self.expectsFailure = try container.decodeIfPresent(Bool.self, forKey: .expectsFailure) ?? false } } diff --git a/Sources/CodeModeEvaluation/EvalRunner.swift b/Sources/CodeModeEvaluation/EvalRunner.swift index 30fba8a..2204a7f 100644 --- a/Sources/CodeModeEvaluation/EvalRunner.swift +++ b/Sources/CodeModeEvaluation/EvalRunner.swift @@ -58,6 +58,11 @@ public final class CodeModeEvalRunner: Sendable { searchDiagnostics = response.diagnostics } + // Logs and diagnostics accumulate across steps and an intermediate + // error stops the run. Overwriting each field per step meant + // validation only ever saw the last step: a scenario could pass while + // step 1 failed in a way step 2 masked, and step 1's logs — the + // evidence for why — were discarded. for executeStep in executeSteps(for: scenario) { toolCalls.append( CodeModeEvalToolCall( @@ -74,15 +79,25 @@ public final class CodeModeEvalRunner: Sendable { ) ) let observed = await observe(call) + // The graded value is the last step's output, and the graded error + // is the last step's error — so a successful repair step clears the + // deliberate failure that preceded it. executionOutput = observed.output - executionLogs = observed.logs - executionDiagnostics = observed.diagnostics observedError = observed.error + executionLogs.append(contentsOf: observed.logs) + executionDiagnostics.append(contentsOf: observed.diagnostics) + + // A repair scenario's first step is meant to fail and the next step + // fixes it. Anything else stops here rather than letting a later + // step mask the failure. + if observed.error != nil, executeStep.expectsFailure == false { + break + } } } catch let error as CodeModeToolError { observedError = error - executionLogs = error.logs - executionDiagnostics = error.diagnostics + executionLogs.append(contentsOf: error.logs) + executionDiagnostics.append(contentsOf: error.diagnostics) } catch { failures.append("Unexpected runner failure: \(error.localizedDescription)") } diff --git a/Sources/CodeModeEvaluation/EvalScenarios.swift b/Sources/CodeModeEvaluation/EvalScenarios.swift index 091e965..44359cb 100644 --- a/Sources/CodeModeEvaluation/EvalScenarios.swift +++ b/Sources/CodeModeEvaluation/EvalScenarios.swift @@ -16,6 +16,9 @@ public enum CodeModeEvalScenarios { filesystemCapabilityDenied, executionConsoleLogs, executionTimeout, + executionUnsettleablePromise, + executionTimerBackoff, + filesystemWholeJobInOneScript, reminderCatalogDiscovery, catalogFileSystemReadShape, catalogConsoleDiagnostics, @@ -319,10 +322,16 @@ public enum CodeModeEvalScenarios { return api.byJSName["apple.fs.read"]; } """, - executeCode: """ - return await apple.fs.read({ encoding: "utf8" }); - """, executeSteps: [ + // Deliberately wrong, so the next step can repair from the structured + // error. Marked so the runner does not treat it as the run failing. + CodeModeEvalExecuteStep( + code: """ + return await apple.fs.read({ encoding: "utf8" }); + """, + allowedCapabilities: [.fsRead], + expectsFailure: true + ), CodeModeEvalExecuteStep( code: """ const result = await apple.fs.read({ path: "tmp:repair.txt", encoding: "utf8" }); @@ -485,10 +494,10 @@ public enum CodeModeEvalScenarios { ) ) - public static let executionTimeout = CodeModeEvalScenario( - id: "execution.timeout", - title: "Unresolved promises time out", - task: "Await a never-resolving JavaScript promise. Do not catch the error in JavaScript; let executeJavaScript surface the structured execution-timeout error.", + public static let executionUnsettleablePromise = CodeModeEvalScenario( + id: "execution.unsettleable-promise", + title: "Unresolvable promises are reported, not waited out", + task: "Await a never-resolving JavaScript promise. Do not catch the error in JavaScript; let executeJavaScript surface the structured error.", executeCode: """ await new Promise(() => {}); return { never: true }; @@ -499,10 +508,93 @@ public enum CodeModeEvalScenarios { toolOrder: [.executeJavaScript], exactAllowedCapabilities: [], requiredExecuteCodeFragments: ["new Promise"], + expectedErrorCode: "JS_RUNTIME_ERROR" + ) + ) + + public static let executionTimeout = CodeModeEvalScenario( + id: "execution.timeout", + title: "CPU-bound scripts hit the execution timeout", + task: "Run a CPU-bound infinite loop. Do not catch the error in JavaScript; let executeJavaScript surface the structured execution-timeout error.", + executeCode: """ + while (true) {} + """, + allowedCapabilities: [], + timeoutMs: 200, + expectation: CodeModeEvalExpectation( + toolOrder: [.executeJavaScript], + exactAllowedCapabilities: [], + requiredExecuteCodeFragments: ["while"], expectedErrorCode: "EXECUTION_TIMEOUT" ) ) + public static let filesystemWholeJobInOneScript = CodeModeEvalScenario( + id: "fs.whole-job-one-script", + title: "A multi-step job runs as one script", + task: "Read every .json receipt in documents:receipts, total the amounts for trip 'lisbon', and return { total, count }. Do the whole job in a single executeJavaScript call — list, read, filter, and sum inside the script — and return only the totals, not the receipts.", + searchCode: """ + async () => { + return api.references + .filter(ref => ["fs.list", "fs.read"].includes(ref.capability)) + .map(ref => ref.dts) + .join("\\n\\n"); + } + """, + executeCode: """ + const entries = await apple.fs.list({ path: 'documents:receipts' }); + let total = 0; + let count = 0; + for (const entry of entries) { + if (entry.isDirectory || !entry.name.endsWith('.json')) continue; + const { text } = await apple.fs.read({ path: entry.path }); + const receipt = JSON.parse(text); + if (receipt.trip !== 'lisbon') continue; + total += receipt.amount; + count += 1; + } + return { total, count }; + """, + allowedCapabilities: [.fsList, .fsRead], + seedFiles: [ + CodeModeEvalSeedFile(path: "documents:receipts/a.json", text: #"{"trip":"lisbon","amount":12}"#), + CodeModeEvalSeedFile(path: "documents:receipts/b.json", text: #"{"trip":"porto","amount":99}"#), + CodeModeEvalSeedFile(path: "documents:receipts/c.json", text: #"{"trip":"lisbon","amount":30}"#), + ], + expectation: CodeModeEvalExpectation( + // One search, one execute. A transcript that splits the list/read/sum + // across several executions fails here — that is the whole point of + // the scenario, and it only has teeth against real LLM transcripts. + toolOrder: [.searchJavaScriptAPI, .executeJavaScript], + exactAllowedCapabilities: [.fsList, .fsRead], + forbiddenCapabilities: [.fsWrite, .fsDelete, .fsMove], + requiredExecuteCodeFragments: ["apple.fs.list", "apple.fs.read"], + expectedOutput: .object(["total": .number(42), "count": .number(2)]) + ) + ) + + public static let executionTimerBackoff = CodeModeEvalScenario( + id: "execution.timer-backoff", + title: "setTimeout honours its delay", + task: "Use setTimeout to wait before returning, and cancel a second timer with clearTimeout so its callback never runs.", + executeCode: """ + const marks = []; + const cancelled = setTimeout(() => { marks.push('cancelled'); }, 5); + clearTimeout(cancelled); + await new Promise(resolve => setTimeout(resolve, 20)); + marks.push('resumed'); + return { marks }; + """, + allowedCapabilities: [], + timeoutMs: 2_000, + expectation: CodeModeEvalExpectation( + toolOrder: [.executeJavaScript], + exactAllowedCapabilities: [], + requiredExecuteCodeFragments: ["setTimeout"], + expectedOutput: .object(["marks": .array([.string("resumed")])]) + ) + ) + public static let reminderCatalogDiscovery = CodeModeEvalScenario( id: "catalog.reminder-create", title: "Reminder helper discovery", diff --git a/Sources/CodeModeMacros/CodeModeMacro.swift b/Sources/CodeModeMacros/CodeModeMacro.swift index 72f6086..6d19b3f 100644 --- a/Sources/CodeModeMacros/CodeModeMacro.swift +++ b/Sources/CodeModeMacros/CodeModeMacro.swift @@ -116,12 +116,13 @@ public struct CodeModeMacro: ExtensionMacro { jsPath: \(literal(path)), title: \(literal(title(for: path))), summary: \(literal(summary)), - tags: [\(literal(parentPath(for: path)))], - example: "await \(path)({})", + tags: \(tagsSource(for: path)), + example: \(literal(example(for: path, fields: argumentFields.fields))), requiredArguments: [\(argumentFields.fields.filter { !$0.optional }.map { literal($0.name) }.joined(separator: ", "))], optionalArguments: [\(argumentFields.fields.filter(\.optional).map { literal($0.name) }.joined(separator: ", "))], argumentTypes: \(argumentTypesSource(for: argumentFields.fields)), argumentHints: \(argumentHintsSource(for: argumentFields.fields)), + argumentConstraints: \(argumentConstraintsSource(for: argumentFields.fields)), resultSummary: \(literal(returnsVoid ? "null" : resultInfo?.summary ?? "JSON value")), handler: { arguments, _ in try CodeModeAsyncBridge.run { @@ -275,7 +276,8 @@ public struct CodeModeMacro: ExtensionMacro { swiftType: typeInfo.swiftType, argumentType: typeInfo.argumentType, optional: typeInfo.optional, - description: attributeName.flatMap { attributeString(named: $0, attributes: variable.attributes).first } + description: attributeName.flatMap { attributeString(named: $0, attributes: variable.attributes).first }, + isStringEnum: typeInfo.isStringEnum ) ) } @@ -471,6 +473,60 @@ public struct CodeModeMacro: ExtensionMacro { return segments.dropLast().joined(separator: ".") } + /// Every path segment, so a helper is findable by its namespace *and* its + /// domain — the old single parent-path tag made `myapp.tasks.complete` + /// discoverable only by searching for "myapp.tasks". + private static func tagsSource(for path: String) -> String { + let segments = path.split(separator: ".").map(String.init) + var tags: [String] = [] + for segment in segments where tags.contains(segment) == false { + tags.append(segment) + } + if segments.count > 1 { + tags.append(segments.dropLast().joined(separator: ".")) + } + return "[" + tags.map(literal).joined(separator: ", ") + "]" + } + + /// A call the model can pattern-match, rather than the vacuous + /// `await path({})` that was emitted regardless of the arguments. + private static func example(for path: String, fields: [ToolField]) -> String { + let required = fields.filter { $0.optional == false } + guard required.isEmpty == false else { + return "await \(path)({})" + } + let pairs = required.map { field in + "\(field.name): \(placeholder(for: field))" + } + return "await \(path)({ " + pairs.joined(separator: ", ") + " })" + } + + private static func placeholder(for field: ToolField) -> String { + switch field.argumentType { + case "number": + return "0" + case "bool": + return "true" + case "array": + return "[]" + case "object": + return "{}" + default: + return "\"\(field.name)\"" + } + } + + private static func argumentConstraintsSource(for fields: [ToolField]) -> String { + let enumFields = fields.filter(\.isStringEnum) + guard enumFields.isEmpty == false else { + return "CapabilityArgumentConstraints.none" + } + let entries = enumFields.map { field in + "\(literal(field.name)): \(field.swiftType).codeModeAllowedValues" + }.joined(separator: ", ") + return "CapabilityArgumentConstraints(allowedStringValues: [\(entries)])" + } + private static func literal(_ value: String) -> String { let escaped = value .replacingOccurrences(of: "\\", with: "\\\\") @@ -511,6 +567,7 @@ private struct ToolField { var argumentType: String var optional: Bool var description: String? + var isStringEnum = false } private struct TypeInfo { @@ -518,6 +575,7 @@ private struct TypeInfo { var argumentType: String var optional: Bool var isSupported: Bool + var isStringEnum = false init(typeSyntax rawType: String) { let trimmed = rawType.replacingOccurrences(of: " ", with: "") @@ -554,6 +612,15 @@ private struct TypeInfo { if unwrapped.hasPrefix("["), unwrapped.hasSuffix("]") { self.argumentType = unwrapped.contains(":") ? "object" : "array" self.isSupported = Self.isSupportedCollection(unwrapped) + } else if Self.isSimpleIdentifier(unwrapped), Self.knownNonEnumTypes.contains(unwrapped) == false { + // An unrecognized simple type is taken to be a + // `CodeModeStringEnum`, the same assumption @BuiltInCodeMode + // makes. Host providers could not express constrained values at + // all before this, so their `allowedStringValues` was always + // empty while built-ins advertised theirs. + self.argumentType = "string" + self.isStringEnum = true + self.isSupported = true } else { self.argumentType = "any" self.isSupported = false @@ -561,6 +628,23 @@ private struct TypeInfo { } } + /// Types that are plainly not `CodeModeStringEnum`s. Without this list a + /// natural mistake such as `var date: Date` would be taken for an enum and + /// surface as a confusing error inside generated code instead of a macro + /// diagnostic pointing at the property. + private static let knownNonEnumTypes: Set = [ + "Date", "URL", "Data", "UUID", "Decimal", "Any", "AnyObject", "AnyHashable", + "Never", "Void", "Character", "Substring", "Int8", "Int16", "Int32", "Int64", + "UInt", "UInt8", "UInt16", "UInt32", "UInt64", "CGFloat", "NSNumber", "NSString", + ] + + private static func isSimpleIdentifier(_ type: String) -> Bool { + guard let first = type.first, first.isLetter || first == "_" else { + return false + } + return type.allSatisfy { $0.isLetter || $0.isNumber || $0 == "_" || $0 == "." } + } + private static func isSupportedCollection(_ type: String) -> Bool { let scalarTypes: Set = ["String", "Bool", "Int", "Double", "Float", "JSONValue"] let inner = String(type.dropFirst().dropLast()) diff --git a/TODO.md b/TODO.md index 282f76a..8b6aa01 100644 --- a/TODO.md +++ b/TODO.md @@ -37,11 +37,18 @@ just the watchdog: serialization, cancellation, catch-proof termination, context recovery). - [ ] **JS heap / memory cap** — nothing bounds `JSContext`/`JSContextGroup` heap; `new Array(1e9)` can still exhaust host memory. NOT addressed. -- [ ] **Bound on concurrent executions** — `executionQueue` is `.concurrent` - with spin-waiting workers (`BridgeRuntime.swift:25-29`), so N long-running - scripts occupy N threads. No concurrency cap. NOT addressed. -- [ ] `runOnExecutionQueue` still has no task-cancellation handler wired into the - dispatched block (best-effort only). NOT addressed. +- [x] **Bound on concurrent executions** — `ExecutionLimits.maxConcurrentExecutions` + (default 8) is enforced by the `ExecutionSlots` actor, acquired in the async + context *before* dispatch, so excess executions park as suspended tasks rather + than blocked GCD threads. Search has its own slots so discovery never waits + behind long scripts. Covered by `concurrentExecutionsAllCompleteAndStayIsolated`, + `executionsBeyondTheSlotLimitQueueRatherThanFail`, and + `searchDoesNotWaitBehindALongExecution`. +- [x] Cancellation reaches `runOnExecutionQueue` at every stage: a cancel while + waiting for a slot throws before anything is dispatched + (`cancellingWhileWaitingForASlotNeverRunsTheScript`), and once dispatched, + `cancel()`, a cancelled `call.result` await, and dropping both the call and its + events stream all set the cancellation controller the runtime polls. ### 2. `fetch` has no destination restrictions or size limits [x] - [x] `NetworkAccessPolicy` on `CodeModeConfiguration`: by default requests are @@ -227,9 +234,15 @@ tool model). - [x] `swift test` passes — 206 tests, 0 failures, ~5s. Note: the *first* verification run hung indefinitely and exposed the watchdog re-arm bug (see CRITICAL ISSUES §1); after the fix the suite is green. - - [ ] Watchdog terminates `while(true){}` on an iOS device/simulator — - verified on macOS only; JSC ships per-OS, so worth one manual check on a - simulator before tagging. + - [x] Watchdog terminates `while(true){}` on an iOS simulator — verified + 2026-10-01 on an iPhone 16 Pro simulator, iOS 18.6, by running the full + `CodeModeTests` target there (`xcodebuild test -scheme CodeMode-Package + -destination 'platform=iOS Simulator,…' -only-testing:CodeModeTests`). All + seven watchdog tests pass: CPU loop, loop inside a promise chain, + catch-proof termination, cancel mid-loop, runaway getter, serialization + budget, and search. Keychain storage tests skip there — an unsigned test + bundle has no keychain entitlement — and everything else passes. Still + simulator, not device. ## SUGGESTED ORDER OF ATTACK 1. Quick wins: tag `0.1.0`, fix calendar-span drift, add core-policy-layer tests, @@ -238,6 +251,84 @@ tool model). → "resource- and network-bounded runtime." 3. Metadata-consolidation refactor + macro decision, protected by new drift tests. +## REVIEW FOLLOW-UP — STILL OPEN +From `REVIEW.md`. Everything in P0 and P2 is done; these are the parts of the P1 +items that were not completed, kept explicit so they do not read as finished. + +### #9 — async bridge ABI (partially done) +Done: real timer queue (`setTimeout`/`clearTimeout` honour delays and cancel), +immediate diagnosis of unsettleable promises, `BRIDGE_FAILURES_NOT_SURFACED` for +swallowed rejections, bounded concurrent executions (actor slots, separate for +search), `ContinuousClock` deadlines behind an injectable `RuntimeClock`, and +cancellation once neither the call nor its events stream is reachable. +- [ ] **`CapabilityHandler` is still synchronous**, so `Promise.all([fetch(a), fetch(b)])` + is still serial. This is the change that requires resolving JS promises from + Swift via retained resolve/reject `JSValue`s on a per-execution serial executor. +- [ ] **Timeout/cancel still cannot interrupt an in-flight native call.** The + watchdog only traps while JS runs; `MediaBridge.transcode` can still overrun a + short `timeoutMs` by minutes. +- [ ] **`CodeModeAsyncBridge.run` still parks a semaphore** for up to 30s (longer + than the 10s default execution timeout) for async host tools. +- [ ] Optional: pool `JSVirtualMachine`s. Constraint to respect — the JSC time + limit is per context *group*, so a pooled VM must never host two concurrent + executions. + +### #11 — generate the JS bindings from the catalog +Done: the concrete drift found while generating declarations is fixed (keychain +object/positional, `fs.promises.rename`/`copyFile` options, location argument +pass-through, tool-description calling conventions). +- [ ] `RuntimeJavaScript.bootstrap` is still ~250 hand-written lines that can + drift from the catalog again. Wrappers that inject hidden arguments + (`createEvent` forces `operation: 'create'`) still cannot be expressed in + catalog metadata, so the generated `.d.ts` advertises an argument the wrapper + overrides. + +### #13 — provider ergonomics + MCP +Done: catalog unfrozen (post-init `register(provider:)` works and stays in sync), +enum constraints, real examples, per-segment tags. +- [ ] `@CodeModeProvider(namespace:)` on a type with `@CodeModeTool` on methods — + today it is still one struct per JavaScript function. +- [ ] `requiredPermissions` and typed results for provider registrations. +- [ ] MCP adapter materializing an MCP server's tools as `CodeModeRegistration`s + under `mcp..`. Blocked on async handlers (#9). + +### Test seams +Done: EventKit event/reminder serialization is now covered directly (the two +implicit-unwrap crashes), concurrent execution is covered, CI runners are pinned +to `macos-15` rather than `macos-latest`. `FetchTaskHandler`'s redirect +re-validation and size-cap cancellation turned out to be covered already +(`NetworkAccessPolicyTests`), contrary to the review. +- [ ] **Store protocols for EventKit/Health/Photos/Home/Contacts**, mirroring + `CodeModeFileSystem`. Their create/update/filter logic still executes nowhere: + the bridges hard-instantiate `EKEventStore()`, `HKHealthStore()`, `PHAsset`, so + tests can only reach permission denial and argument validation. +- [ ] A simulator test job, so UIKit presenters, AlarmKit, and the TCC-gated happy + paths are executed somewhere rather than only compiled. + +### Smaller P2 items not taken +- [ ] **No "prompt pending" permission status.** `request*` waits 10s and ignores + the timeout, so a user who takes 11s gets `PERMISSION_DENIED` while the dialog + is still on screen. Needs a new `PermissionStatus` case, which is source-breaking + for host code switching exhaustively. + +## DECLINED REVIEW FINDINGS +- **"Stop imposing swift-syntax on every consumer" (REVIEW.md P1 #12)** — + considered 2026-07-25 and declined. The proposal is to check in the expanded + `@BuiltInCodeMode` members so the `CodeMode` target can drop its + `CodeModeMacros` dependency. Declined because: + - The cost it cites is already measured and mitigated: prebuilt swift-syntax is + default-on for macros on the supported toolchains (`PLAN-registration-macros.md` + §"The swift-syntax cost, measured today"), so consumers pay ~0 incremental + build time, and the version range is already widened to `602.0.0..<604.0.0` + for `mlx-swift-lm` co-resolution — the one concrete conflict named. + - The migration is effectively one-way across 117 tool definitions and strips + the `@ToolParam("hint")` annotations off the property declarations, leaving + the hints readable only inside the generated argument arrays. + - It works against REVIEW #11 and #13, which both ask for *more* generation + from this metadata, not less. + Revisit if a consumer hits a swift-syntax version conflict the range cannot + absorb, or if prebuilts stop being default-on. + ## OPEN QUESTIONS FOR THE USER - [ ] Decision on the private-but-exported JSC symbol (App Store risk)? - [ ] After CI is green: tag `0.1.0` and restore the versioned install snippet? diff --git a/Tests/CodeModeAuthoringTests/CodeModeMacroExpansionTests.swift b/Tests/CodeModeAuthoringTests/CodeModeMacroExpansionTests.swift index a6bed25..513097c 100644 --- a/Tests/CodeModeAuthoringTests/CodeModeMacroExpansionTests.swift +++ b/Tests/CodeModeAuthoringTests/CodeModeMacroExpansionTests.swift @@ -69,7 +69,7 @@ private struct FailingTaskTool: CodeModeProvider { #expect(registration.jsPath == "myapp.tasks.complete") #expect(registration.title == "complete") #expect(registration.summary == "Mark a task complete.") - #expect(registration.tags == ["myapp.tasks"]) + #expect(registration.tags == ["myapp", "tasks", "complete", "myapp.tasks"]) #expect(registration.requiredArguments == ["id"]) #expect(registration.optionalArguments == ["note"]) #expect(registration.argumentTypes["id"] == .string) @@ -233,12 +233,13 @@ private func observe(_ call: JavaScriptExecutionCall) async -> ObservedExecution jsPath: "macro.api.greet", title: "greet", summary: "Greet a person.", - tags: ["macro.api"], - example: "await macro.api.greet({})", + tags: ["macro", "api", "greet", "macro.api"], + example: "await macro.api.greet({ name: \\"name\\" })", requiredArguments: ["name"], optionalArguments: ["excited"], argumentTypes: ["name": CapabilityArgumentType.string, "excited": CapabilityArgumentType.bool], argumentHints: ["name": "Person name", "excited": "Whether to add emphasis"], + argumentConstraints: CapabilityArgumentConstraints.none, resultSummary: "Greeting payload", handler: { arguments, _ in try CodeModeAsyncBridge.run { @@ -292,12 +293,13 @@ private func observe(_ call: JavaScriptExecutionCall) async -> ObservedExecution jsPath: "macro.api.ping", title: "ping", summary: "Ping.", - tags: ["macro.api"], + tags: ["macro", "api", "ping", "macro.api"], example: "await macro.api.ping({})", requiredArguments: [], optionalArguments: [], argumentTypes: [:], argumentHints: [:], + argumentConstraints: CapabilityArgumentConstraints.none, resultSummary: "null", handler: { arguments, _ in try CodeModeAsyncBridge.run { @@ -511,3 +513,76 @@ private func observe(_ call: JavaScriptExecutionCall) async -> ObservedExecution macros: codeModeTestMacros() ) } + +// MARK: - Constrained argument values for host providers + +/// Host providers could not express constrained values at all: the macro +/// rejected any non-primitive property type, so their `allowedStringValues` was +/// always empty while built-ins advertised theirs. This is the parity fix. +enum DemoPriority: String, CodeModeStringEnum { + case low + case normal + case high +} + +@CodeMode(path: "macro.api.prioritize", description: "Set a priority.") +struct PrioritizeTool: Sendable { + struct Arguments { + @CodeModeParam("Task identifier") + var id: String + @CodeModeParam("low, normal (default), or high") + var priority: DemoPriority? + } + + struct Result { + var id: String + var priority: String + } + + func call(arguments: Arguments) throws -> Result { + Result(id: arguments.id, priority: (arguments.priority ?? .normal).rawValue) + } +} + +@Test func codeModeMacroAdvertisesEnumConstraintsForHostProviders() throws { + let registration = try #require(PrioritizeTool().codeModeRegistrations().first) + + #expect(registration.argumentTypes["priority"] == .string) + #expect(registration.argumentConstraints.allowedStringValues["priority"] == ["low", "normal", "high"]) + // The generated example is a call the model can pattern-match, not the + // vacuous `await path({})` that used to be emitted regardless of arguments. + #expect(registration.example == #"await macro.api.prioritize({ id: "id" })"#) + // Tags cover every path segment, so the helper is findable by namespace and + // by name, not only by the full parent path. + #expect(registration.tags.contains("prioritize")) + #expect(registration.tags.contains("macro.api")) +} + +@Test func codeModeMacroRejectsValuesOutsideTheConstraint() async throws { + let tools = CodeModeAgentTools(config: CodeModeConfiguration(codeModeProviders: [PrioritizeTool()])) + + let denied = try await tools.executeJavaScript( + JavaScriptExecutionRequest( + code: #"return await macro.api.prioritize({ id: "1", priority: "urgent" });"#, + allowedCapabilities: [], + allowedCapabilityKeys: ["macro.api.prioritize"] + ) + ) + var deniedError: CodeModeToolError? + do { + _ = try await denied.result + } catch let error as CodeModeToolError { + deniedError = error + } + #expect(deniedError?.code == "INVALID_ARGUMENTS") + + let accepted = try await tools.executeJavaScript( + JavaScriptExecutionRequest( + code: #"return await macro.api.prioritize({ id: "1", priority: "high" });"#, + allowedCapabilities: [], + allowedCapabilityKeys: ["macro.api.prioritize"] + ) + ) + let result = try await accepted.result + #expect(result.output?.objectValue?["priority"] == .string("high")) +} diff --git a/Tests/CodeModeEvalTests/CodeModeEvalRunnerTests.swift b/Tests/CodeModeEvalTests/CodeModeEvalRunnerTests.swift index 340f01b..5580d09 100644 --- a/Tests/CodeModeEvalTests/CodeModeEvalRunnerTests.swift +++ b/Tests/CodeModeEvalTests/CodeModeEvalRunnerTests.swift @@ -142,3 +142,147 @@ private func failureSummary(_ results: [CodeModeEvalResult]) -> String { } .joined(separator: "\n") } + +// MARK: - Multi-step scenarios + +// The runner overwrote executionOutput/Logs/Diagnostics/observedError on every +// step and did not stop on an intermediate error, so validation saw only the last +// step: a scenario could pass while step 1 failed in a way step 2 masked, and +// step 1's logs — the evidence — were gone. + +@Test func multiStepScenariosAccumulateLogsAcrossSteps() async throws { + let scenario = CodeModeEvalScenario( + id: "multi-step-log-accumulation", + title: "Logs from every step survive", + task: "internal test", + executeSteps: [ + CodeModeEvalExecuteStep(code: "console.log('step-one-ran'); return 1;", allowedCapabilities: []), + CodeModeEvalExecuteStep(code: "console.log('step-two-ran'); return 2;", allowedCapabilities: []), + ], + expectation: CodeModeEvalExpectation( + toolOrder: [.executeJavaScript, .executeJavaScript], + exactAllowedCapabilities: [], + requiredExecutionLogFragments: ["step-one-ran", "step-two-ran"], + expectedOutput: .number(2) + ) + ) + + let result = await CodeModeEvalRunner().run(scenario) + #expect(result.passed, "\(result.failures)") + #expect(result.executionLogs.contains { $0.message.contains("step-one-ran") }) + #expect(result.executionLogs.contains { $0.message.contains("step-two-ran") }) + #expect(result.executionOutput == .number(2)) +} + +@Test func anIntermediateStepFailureStopsTheRunAndIsReported() async throws { + let scenario = CodeModeEvalScenario( + id: "multi-step-intermediate-failure", + title: "A failing first step is not masked by a passing second", + task: "internal test", + executeSteps: [ + // Denied: the step declares no capabilities. + CodeModeEvalExecuteStep(code: "return await apple.fs.read({ path: 'tmp:nope.txt' });", allowedCapabilities: []), + CodeModeEvalExecuteStep(code: "return 'second step succeeded';", allowedCapabilities: []), + ], + expectation: CodeModeEvalExpectation( + toolOrder: [.executeJavaScript], + exactAllowedCapabilities: [], + expectedErrorCode: "CAPABILITY_DENIED" + ) + ) + + let result = await CodeModeEvalRunner().run(scenario) + #expect(result.error?.code == "CAPABILITY_DENIED") + // The second step must not have run and must not have supplied the output. + #expect(result.toolCalls.filter { $0.tool == .executeJavaScript }.count == 1) + #expect(result.executionOutput != .string("second step succeeded")) +} + +// MARK: - Grading the one-script-per-job standard + +// The docs now tell agents to do a whole job in one executeJavaScript call. That +// is only a real standard if a transcript that ignores it fails, so this grades +// the shape rather than just asserting the reference solution. + +@Test func aTranscriptSplitAcrossExecutionsFailsTheOneScriptScenario() { + let scenario = CodeModeEvalScenarios.filesystemWholeJobInOneScript + + // The anti-pattern: list in one call, read in another, sum in a third — + // every intermediate result round-tripped through the model's context. + let split = [ + CodeModeEvalToolCall(tool: .searchJavaScriptAPI, code: scenario.searchCode ?? ""), + CodeModeEvalToolCall( + tool: .executeJavaScript, + code: "return await apple.fs.list({ path: 'documents:receipts' });", + allowedCapabilities: [.fsList] + ), + CodeModeEvalToolCall( + tool: .executeJavaScript, + code: "return await apple.fs.read({ path: 'documents:receipts/a.json' });", + allowedCapabilities: [.fsRead] + ), + CodeModeEvalToolCall( + tool: .executeJavaScript, + code: "return await apple.fs.read({ path: 'documents:receipts/c.json' });", + allowedCapabilities: [.fsRead] + ), + ] + + let failures = CodeModeEvalRunner().validateTranscript( + scenario: scenario, + toolCalls: split, + searchResult: .string("fs.list fs.read"), + executionOutput: .object(["total": .number(42), "count": .number(2)]), + error: nil + ) + + #expect(failures.contains { $0.contains("Tool order") }) +} + +@Test func theSingleScriptTranscriptPassesTheOneScriptScenario() { + let scenario = CodeModeEvalScenarios.filesystemWholeJobInOneScript + + let single = [ + CodeModeEvalToolCall(tool: .searchJavaScriptAPI, code: scenario.searchCode ?? ""), + CodeModeEvalToolCall( + tool: .executeJavaScript, + code: scenario.executeCode ?? "", + allowedCapabilities: [.fsList, .fsRead] + ), + ] + + let failures = CodeModeEvalRunner().validateTranscript( + scenario: scenario, + toolCalls: single, + searchResult: .string("fs.list fs.read"), + executionOutput: .object(["total": .number(42), "count": .number(2)]), + error: nil + ) + + #expect(failures.isEmpty, "\(failures)") +} + +@Test func anOverBroadCapabilityRequestFailsEvenWithTheRightAnswer() { + let scenario = CodeModeEvalScenarios.filesystemWholeJobInOneScript + + // Minimization is now stated plainly in the tool description, so grading it + // is no longer an unstated rule. + let overBroad = [ + CodeModeEvalToolCall(tool: .searchJavaScriptAPI, code: scenario.searchCode ?? ""), + CodeModeEvalToolCall( + tool: .executeJavaScript, + code: scenario.executeCode ?? "", + allowedCapabilities: [.fsList, .fsRead, .fsWrite] + ), + ] + + let failures = CodeModeEvalRunner().validateTranscript( + scenario: scenario, + toolCalls: overBroad, + searchResult: .string("fs.list fs.read"), + executionOutput: .object(["total": .number(42), "count": .number(2)]), + error: nil + ) + + #expect(failures.contains { $0.contains("Allowed capabilities") || $0.contains("Forbidden capabilities") }) +} diff --git a/Tests/CodeModeTests/AgentGuidanceTests.swift b/Tests/CodeModeTests/AgentGuidanceTests.swift new file mode 100644 index 0000000..9dbbd94 --- /dev/null +++ b/Tests/CodeModeTests/AgentGuidanceTests.swift @@ -0,0 +1,171 @@ +import Foundation +import Testing +@testable import CodeMode + +// The tool descriptions and the generated TypeScript teach an agent what the +// helpers are called. Nothing taught it what shape its work should take, so an +// agent could reasonably use executeJavaScript as a thin RPC — one call per +// operation — which is the pattern this package exists to replace. + +@Test func everyTierStatesTheCoreIdea() { + // A host dropping to a smaller tier must not lose a rule, only the examples. + for length in CodeModeAgentGuidance.Length.allCases { + let text = CodeModeAgentGuidance.systemPrompt(length) + #expect(text.contains("One `executeJavaScript` call should do the entire job")) + #expect(text.contains("Request only the capabilities your script actually calls")) + } +} + +@Test func tiersGrowStrictlyAndAreAdditive() { + let brief = CodeModeAgentGuidance.systemPrompt(.brief) + let standard = CodeModeAgentGuidance.systemPrompt(.standard) + let full = CodeModeAgentGuidance.systemPrompt(.full) + + #expect(brief.count < standard.count) + #expect(standard.count < full.count) + // Additive, not rewritten: the larger tiers contain the smaller ones verbatim. + #expect(standard.contains(brief)) + #expect(full.contains(standard)) +} + +@Test func largerTiersAddExamplesAndFailureModes() { + let standard = CodeModeAgentGuidance.systemPrompt(.standard) + let full = CodeModeAgentGuidance.systemPrompt(.full) + + #expect(standard.contains("```javascript")) + #expect(standard.contains("searchJavaScriptAPI")) + #expect(full.contains("Anti-patterns")) + #expect(full.contains("BRIDGE_FAILURES_NOT_SURFACED")) + // The concurrency reality must not be over-promised in the fan-out example. + #expect(full.contains("`Promise.all` does not make these overlap")) +} + +@Test func aBudgetSelectsTheLargestTierThatFits() { + let brief = CodeModeAgentGuidance.Length.brief.approximateTokenCount + let standard = CodeModeAgentGuidance.Length.standard.approximateTokenCount + let full = CodeModeAgentGuidance.Length.full.approximateTokenCount + + #expect(CodeModeAgentGuidance.length(forApproximateTokenBudget: full) == .full) + #expect(CodeModeAgentGuidance.length(forApproximateTokenBudget: full + 5_000) == .full) + #expect(CodeModeAgentGuidance.length(forApproximateTokenBudget: full - 1) == .standard) + #expect(CodeModeAgentGuidance.length(forApproximateTokenBudget: standard) == .standard) + #expect(CodeModeAgentGuidance.length(forApproximateTokenBudget: standard - 1) == .brief) + #expect(CodeModeAgentGuidance.length(forApproximateTokenBudget: brief) == .brief) +} + +@Test func aBudgetTooSmallForAnythingStillReturnsTheCoreIdea() { + // Deliberately a floor rather than an empty string: some guidance is the + // difference between an agent that batches and one that does not. + #expect(CodeModeAgentGuidance.systemPrompt(approximateTokenBudget: 0) == CodeModeAgentGuidance.systemPrompt(.brief)) + #expect(CodeModeAgentGuidance.systemPrompt(approximateTokenBudget: -100) == CodeModeAgentGuidance.systemPrompt(.brief)) +} + +@Test func guidanceExamplesUseHelpersAndCapabilitiesThatActuallyExist() async throws { + let (tools, sandbox) = try makeTools() + defer { cleanup(sandbox) } + + let full = CodeModeAgentGuidance.systemPrompt(.full) + let jsNames = Set(tools.capabilities().flatMap(\.jsNames)) + + // Guidance naming a helper the runtime does not install would teach the exact + // failure it is trying to prevent. + for helper in ["apple.fs.list", "apple.fs.read", "apple.fs.move"] { + #expect(full.contains(helper)) + #expect(jsNames.contains(helper), "guidance references missing helper \(helper)") + } + + for capability in ["fs.list", "fs.read", "fs.move"] { + #expect(CapabilityID(rawValue: capability) != nil, "guidance references missing capability \(capability)") + } +} + +@Test func theWorkedExampleActuallyRuns() async throws { + let (tools, sandbox) = try makeTools() + defer { cleanup(sandbox) } + + // The example is the shape we ask agents to copy, so it has to be a script + // this runtime accepts — arguments, calling conventions, and all. The setup + // lines are the only addition. + let observed = try await execute( + tools, + request: JavaScriptExecutionRequest( + code: """ + await apple.fs.mkdir({ path: 'documents:receipts/lisbon' }); + await apple.fs.write({ + path: 'documents:receipts/a.json', + data: JSON.stringify({ trip: 'lisbon', amount: 12 }) + }); + await apple.fs.write({ + path: 'documents:receipts/b.json', + data: JSON.stringify({ trip: 'porto', amount: 99 }) + }); + + const entries = await apple.fs.list({ path: 'documents:receipts' }); + let total = 0; + const filed = []; + + for (const entry of entries) { + if (entry.isDirectory || !entry.name.endsWith('.json')) continue; + const { text } = await apple.fs.read({ path: entry.path }); + const receipt = JSON.parse(text); + if (receipt.trip !== 'lisbon') continue; + total += receipt.amount; + await apple.fs.move({ + from: entry.path, + to: `documents:receipts/lisbon/${entry.name}`, + overwrite: true + }); + filed.push(entry.name); + } + + return { total, filedCount: filed.length }; + """, + allowedCapabilities: [.fsMkdir, .fsWrite, .fsList, .fsRead, .fsMove] + ) + ) + + #expect(observed.error == nil) + let output = try #require(observed.result?.output?.objectValue) + #expect(output.int("total") == 12) + #expect(output.int("filedCount") == 1) +} + +@Test func everyHelperCallInTheDocsNamesARealHelperAndRealArguments() async throws { + let (tools, sandbox) = try makeTools(hostPlatform: .iOS) + defer { cleanup(sandbox) } + + let references = tools.capabilities() + let byJSName = Dictionary(references.flatMap { ref in ref.jsNames.map { ($0, ref) } }, uniquingKeysWith: { first, _ in first }) + + // An example naming a helper or argument the runtime does not have teaches + // the exact failure the docs exist to prevent — and the wrong argument name + // is the easy mistake (`identifier` vs `identifiers` got through once). + // Parsed rather than listed, so new examples are checked automatically. + let texts = [CodeModeAgentToolDescriptions.executeJavaScript.description] + + CodeModeAgentGuidance.Length.allCases.map { CodeModeAgentGuidance.systemPrompt($0) } + let call = try Regex(#"((?:apple|ios)\.[A-Za-z.]+)\(\{([^}]*)\}"#) + let key = try Regex(#"(?:^|[,{\s])([A-Za-z_][A-Za-z0-9_]*)\s*(?=[:,}]|$)"#) + + var checked = 0 + for text in texts { + for match in text.matches(of: call) { + let helper = String(match.output[1].substring!) + let reference = try #require(byJSName[helper], "docs reference missing helper \(helper)") + let known = Set(reference.requiredArguments + reference.optionalArguments) + let body = String(match.output[2].substring!) + for argument in body.matches(of: key).map({ String($0.output[1].substring!) }) { + #expect(known.contains(argument), "\(helper) has no argument '\(argument)'") + } + checked += 1 + } + } + #expect(checked >= 5, "the parser found too few helper calls to be checking anything") +} + +@Test func theToolDescriptionStaysWithinItsBudget() { + // Sent on every turn and not trimmable by the host — unlike the guidance + // tiers, which exist precisely so worked examples can be budgeted. Examples + // belong there; this guard stops the description regrowing them. + #expect(CodeModeAgentToolDescriptions.executeJavaScript.description.count < 4_800) + #expect(CodeModeAgentToolDescriptions.executeJavaScript.description.contains("const results = []") == false) +} diff --git a/Tests/CodeModeTests/CodeModeProviderTests.swift b/Tests/CodeModeTests/CodeModeProviderTests.swift index 67387c9..1574388 100644 --- a/Tests/CodeModeTests/CodeModeProviderTests.swift +++ b/Tests/CodeModeTests/CodeModeProviderTests.swift @@ -578,7 +578,11 @@ private func jsonLiteral(_ value: JSONValue) -> String { #expect(result.bool("done") == true) } -@Test func builtInCapabilitiesCanBeAllowedByCapabilityKey() async throws { +// `allowedCapabilityKeys` accepts arbitrary strings and is not validated against +// `CapabilityID` at decode time, so it must never reach a built-in bridge — a host +// that vets only the strictly-typed `allowedCapabilities` would otherwise be +// bypassed by spelling the same ID in the loose field. +@Test func builtInCapabilitiesAreNotGrantedByCapabilityKey() async throws { let (tools, sandbox) = try makeTools() defer { cleanup(sandbox) } @@ -590,7 +594,83 @@ private func jsonLiteral(_ value: JSONValue) -> String { ) ) let observed = await observe(call) - #expect(observed.result?.output == .null) + let error = try #require(observed.error) + #expect(error.code == "CAPABILITY_DENIED") + #expect(error.capabilityKey == CapabilityID.keychainRead.codeModeKey) +} + +@Test func builtInCapabilitiesRemainGrantedByCapabilityID() async throws { + let (tools, sandbox) = try makeTools() + defer { cleanup(sandbox) } + + let call = try await tools.executeJavaScript( + JavaScriptExecutionRequest( + code: #"return await apple.keychain.get("missing-provider-key-test");"#, + allowedCapabilities: [.keychainRead] + ) + ) + let observed = await observe(call) + // The subject is capability gating, not keychain storage: reaching the + // bridge at all is the proof. Where the test host has no keychain + // entitlement the bridge then fails for that reason, which is still not a + // denial. + #expect(observed.error?.code != "CAPABILITY_DENIED") + if keychainIsAvailable { + #expect(observed.result?.output == .null) + } +} + +@Test func hostCapabilityGrantOverridesModelAuthoredAllowlist() async throws { + let (tools, sandbox) = try makeTools( + capabilityGrant: .only([.fsRead], capabilityKeys: []), + codeModeProviders: [ManualProvider()] + ) + defer { cleanup(sandbox) } + + // The script asks for keychain.read and a provider key; the host grants neither. + let call = try await tools.executeJavaScript( + JavaScriptExecutionRequest( + code: #"return await apple.keychain.get("anything");"#, + allowedCapabilities: [.keychainRead], + allowedCapabilityKeys: ["myapp.api.doTheThing"] + ) + ) + let observed = await observe(call) + let error = try #require(observed.error) + #expect(error.code == "CAPABILITY_DENIED") + #expect(error.suggestions.contains { $0.contains("host app's capability grant") }) + #expect( + error.diagnostics.contains { $0.code == "CAPABILITY_WITHHELD_BY_HOST" } + ) + + let providerCall = try await tools.executeJavaScript( + JavaScriptExecutionRequest( + code: #"return await myapp.api.doTheThing({ id: "123" });"#, + allowedCapabilities: [], + allowedCapabilityKeys: ["myapp.api.doTheThing"] + ) + ) + let providerObserved = await observe(providerCall) + #expect(providerObserved.error?.code == "CAPABILITY_DENIED") +} + +@Test func hostCapabilityGrantPermitsWhatItIncludes() async throws { + let (tools, sandbox) = try makeTools( + capabilityGrant: .only([], capabilityKeys: ["myapp.api.doTheThing"]), + codeModeProviders: [ManualProvider()] + ) + defer { cleanup(sandbox) } + + let call = try await tools.executeJavaScript( + JavaScriptExecutionRequest( + code: #"return await myapp.api.doTheThing({ id: "123" });"#, + allowedCapabilities: [], + allowedCapabilityKeys: ["myapp.api.doTheThing"] + ) + ) + let observed = await observe(call) + let result = try #require(observed.result?.output?.objectValue) + #expect(result.string("id") == "123") } @Test func customProviderFunctionErrorsRemainStructured() async throws { @@ -632,3 +712,78 @@ private func jsonLiteral(_ value: JSONValue) -> String { #expect(result.string("custom") == "myapp.api.doTheThing") #expect(result.string("builtIn") == CapabilityID.weatherRead.rawValue) } + +// MARK: - Post-init provider registration + +private struct LateProvider: CodeModeProvider { + let codeModePath = "myapp.late" + + func codeModeRegistrations() -> [CodeModeRegistration] { + [ + CodeModeRegistration( + capabilityKey: "myapp.late.echo", + jsPath: "myapp.late.echo", + title: "echo", + summary: "Echo a value registered after init.", + tags: ["myapp", "late"], + example: #"await myapp.late.echo({ value: "hi" })"#, + requiredArguments: ["value"], + argumentTypes: ["value": .string], + argumentHints: ["value": "Value to echo"], + resultSummary: "Echoed value" + ) { arguments, _ in + .object(["value": .string(try CodeModeArgumentDecoder.requireString("value", in: arguments))]) + }, + ] + } +} + +// The catalog used to be snapshotted in CodeModeAgentTools.init, so the +// registry's public post-init register(...) methods were unreachable in the +// supported flow — and would have desynced search from execution if reached. +@Test func providersRegisteredAfterInitAreExecutable() async throws { + let (tools, sandbox) = try makeTools() + defer { cleanup(sandbox) } + + let before = try await tools.executeJavaScript( + JavaScriptExecutionRequest( + code: #"return await myapp.late.echo({ value: "hi" });"#, + allowedCapabilities: [], + allowedCapabilityKeys: ["myapp.late.echo"] + ) + ) + // Not registered yet: the helper simply does not exist in the context. + let beforeObserved = await observe(before) + #expect(beforeObserved.error != nil) + + tools.register(provider: LateProvider()) + + let call = try await tools.executeJavaScript( + JavaScriptExecutionRequest( + code: #"return await myapp.late.echo({ value: "hi" });"#, + allowedCapabilities: [], + allowedCapabilityKeys: ["myapp.late.echo"] + ) + ) + let observed = await observe(call) + #expect(observed.result?.output?.objectValue?.string("value") == "hi") +} + +@Test func providersRegisteredAfterInitAppearInSearchAndTypeDeclarations() async throws { + let (tools, sandbox) = try makeTools() + defer { cleanup(sandbox) } + + #expect(tools.capabilities().contains { $0.capability == "myapp.late.echo" } == false) + + tools.register(provider: LateProvider()) + + // Search must not advertise anything execution cannot invoke, and vice versa. + let reference = try #require(tools.capabilities().first { $0.capability == "myapp.late.echo" }) + #expect(reference.dts.contains("value: string;")) + #expect(tools.typeDeclarations().contains("late: {")) + + let response = try await tools.searchJavaScriptAPI( + JavaScriptAPISearchRequest(code: #"async () => { return api.byJSName["myapp.late.echo"].summary; }"#) + ) + #expect(response.result == .string("Echo a value registered after init.")) +} diff --git a/Tests/CodeModeTests/CompletionWaitTests.swift b/Tests/CodeModeTests/CompletionWaitTests.swift new file mode 100644 index 0000000..450f9e4 --- /dev/null +++ b/Tests/CodeModeTests/CompletionWaitTests.swift @@ -0,0 +1,62 @@ +import Foundation +import Testing +@testable import CodeMode + +/// Serialized: these assert on a dispatched block getting a thread, which is exactly +/// what the watchdog and event-loop suites make scarce when run alongside them. +@Suite(.serialized) struct CompletionWaitTests { + + // The pattern this replaces — a `var` written from a framework's completion + // queue and read after a semaphore wait whose timeout result was discarded — is + // both a data race and an ambiguity: the caller could not tell "timed out" from + // "the API returned nothing". + + @Test func completionWaitReturnsTheDeliveredValue() throws { + // 60s, not 5: other suites block GCD workers for the length of a timer (the + // synchronous bridge model holds a thread per running execution), so even an + // immediate `global().async` can wait seconds for a thread on a loaded + // runner. The assertion is that the delivered value comes back, not how + // fast — the thread pressure itself is what the async ABI (#9) removes. + let value = try CompletionWait.value(timeout: 60, operationName: "test") { complete in + DispatchQueue.global().async { complete([1, 2, 3]) } + } + #expect(value == [1, 2, 3]) + } + + @Test func completionWaitThrowsRatherThanReturningAnEmptyResult() { + do { + // Never completes: the old shape returned `.array([])`, indistinguishable + // from a genuinely empty fetch. + let value: [Int] = try CompletionWait.value(timeout: 0.1, operationName: "reminders.read") { _ in } + Issue.record("Expected a timeout, got \(value)") + } catch { + #expect(requireBridgeErrorCode(error) == "EXECUTION_TIMEOUT") + } + } + + @Test func completionWaitToleratesALateCallback() throws { + // The late write must land in the box rather than racing a caller's read. + let lateCallbackFinished = DispatchSemaphore(value: 0) + do { + let _: Int = try CompletionWait.value(timeout: 0.05, operationName: "test") { complete in + DispatchQueue.global().asyncAfter(deadline: .now() + 0.2) { + complete(7) + lateCallbackFinished.signal() + } + } + Issue.record("Expected a timeout") + } catch { + #expect(requireBridgeErrorCode(error) == "EXECUTION_TIMEOUT") + } + + // Generous on purpose: other suites block GCD workers for the length of a + // timer, so a 0.2s dispatch can be starved for seconds on a loaded runner. + // The assertion is that the late callback lands at all. + #expect(lateCallbackFinished.wait(timeout: .now() + 60) == .success) + } + + @Test func completionWaitReportsWhetherTheCallbackArrived() { + #expect(CompletionWait.completion(timeout: 5) { complete in complete() }) + #expect(CompletionWait.completion(timeout: 0.05) { _ in } == false) + } +} diff --git a/Tests/CodeModeTests/DefensiveDecodingTests.swift b/Tests/CodeModeTests/DefensiveDecodingTests.swift index 8a4485c..e9ede14 100644 --- a/Tests/CodeModeTests/DefensiveDecodingTests.swift +++ b/Tests/CodeModeTests/DefensiveDecodingTests.swift @@ -56,6 +56,50 @@ private func decodeRequest(_ json: String) throws -> JavaScriptExecutionRequest } } +// MARK: - Adversarial numbers must not trap + +@Test func executionRequestRejectsNonRepresentableTimeout() throws { + // `Int.init(_: Double)` traps on these. This decode path runs on + // model-authored JSON before any script does, so a trap here is an + // in-process kill of the host app. + for literal in ["1e300", "-1e300", "1e400", "-1e400"] { + #expect(throws: (any Error).self) { + _ = try decodeRequest(#"{"code":"return 1;","allowedCapabilities":[],"timeoutMs":\#(literal)}"#) + } + #expect(throws: (any Error).self) { + _ = try decodeRequest(#"{"code":"return 1;","allowedCapabilities":[],"timeoutMs":"\#(literal)"}"#) + } + } + + for literal in ["inf", "-inf", "infinity", "nan", "NaN"] { + #expect(throws: (any Error).self) { + _ = try decodeRequest(#"{"code":"return 1;","allowedCapabilities":[],"timeoutMs":"\#(literal)"}"#) + } + } +} + +@Test func executionRequestClampsTimeoutToAdvertisedBounds() throws { + // The schema advertises 1...60000; out-of-range values are clamped rather + // than failing the call. + #expect(try decodeRequest(#"{"code":"return 1;","allowedCapabilities":[],"timeoutMs":600000}"#).timeoutMs == 60_000) + #expect(try decodeRequest(#"{"code":"return 1;","allowedCapabilities":[],"timeoutMs":0}"#).timeoutMs == 1) + #expect(try decodeRequest(#"{"code":"return 1;","allowedCapabilities":[],"timeoutMs":-5}"#).timeoutMs == 1) + #expect(JavaScriptExecutionRequest(code: "", allowedCapabilities: [], timeoutMs: 10_000_000).timeoutMs == 60_000) +} + +@Test func jsonValueIntConversionIsTotal() { + #expect(JSONValue.number(.infinity).intValue == nil) + #expect(JSONValue.number(-.infinity).intValue == nil) + #expect(JSONValue.number(.nan).intValue == nil) + #expect(JSONValue.number(1e300).intValue == nil) + #expect(JSONValue.number(-1e300).intValue == nil) + #expect(JSONValue.number(9.3e18).intValue == nil) + #expect(JSONValue.number(42).intValue == 42) + #expect(JSONValue.number(-42.9).intValue == -42) + #expect(JSONValue.number(Double(Int.max)).intValue == nil) + #expect(JSONValue.number(Double(Int.min)).intValue == Int.min) +} + @Test func executionRequestRoundTripsThroughCodable() throws { let original = JavaScriptExecutionRequest( code: "return 1;", @@ -118,6 +162,38 @@ private func makeEchoRegistry(argumentTypes: [String: CapabilityArgumentType]) - } } +@Test func registryRejectsNonFiniteDeclaredNumbers() throws { + let registry = makeEchoRegistry(argumentTypes: ["timeoutMs": .number]) + let (context, sandbox) = try makeInvocationContext(allowedCapabilities: [.fsRead]) + defer { cleanup(sandbox) } + + // Straight through as a JSON number... + #expect(throws: (any Error).self) { + _ = try registry.invoke( + CapabilityID.fsRead.rawValue, + arguments: ["timeoutMs": .number(.infinity)], + context: context + ) + } + // ...and smuggled through string coercion, which `Double.init(String)` accepts. + for spelling in ["inf", "-infinity", "nan"] { + #expect(throws: (any Error).self) { + _ = try registry.invoke( + CapabilityID.fsRead.rawValue, + arguments: ["timeoutMs": .string(spelling)], + context: context + ) + } + } + + let ok = try registry.invoke( + CapabilityID.fsRead.rawValue, + arguments: ["timeoutMs": .string("2500")], + context: context + ) + #expect(ok.objectValue?["timeoutMs"] == .number(2_500)) +} + @Test func registryDoesNotCoerceNonBooleanStringsOrBadNumbers() throws { let registry = makeEchoRegistry(argumentTypes: ["count": .number, "flag": .bool]) let (context, sandbox) = try makeInvocationContext(allowedCapabilities: [.fsRead]) diff --git a/Tests/CodeModeTests/EventKitBridgeTests.swift b/Tests/CodeModeTests/EventKitBridgeTests.swift index 6fb95b9..da6c32a 100644 --- a/Tests/CodeModeTests/EventKitBridgeTests.swift +++ b/Tests/CodeModeTests/EventKitBridgeTests.swift @@ -135,3 +135,44 @@ import Testing #expect(observed.error?.code == "PERMISSION_DENIED") } + +// MARK: - Serialization of degenerate items + +#if canImport(EventKit) +import EventKit + +// EventKit's serialization logic executes nowhere today — not in unit tests, not +// in CI, not in evals — because the bridge hard-instantiates EKEventStore and the +// tests can only reach permission denial. These construct the model objects +// directly, which needs no permission, and cover the two implicitly-unwrapped +// fields that crashed the execution thread. + +@Test func eventSerializationSurvivesAnEventWithNoCalendar() throws { + let store = EKEventStore() + let event = EKEvent(eventStore: store) + event.title = "Orphaned" + event.startDate = Date(timeIntervalSince1970: 1_700_000_000) + event.endDate = event.startDate.addingTimeInterval(3_600) + + // `EKEvent.calendar` is `EKCalendar!` and is nil for an orphaned event, while + // every neighbouring field was nil-coalesced. + #expect(event.calendar == nil) + + let json = try #require(EventKitBridge.eventJSONForTesting(event).objectValue) + #expect(json.string("title") == "Orphaned") + #expect(json.string("calendarIdentifier") == "") + #expect(json.string("calendarTitle") == "") + #expect(json.string("notes") == "") +} + +@Test func reminderSerializationSurvivesAMissingTitle() throws { + let store = EKEventStore() + let reminder = EKReminder(eventStore: store) + + // `EKReminder.title` is `String!`. + let json = try #require(EventKitBridge.reminderJSONForTesting(reminder).objectValue) + #expect(json.string("title") == "") + #expect(json.string("calendarIdentifier") == "") + #expect(json.bool("isCompleted") == false) +} +#endif diff --git a/Tests/CodeModeTests/EventLoopTests.swift b/Tests/CodeModeTests/EventLoopTests.swift new file mode 100644 index 0000000..09886c2 --- /dev/null +++ b/Tests/CodeModeTests/EventLoopTests.swift @@ -0,0 +1,572 @@ +import Foundation +import Testing +@testable import CodeMode + +/// Serialized: most of these hold a GCD worker for the length of a timer, and several +/// pin a slot deliberately. Run concurrently with each other on a small CI +/// runner they starve the global pool — and each other — for tens of seconds. +@Suite(.serialized) struct EventLoopTests { + + // `setTimeout` used to invoke its callback synchronously and ignore the delay. + // Three things were wrong at once: clearTimeout could never cancel, an error + // thrown by a callback propagated to setTimeout's *caller*, and + // `await new Promise(r => setTimeout(r, 2000))` — the standard backoff — was a + // hot loop that hammered remote APIs through fetch. + + @Test func setTimeoutDefersItsCallbackInsteadOfRunningItInline() async throws { + let (tools, sandbox) = try makeTools(clock: VirtualClock()) + defer { cleanup(sandbox) } + + let observed = try await execute( + tools, + request: JavaScriptExecutionRequest( + code: """ + const order = []; + setTimeout(() => { order.push('timer'); }, 0); + order.push('sync'); + await new Promise(resolve => setTimeout(resolve, 1)); + return { order }; + """, + allowedCapabilities: [] + ) + ) + + #expect(observed.error == nil) + let order = try #require(observed.result?.output?.objectValue?.array("order")) + #expect(order == [.string("sync"), .string("timer")]) + } + + @Test func setTimeoutActuallyWaitsTheRequestedDelay() async throws { + let clock = VirtualClock() + let (tools, sandbox) = try makeTools(clock: clock) + defer { cleanup(sandbox) } + + let observed = try await execute( + tools, + request: JavaScriptExecutionRequest( + code: "await new Promise(resolve => setTimeout(resolve, 150)); return 'done';", + allowedCapabilities: [], + timeoutMs: 5_000 + ) + ) + + #expect(observed.result?.output == .string("done")) + // The delay was previously ignored entirely, so a backoff loop spun. Under + // a virtual clock the assertion is exact: the runtime slept for precisely + // the delay it was asked for — not "at least" it, and not a wall-clock + // bound that a loaded runner could stretch. + #expect(clock.elapsed == .milliseconds(150)) + } + + @Test func clearTimeoutCancelsAPendingCallback() async throws { + let (tools, sandbox) = try makeTools(clock: VirtualClock()) + defer { cleanup(sandbox) } + + let observed = try await execute( + tools, + request: JavaScriptExecutionRequest( + code: """ + const fired = []; + const id = setTimeout(() => { fired.push('should-not-run'); }, 5); + clearTimeout(id); + await new Promise(resolve => setTimeout(resolve, 20)); + return { fired }; + """, + allowedCapabilities: [] + ) + ) + + #expect(observed.result?.output?.objectValue?.array("fired") == []) + } + + @Test func timersFireInDueOrderNotRegistrationOrder() async throws { + let (tools, sandbox) = try makeTools(clock: VirtualClock()) + defer { cleanup(sandbox) } + + // Registered late-first, so registration order and due order disagree. This + // must hold however far the host's sleep overshoots — if enough time passes + // for both to come due in one tick, they still fire shortest-delay first. + let observed = try await execute( + tools, + request: JavaScriptExecutionRequest( + code: """ + const order = []; + setTimeout(() => order.push('b'), 10); + setTimeout(() => order.push('a'), 5); + await new Promise(resolve => setTimeout(resolve, 30)); + return { order }; + """, + allowedCapabilities: [] + ) + ) + + #expect(observed.result?.output?.objectValue?.array("order") == [.string("a"), .string("b")]) + } + + @Test func timersComingDueInOneTickStillFireInDueOrder() async throws { + // A slow machine wakes late. Modelled exactly: every sleep overshoots by + // 10ms, so the first wake (aimed at 5ms) lands at 15ms with *both* timers + // due in a single advanceTimers call — the case that sorting by + // registration id alone got wrong. Registered late-first so the two + // orderings disagree. Before the clock seam this condition could only be + // reproduced by a loaded CI runner. + let clock = VirtualClock(overshoot: .milliseconds(10)) + let (tools, sandbox) = try makeTools(clock: clock) + defer { cleanup(sandbox) } + + let observed = try await execute( + tools, + request: JavaScriptExecutionRequest( + code: """ + const order = []; + setTimeout(() => order.push('later'), 10); + setTimeout(() => order.push('sooner'), 5); + await new Promise(resolve => setTimeout(resolve, 40)); + return { order }; + """, + allowedCapabilities: [] + ) + ) + + #expect(observed.result?.output?.objectValue?.array("order") == [.string("sooner"), .string("later")]) + } + + @Test func timersDueAtTheSameInstantFireInRegistrationOrder() async throws { + let (tools, sandbox) = try makeTools(clock: VirtualClock()) + defer { cleanup(sandbox) } + + // Equal delays, so only registration order can break the tie. + let observed = try await execute( + tools, + request: JavaScriptExecutionRequest( + code: """ + const order = []; + setTimeout(() => order.push('first'), 5); + setTimeout(() => order.push('second'), 5); + setTimeout(() => order.push('third'), 5); + await new Promise(resolve => setTimeout(resolve, 30)); + return { order }; + """, + allowedCapabilities: [] + ) + ) + + #expect( + observed.result?.output?.objectValue?.array("order") + == [.string("first"), .string("second"), .string("third")] + ) + } + + @Test func anErrorInATimerCallbackBecomesADiagnosticNotACallerThrow() async throws { + let (tools, sandbox) = try makeTools(clock: VirtualClock()) + defer { cleanup(sandbox) } + + let observed = try await execute( + tools, + request: JavaScriptExecutionRequest( + code: """ + setTimeout(() => { throw new Error('boom'); }, 1); + await new Promise(resolve => setTimeout(resolve, 20)); + return 'survived'; + """, + allowedCapabilities: [] + ) + ) + + // Previously this propagated to whoever called setTimeout, which is not how + // an event loop behaves. + #expect(observed.result?.output == .string("survived")) + let diagnostics = try #require(observed.result?.diagnostics) + #expect(diagnostics.contains { $0.code == "TIMER_CALLBACK_ERROR" && $0.message.contains("boom") }) + } + + @Test func anUnsettleablePromiseIsReportedImmediately() async throws { + let clock = VirtualClock() + let (tools, sandbox) = try makeTools(clock: clock) + defer { cleanup(sandbox) } + + // No resolve path and no queued timer: nothing in this runtime can advance + // it, so sleeping to the deadline buys nothing. + let observed = try await execute( + tools, + request: JavaScriptExecutionRequest( + code: "await new Promise(() => {}); return 1;", + allowedCapabilities: [], + timeoutMs: 30_000 + ) + ) + + #expect(observed.error?.code == "JS_RUNTIME_ERROR") + #expect(observed.error?.message.contains("can never settle") == true) + // "Immediately" means the runtime never slept at all. + #expect(clock.elapsed == .zero) + } + + @Test func aTimerStillPendingAtTheDeadlineTimesOut() async throws { + let clock = VirtualClock() + let (tools, sandbox) = try makeTools(clock: clock) + defer { cleanup(sandbox) } + + // A timer due long after the deadline must not be waited out. + let observed = try await execute( + tools, + request: JavaScriptExecutionRequest( + code: "await new Promise(resolve => setTimeout(resolve, 30000)); return 1;", + allowedCapabilities: [], + timeoutMs: 100 + ) + ) + + #expect(observed.error?.code == "EXECUTION_TIMEOUT") + // Stopped exactly at the 100ms deadline, never anywhere near the 30s timer. + #expect(clock.elapsed == .milliseconds(100)) + } + + @Test func aLongBackoffStaysCancellable() async throws { + let (tools, sandbox) = try makeTools() + defer { cleanup(sandbox) } + + let call = try await tools.executeJavaScript( + JavaScriptExecutionRequest( + code: "await new Promise(resolve => setTimeout(resolve, 20000)); return 1;", + allowedCapabilities: [], + timeoutMs: 60_000 + ) + ) + + Task { + try? await Task.sleep(nanoseconds: 100_000_000) + call.cancel() + } + + let started = Date() + let observed = await observe(call) + #expect(observed.error?.code == "CANCELLED") + // Proves cancel interrupted the 20s timer rather than waiting it out. + #expect(Date().timeIntervalSince(started) < 15) + } + + // MARK: - Swallowed bridge failures + + @Test func aForgottenAwaitOnAFailedCallIsSurfacedAsADiagnostic() async throws { + let (tools, sandbox) = try makeTools() + defer { cleanup(sandbox) } + + // Nothing installs an unhandled-rejection hook, so this used to complete with + // the agent told "done" while a CAPABILITY_DENIED was silently discarded. + let observed = try await execute( + tools, + request: JavaScriptExecutionRequest( + code: """ + apple.keychain.get({ key: 'never-allowed' }).catch(() => {}); + await new Promise(resolve => setTimeout(resolve, 10)); + return 'looks fine'; + """, + allowedCapabilities: [] + ) + ) + + #expect(observed.result?.output == .string("looks fine")) + let diagnostics = try #require(observed.result?.diagnostics) + let swallowed = try #require(diagnostics.first { $0.code == "BRIDGE_FAILURES_NOT_SURFACED" }) + #expect(swallowed.message.contains("keychain.read")) + #expect(swallowed.message.contains("CAPABILITY_DENIED")) + #expect(swallowed.suggestions.contains { $0.contains("missing await") }) + } + + @Test func aCleanRunGetsNoSwallowedFailureDiagnostic() async throws { + let (tools, sandbox) = try makeTools() + defer { cleanup(sandbox) } + + let observed = try await execute( + tools, + request: JavaScriptExecutionRequest( + code: "await apple.fs.write({ path: 'tmp:ok.txt', data: 'x' }); return 'ok';", + allowedCapabilities: [.fsWrite] + ) + ) + + #expect(observed.result?.diagnostics.contains { $0.code == "BRIDGE_FAILURES_NOT_SURFACED" } == false) + } + + // MARK: - Bounded concurrency + + @Test func concurrentExecutionsAllCompleteAndStayIsolated() async throws { + let (tools, sandbox) = try makeTools( + executionLimits: ExecutionLimits(maxConcurrentExecutions: 3) + ) + defer { cleanup(sandbox) } + + // The concurrent execution queue is a core design point but had no test, and + // no width limit: each execution blocks its thread for the whole run, so N + // parallel calls meant N blocked GCD threads and N live JavaScriptCore VMs. + // More executions than slots must still all complete, each with its own + // context and its own result. + let results = try await withThrowingTaskGroup(of: (Int, JSONValue?).self) { group in + for index in 0..<12 { + group.addTask { + let observed = try await execute( + tools, + request: JavaScriptExecutionRequest( + code: """ + await apple.fs.write({ path: 'tmp:worker-\(index).txt', data: 'value-\(index)' }); + const read = await apple.fs.read({ path: 'tmp:worker-\(index).txt' }); + return { index: \(index), text: read.text, leaked: typeof globalThis.__leaked }; + """, + allowedCapabilities: [.fsRead, .fsWrite], + timeoutMs: 20_000 + ) + ) + return (index, observed.result?.output) + } + } + + var collected: [Int: JSONValue?] = [:] + for try await (index, output) in group { + collected[index] = output + } + return collected + } + + #expect(results.count == 12) + for index in 0..<12 { + let output = try #require(results[index]??.objectValue) + #expect(output.int("index") == index) + #expect(output.string("text") == "value-\(index)") + // A fresh JSContext per execution: no state carries between them. + #expect(output.string("leaked") == "undefined") + } + } + + @Test func executionsBeyondTheSlotLimitQueueRatherThanFail() async throws { + let (tools, sandbox) = try makeTools( + executionLimits: ExecutionLimits(maxConcurrentExecutions: 1), + clock: VirtualClock() + ) + defer { cleanup(sandbox) } + + // With one slot these serialize; the point is that the extras wait for a slot + // instead of erroring, and that waiting for a slot does not count against the + // per-execution timeout. + let outputs = try await withThrowingTaskGroup(of: JSONValue?.self) { group in + for index in 0..<4 { + group.addTask { + try await execute( + tools, + request: JavaScriptExecutionRequest( + code: "await new Promise(r => setTimeout(r, 60)); return \(index);", + allowedCapabilities: [], + timeoutMs: 500 + ) + ).result?.output + } + } + var collected: [JSONValue?] = [] + for try await output in group { + collected.append(output) + } + return collected + } + + #expect(outputs.compactMap { $0?.intValue }.sorted() == [0, 1, 2, 3]) + } + + @Test func aResultThatSettlesRightAtTheDeadlineIsNotDiscarded() async throws { + let clock = VirtualClock() + let (tools, sandbox) = try makeTools(clock: clock) + defer { cleanup(sandbox) } + + // Timer and deadline at the same instant. The settled state must be read + // before the deadline is enforced: checking the deadline first turns a + // script that fulfilled exactly on time into a spurious timeout, discarding + // a result that already exists. Under a real clock this was a 12-iteration + // race; under a virtual one it is a single exact case. + let observed = try await execute( + tools, + request: JavaScriptExecutionRequest( + code: "await new Promise(resolve => setTimeout(resolve, 40)); return 'settled';", + allowedCapabilities: [], + timeoutMs: 40 + ) + ) + + #expect(observed.error == nil) + #expect(observed.result?.output == .string("settled")) + } + + // MARK: - Lifetime and slot semantics + + @Test func iteratingEventsAfterDroppingTheCallDoesNotCancelTheExecution() async throws { + let (tools, sandbox) = try makeTools() + defer { cleanup(sandbox) } + + // Swift does not keep a local alive to the end of its scope, so a host that + // reads `call.events` and never touches `call` again may have it released + // mid-iteration in an optimized build. Cancelling in the call's deinit cut + // that host off. Force the release explicitly here rather than hoping the + // optimizer does it. + var call: JavaScriptExecutionCall? = try await tools.executeJavaScript( + JavaScriptExecutionRequest( + code: """ + console.log('start'); + await new Promise(resolve => setTimeout(resolve, 200)); + console.log('end'); + return 'finished'; + """, + allowedCapabilities: [] + ) + ) + let events = call!.events + call = nil + + var sawFinished = false + var sawEnd = false + for await event in events { + if case .finished = event { sawFinished = true } + if case let .log(entry) = event, entry.message == "end" { sawEnd = true } + } + + #expect(sawEnd) + #expect(sawFinished) + } + + @Test func droppingBothTheCallAndItsStreamCancelsTheExecution() async throws { + // One slot: if the dropped execution were still running, the second call + // could not start until its 20s timer elapsed. + let (tools, sandbox) = try makeTools( + executionLimits: ExecutionLimits(maxConcurrentExecutions: 1) + ) + defer { cleanup(sandbox) } + + func startAndDrop() async throws { + let dropped = try await tools.executeJavaScript( + JavaScriptExecutionRequest( + code: "await new Promise(resolve => setTimeout(resolve, 20000)); return 1;", + allowedCapabilities: [], + timeoutMs: 60_000 + ) + ) + _ = dropped.events + // Give it a moment to actually occupy the slot before we drop it. + try await Task.sleep(nanoseconds: 100_000_000) + } + try await startAndDrop() + + let started = Date() + let observed = try await execute( + tools, + request: JavaScriptExecutionRequest(code: "return 'got the slot';", allowedCapabilities: [], timeoutMs: 5_000) + ) + + #expect(observed.result?.output == .string("got the slot")) + #expect(Date().timeIntervalSince(started) < 15, "the dropped execution must have released its slot") + } + + @Test func cancellingWhileWaitingForASlotNeverRunsTheScript() async throws { + let (tools, sandbox) = try makeTools( + executionLimits: ExecutionLimits(maxConcurrentExecutions: 1) + ) + defer { cleanup(sandbox) } + + // Occupy the only slot. + let occupant = try await tools.executeJavaScript( + JavaScriptExecutionRequest( + code: "await new Promise(resolve => setTimeout(resolve, 3000)); return 'occupant';", + allowedCapabilities: [], + timeoutMs: 30_000 + ) + ) + try await Task.sleep(nanoseconds: 100_000_000) + + // This one parks waiting for the slot. Cancel it there: it must report + // CANCELLED and must never have executed — the file it would write is the + // evidence. + let parked = try await tools.executeJavaScript( + JavaScriptExecutionRequest( + code: "await apple.fs.write({ path: 'tmp:ran.txt', data: 'x' }); return 'ran';", + allowedCapabilities: [.fsWrite], + timeoutMs: 30_000 + ) + ) + try await Task.sleep(nanoseconds: 100_000_000) + parked.cancel() + + let parkedOutcome = await observe(parked) + #expect(parkedOutcome.error?.code == "CANCELLED") + #expect(FileManager.default.fileExists(atPath: sandbox.tmp.appendingPathComponent("ran.txt").path) == false) + + // The occupant is unaffected and the slot still hands over cleanly afterwards. + let occupantOutcome = await observe(occupant) + #expect(occupantOutcome.result?.output == .string("occupant")) + } + + @Test func cancellationInterruptsASleepImmediately() throws { + let controller = ExecutionCancellationController() + let entered = DispatchSemaphore(value: 0) + let woke = DispatchSemaphore(value: 0) + + // A polling sleep would not notice the cancel until its next slice; an + // interruptible one returns as soon as the cancel lands. Synchronous test on + // purpose: the sleeper is a real blocked thread, as it is in the runtime. + DispatchQueue.global().async { + entered.signal() + controller.sleep(until: Date(timeIntervalSinceNow: 30)) + woke.signal() + } + + // Wait for the sleeper to actually be scheduled before cancelling. On a + // starved runner the dispatch alone can take seconds, and cancelling before + // the sleep has begun measures scheduling, not the wake-up this test is + // about. The generous bound here is for scheduling; the tight one below is + // for the wake. + #expect(entered.wait(timeout: .now() + 60) == .success, "sleeper never got a thread") + let cancelledAt = Date() + controller.cancel() + + #expect(woke.wait(timeout: .now() + 10) == .success) + #expect(Date().timeIntervalSince(cancelledAt) < 5) + } + + @Test func sleepReturnsAtTheDeadlineWhenNotCancelled() { + let controller = ExecutionCancellationController() + let started = Date() + controller.sleep(until: Date(timeIntervalSinceNow: 0.1)) + #expect(Date().timeIntervalSince(started) >= 0.09) + #expect(controller.isCancelled == false) + } + + @Test func searchDoesNotWaitBehindALongExecution() async throws { + let (tools, sandbox) = try makeTools( + executionLimits: ExecutionLimits(maxConcurrentExecutions: 1) + ) + defer { cleanup(sandbox) } + + // Occupy the only execution slot with a script that writes a file after + // 3s. If search shared that slot it would wait for it, and the file would + // exist by the time search returned. + let occupant = try await tools.executeJavaScript( + JavaScriptExecutionRequest( + code: "console.log('running'); await new Promise(r => setTimeout(r, 3000)); await apple.fs.write({ path: 'tmp:occupant-done.txt', data: 'x' }); return 1;", + allowedCapabilities: [.fsWrite], + timeoutMs: 30_000 + ) + ) + // Synchronize on the occupant actually holding the slot, not on a sleep. + var running = false + for await event in occupant.events { + if case let .log(entry) = event, entry.message == "running" { running = true; break } + if case .finished = event { break } + } + #expect(running, "the occupant never started") + + let response = try await tools.searchJavaScriptAPI( + JavaScriptAPISearchRequest(code: "async () => api.references.length") + ) + let occupantFinished = FileManager.default.fileExists(atPath: sandbox.tmp.appendingPathComponent("occupant-done.txt").path) + + #expect((response.result?.intValue ?? 0) > 0) + #expect(occupantFinished == false, "search waited for the execution slot") + occupant.cancel() + _ = await observe(occupant) + } +} diff --git a/Tests/CodeModeTests/ExecutionLimitsTests.swift b/Tests/CodeModeTests/ExecutionLimitsTests.swift new file mode 100644 index 0000000..7dae8ec --- /dev/null +++ b/Tests/CodeModeTests/ExecutionLimitsTests.swift @@ -0,0 +1,131 @@ +import Foundation +import Testing +@testable import CodeMode + +// Nothing bounded a script's output before: the whole result was stringified and +// copied into Swift, every console.log was retained and re-copied into each +// CodeModeToolError, and the events stream buffered without limit. One script +// could jetsam an iOS host. Every limit degrades with a diagnostic instead of +// failing, because for an LLM consumer a truncated result beats an OOM. + +@Test func oversizedResultsAreReplacedWithAParseableDescriptor() async throws { + let (tools, sandbox) = try makeTools( + executionLimits: ExecutionLimits(maxResultCharacters: 2_000, truncatedResultPreviewCharacters: 200) + ) + defer { cleanup(sandbox) } + + let observed = try await execute( + tools, + request: JavaScriptExecutionRequest( + code: "return Array.from({ length: 5000 }, (_, i) => 'entry-' + i);", + allowedCapabilities: [] + ) + ) + + #expect(observed.error == nil) + let output = try #require(observed.result?.output?.objectValue) + #expect(output.bool("__codeModeTruncated") == true) + #expect((output.int("characterCount") ?? 0) > 2_000) + #expect(output.int("limit") == 2_000) + #expect((output.string("preview")?.count ?? 0) <= 200) + + let diagnostics = try #require(observed.result?.diagnostics) + #expect(diagnostics.contains { $0.code == "RESULT_TRUNCATED" }) +} + +@Test func resultsWithinTheLimitAreUntouched() async throws { + let (tools, sandbox) = try makeTools( + executionLimits: ExecutionLimits(maxResultCharacters: 100_000) + ) + defer { cleanup(sandbox) } + + let observed = try await execute( + tools, + request: JavaScriptExecutionRequest(code: "return { ok: true, items: [1, 2, 3] };", allowedCapabilities: []) + ) + + let output = try #require(observed.result?.output?.objectValue) + #expect(output.bool("ok") == true) + #expect(output.array("items")?.count == 3) + #expect(observed.result?.diagnostics.contains { $0.code == "RESULT_TRUNCATED" } == false) +} + +@Test func retainedLogsAreCappedWithANotice() async throws { + let (tools, sandbox) = try makeTools( + executionLimits: ExecutionLimits(maxLogEntries: 25) + ) + defer { cleanup(sandbox) } + + let observed = try await execute( + tools, + request: JavaScriptExecutionRequest( + code: "for (let i = 0; i < 500; i++) { console.log('line ' + i); } return 'done';", + allowedCapabilities: [] + ) + ) + + let logs = try #require(observed.result?.logs) + #expect(logs.count == 25) + // The first N are kept: the beginning of a runaway log is where the cause is. + #expect(logs.first?.message == "line 0") + #expect(observed.result?.diagnostics.contains { $0.code == "LOG_LIMIT_REACHED" } == true) +} + +@Test func longLogMessagesAreElidedInTheMiddle() async throws { + let (tools, sandbox) = try makeTools( + executionLimits: ExecutionLimits(maxLogMessageCharacters: 200) + ) + defer { cleanup(sandbox) } + + let observed = try await execute( + tools, + request: JavaScriptExecutionRequest( + code: "console.log('HEAD' + 'x'.repeat(100000) + 'TAIL'); return 'done';", + allowedCapabilities: [] + ) + ) + + let message = try #require(observed.result?.logs.first?.message) + #expect(message.count < 400) + #expect(message.hasPrefix("HEAD")) + #expect(message.hasSuffix("TAIL")) + #expect(message.contains("characters elided")) +} + +@Test func diagnosticsAndPermissionEventsAreCapped() { + let transcript = ExecutionTranscript( + limits: ExecutionLimits(maxDiagnostics: 3, maxPermissionEvents: 2) + ) + + for index in 0..<50 { + transcript.record(diagnostic: ToolDiagnostic(severity: .info, code: "D\(index)", message: "m")) + transcript.record(permissionEvent: PermissionEvent(permission: .contacts, status: .granted)) + } + + let snapshot = transcript.snapshot() + #expect(snapshot.diagnostics.count == 3) + #expect(snapshot.permissionEvents.count == 2) +} + +@Test func theEventStreamBufferIsBounded() async throws { + let (tools, sandbox) = try makeTools( + executionLimits: ExecutionLimits(maxLogEntries: 100_000, maxBufferedEvents: 16) + ) + defer { cleanup(sandbox) } + + // Never drained until the script has finished: with the default unbounded + // policy this buffers every event; bounded, it keeps only the newest. + let call = try await tools.executeJavaScript( + JavaScriptExecutionRequest( + code: "for (let i = 0; i < 5000; i++) { console.log('line ' + i); } return 'done';", + allowedCapabilities: [] + ) + ) + _ = try await call.result + + var received = 0 + for await _ in call.events { + received += 1 + } + #expect(received <= 16) +} diff --git a/Tests/CodeModeTests/ExecutionWatchdogTests.swift b/Tests/CodeModeTests/ExecutionWatchdogTests.swift index 1e6012d..06e2545 100644 --- a/Tests/CodeModeTests/ExecutionWatchdogTests.swift +++ b/Tests/CodeModeTests/ExecutionWatchdogTests.swift @@ -2,171 +2,205 @@ import Foundation import Testing @testable import CodeMode -@Test func executeTerminatesCPUBoundInfiniteLoop() async throws { - let (tools, sandbox) = try makeTools() - defer { cleanup(sandbox) } - - let started = Date() - let observed = try await execute( - tools, - request: JavaScriptExecutionRequest( - code: "while (true) {}", - allowedCapabilities: [], - timeoutMs: 300 +/// Serialized: each of these spins a core at 100% for its budget. Seven at once on a +/// three-core runner is a stall, not a test. +@Suite(.serialized) struct ExecutionWatchdogTests { + + @Test func executeTerminatesCPUBoundInfiniteLoop() async throws { + let (tools, sandbox) = try makeTools() + defer { cleanup(sandbox) } + + let started = Date() + let observed = try await execute( + tools, + request: JavaScriptExecutionRequest( + code: "while (true) {}", + allowedCapabilities: [], + timeoutMs: 300 + ) ) - ) - #expect(observed.result == nil) - #expect(observed.error?.code == "EXECUTION_TIMEOUT") - #expect(Date().timeIntervalSince(started) < 5) -} + #expect(observed.result == nil) + #expect(observed.error?.code == "EXECUTION_TIMEOUT") + // A hang-guard, not a performance assertion: `while (true) {}` must be + // terminated at all. The error code above is the real check, and the bound is + // loose because a shared CI runner is not an idle machine. + #expect(Date().timeIntervalSince(started) < 20) + } -@Test func executeTerminatesCPUBoundLoopInsidePromiseChain() async throws { - let (tools, sandbox) = try makeTools() - defer { cleanup(sandbox) } - - let observed = try await execute( - tools, - request: JavaScriptExecutionRequest( - code: """ - await Promise.resolve(); - while (true) {} - """, - allowedCapabilities: [], - timeoutMs: 300 - ) - ) + @Test func executeTerminatesCPUBoundLoopInsidePromiseChain() async throws { + let (tools, sandbox) = try makeTools() + defer { cleanup(sandbox) } - #expect(observed.result == nil) - #expect(observed.error?.code == "EXECUTION_TIMEOUT") -} + let observed = try await execute( + tools, + request: JavaScriptExecutionRequest( + code: """ + await Promise.resolve(); + while (true) {} + """, + allowedCapabilities: [], + timeoutMs: 300 + ) + ) -@Test func executeTimeoutTerminationIsNotCatchableByScript() async throws { - let (tools, sandbox) = try makeTools() - defer { cleanup(sandbox) } + #expect(observed.result == nil) + #expect(observed.error?.code == "EXECUTION_TIMEOUT") + } - let observed = try await execute( - tools, - request: JavaScriptExecutionRequest( - code: """ - try { - while (true) {} - } catch (error) { - return { swallowed: true }; - } - """, - allowedCapabilities: [], - timeoutMs: 300 + @Test func executeTimeoutTerminationIsNotCatchableByScript() async throws { + let (tools, sandbox) = try makeTools() + defer { cleanup(sandbox) } + + let observed = try await execute( + tools, + request: JavaScriptExecutionRequest( + code: """ + try { + while (true) {} + } catch (error) { + return { swallowed: true }; + } + """, + allowedCapabilities: [], + timeoutMs: 300 + ) ) - ) - #expect(observed.result == nil) - #expect(observed.error?.code == "EXECUTION_TIMEOUT") -} + #expect(observed.result == nil) + #expect(observed.error?.code == "EXECUTION_TIMEOUT") + } -@Test func executeRecoversWithFreshContextAfterTerminatedLoop() async throws { - let (tools, sandbox) = try makeTools() - defer { cleanup(sandbox) } + @Test func executeRecoversWithFreshContextAfterTerminatedLoop() async throws { + let (tools, sandbox) = try makeTools() + defer { cleanup(sandbox) } - let timedOut = try await execute( - tools, - request: JavaScriptExecutionRequest( - code: "while (true) {}", - allowedCapabilities: [], - timeoutMs: 200 + let timedOut = try await execute( + tools, + request: JavaScriptExecutionRequest( + code: "while (true) {}", + allowedCapabilities: [], + timeoutMs: 200 + ) ) - ) - #expect(timedOut.error?.code == "EXECUTION_TIMEOUT") - - let recovered = try await execute( - tools, - request: JavaScriptExecutionRequest( - code: "return { ok: true };", - allowedCapabilities: [] + #expect(timedOut.error?.code == "EXECUTION_TIMEOUT") + + let recovered = try await execute( + tools, + request: JavaScriptExecutionRequest( + code: "return { ok: true };", + allowedCapabilities: [] + ) ) - ) - let payload = try requireJSONObject(from: try #require(recovered.result)) - #expect(payload["ok"] as? Bool == true) -} + let payload = try requireJSONObject(from: try #require(recovered.result)) + #expect(payload["ok"] as? Bool == true) + } -@Test func cancelInterruptsRunningInfiniteLoop() async throws { - let (tools, sandbox) = try makeTools() - defer { cleanup(sandbox) } + @Test func cancelInterruptsRunningInfiniteLoop() async throws { + let (tools, sandbox) = try makeTools() + defer { cleanup(sandbox) } - let call = try await tools.executeJavaScript( - JavaScriptExecutionRequest( - code: "while (true) {}", - allowedCapabilities: [], - timeoutMs: 60_000 + let call = try await tools.executeJavaScript( + JavaScriptExecutionRequest( + code: "while (true) {}", + allowedCapabilities: [], + timeoutMs: 60_000 + ) ) - ) - try await Task.sleep(nanoseconds: 200_000_000) - call.cancel() - let observed = await observe(call) + try await Task.sleep(nanoseconds: 200_000_000) + call.cancel() + let observed = await observe(call) - #expect(observed.result == nil) - #expect(observed.error?.code == "CANCELLED") -} + #expect(observed.result == nil) + #expect(observed.error?.code == "CANCELLED") + } -@Test func executeReturnsResultThatSettlesRightAtDeadline() async throws { - let (tools, sandbox) = try makeTools() - defer { cleanup(sandbox) } - - // A short CPU spin that finishes well under the timeout must return its - // value, not be discarded by a deadline check that races the settled state. - let observed = try await execute( - tools, - request: JavaScriptExecutionRequest( - code: """ - let total = 0; - for (let i = 0; i < 200000; i++) { total += i; } - return { total }; - """, - allowedCapabilities: [], - timeoutMs: 2_000 + @Test func executeReturnsResultThatSettlesRightAtDeadline() async throws { + let (tools, sandbox) = try makeTools() + defer { cleanup(sandbox) } + + // A short CPU spin that finishes well under the timeout must return its + // value, not be discarded by a deadline check that races the settled state. + let observed = try await execute( + tools, + request: JavaScriptExecutionRequest( + code: """ + let total = 0; + for (let i = 0; i < 200000; i++) { total += i; } + return { total }; + """, + allowedCapabilities: [], + timeoutMs: 2_000 + ) ) - ) - let payload = try requireJSONObject(from: try #require(observed.result)) - #expect(payload["total"] != nil) - #expect(observed.error == nil) -} + let payload = try requireJSONObject(from: try #require(observed.result)) + #expect(payload["total"] != nil) + #expect(observed.error == nil) + } + + @Test func executeTerminatesInfiniteGetterDuringResultSerialization() async throws { + let (tools, sandbox) = try makeTools() + defer { cleanup(sandbox) } + + // The result settles fine, but serializing it runs a runaway getter. The + // watchdog must still bound that phase instead of hanging the thread. + let started = Date() + let observed = try await execute( + tools, + request: JavaScriptExecutionRequest( + code: "return { get trap() { while (true) {} } };", + allowedCapabilities: [], + timeoutMs: 300 + ) + ) + + #expect(observed.result?.output == nil) + // The watchdog fired, so this is a timeout — not the INVALID_RESULT + // "must be JSON-serializable" it used to report, which pointed the model at + // the wrong repair entirely. + #expect(observed.error?.code == "EXECUTION_TIMEOUT") + // Same hang-guard reasoning: the runaway getter must be terminated at all. + #expect(Date().timeIntervalSince(started) < 20) + } -@Test func executeTerminatesInfiniteGetterDuringResultSerialization() async throws { - let (tools, sandbox) = try makeTools() - defer { cleanup(sandbox) } - - // The result settles fine, but serializing it runs a runaway getter. The - // watchdog must still bound that phase instead of hanging the thread. - let started = Date() - let observed = try await execute( - tools, - request: JavaScriptExecutionRequest( - code: "return { get trap() { while (true) {} } };", - allowedCapabilities: [], - timeoutMs: 300 + @Test func serializationBudgetDoesNotScaleWithTheExecutionTimeout() async throws { + let (tools, sandbox) = try makeTools() + defer { cleanup(sandbox) } + + // A long execution budget used to buy an equally long serialization budget + // (`max`, not `min`), so a runaway getter after a 30s-budget script got + // another 30s. The serialization phase is bounded independently. + let started = Date() + let observed = try await execute( + tools, + request: JavaScriptExecutionRequest( + code: "return { get trap() { while (true) {} } };", + allowedCapabilities: [], + timeoutMs: 30_000 + ) ) - ) - #expect(observed.result?.output == nil) - #expect(observed.error != nil) - #expect(Date().timeIntervalSince(started) < 5) -} + #expect(observed.error?.code == "EXECUTION_TIMEOUT") + // Proves serialization did not inherit the script's 30s budget. + #expect(Date().timeIntervalSince(started) < 20) + } -@Test func searchTerminatesCPUBoundInfiniteLoop() async throws { - let (tools, sandbox) = try makeTools() - defer { cleanup(sandbox) } + @Test func searchTerminatesCPUBoundInfiniteLoop() async throws { + let (tools, sandbox) = try makeTools() + defer { cleanup(sandbox) } - do { - _ = try await tools.searchJavaScriptAPI( - JavaScriptAPISearchRequest( - code: "async () => { while (true) {} }" + do { + _ = try await tools.searchJavaScriptAPI( + JavaScriptAPISearchRequest( + code: "async () => { while (true) {} }" + ) ) - ) - Issue.record("Expected search to time out") - } catch let error as CodeModeToolError { - #expect(error.code == "SEARCH_TIMEOUT") + Issue.record("Expected search to time out") + } catch let error as CodeModeToolError { + #expect(error.code == "SEARCH_TIMEOUT") + } } } diff --git a/Tests/CodeModeTests/FileSystemBridgeTests.swift b/Tests/CodeModeTests/FileSystemBridgeTests.swift index 2cae75c..2d44369 100644 --- a/Tests/CodeModeTests/FileSystemBridgeTests.swift +++ b/Tests/CodeModeTests/FileSystemBridgeTests.swift @@ -239,3 +239,378 @@ private final class RecordingCodeModeFileSystem: CodeModeFileSystem, @unchecked let payload = try requireJSONObject(from: try #require(observed.result)) #expect(payload["text"] as? String == "fs-from-execute") } + +// MARK: - Destructive-destination guards + +@Test func fileSystemMoveRefusesASandboxRootAsDestination() throws { + let fs = FileSystemBridge() + let (context, sandbox) = try makeInvocationContext() + defer { cleanup(sandbox) } + + _ = try fs.write(arguments: ["path": .string("tmp:junk.txt"), "data": .string("x")], context: context) + _ = try fs.write(arguments: ["path": .string("documents:keepme.txt"), "data": .string("precious")], context: context) + + // `documents:` resolves to the documents root itself, and containment admits + // it. The old unconditional removeItem here deleted the user's Documents. + for destination in ["documents:", "documents:/", "tmp:", "caches:"] { + do { + _ = try fs.move(arguments: [ + "from": .string("tmp:junk.txt"), + "to": .string(destination), + "overwrite": .bool(true), + "recursive": .bool(true), + ], context: context) + Issue.record("Expected fs.move to refuse the root destination \(destination)") + } catch { + #expect(requireBridgeErrorCode(error) == "PATH_POLICY_VIOLATION") + } + } + + let survived = try fs.read(arguments: ["path": .string("documents:keepme.txt")], context: context) + #expect(try requireObject(survived).string("text") == "precious") +} + +@Test func fileSystemCopyRefusesASandboxRootAsDestination() throws { + let fs = FileSystemBridge() + let (context, sandbox) = try makeInvocationContext() + defer { cleanup(sandbox) } + + _ = try fs.write(arguments: ["path": .string("tmp:junk.txt"), "data": .string("x")], context: context) + + #expect(throws: (any Error).self) { + _ = try fs.copy(arguments: [ + "from": .string("tmp:junk.txt"), + "to": .string("documents:"), + "overwrite": .bool(true), + ], context: context) + } +} + +@Test func fileSystemDeleteRefusesASandboxRoot() throws { + let fs = FileSystemBridge() + let (context, sandbox) = try makeInvocationContext() + defer { cleanup(sandbox) } + + do { + _ = try fs.delete(arguments: ["path": .string("documents:"), "recursive": .bool(true)], context: context) + Issue.record("Expected fs.delete to refuse a sandbox root") + } catch { + #expect(requireBridgeErrorCode(error) == "PATH_POLICY_VIOLATION") + } + #expect(FileManager.default.fileExists(atPath: sandbox.documents.path)) +} + +@Test func fileSystemMoveAndCopyRequireExplicitOverwrite() throws { + let fs = FileSystemBridge() + let (context, sandbox) = try makeInvocationContext() + defer { cleanup(sandbox) } + + _ = try fs.write(arguments: ["path": .string("tmp:a.txt"), "data": .string("source")], context: context) + _ = try fs.write(arguments: ["path": .string("tmp:b.txt"), "data": .string("destination")], context: context) + + do { + _ = try fs.copy(arguments: ["from": .string("tmp:a.txt"), "to": .string("tmp:b.txt")], context: context) + Issue.record("Expected fs.copy to refuse an existing destination without overwrite") + } catch { + #expect(requireBridgeErrorCode(error) == "INVALID_ARGUMENTS") + } + + // The destination is untouched by the refused call. + let intact = try fs.read(arguments: ["path": .string("tmp:b.txt")], context: context) + #expect(try requireObject(intact).string("text") == "destination") + + _ = try fs.copy(arguments: [ + "from": .string("tmp:a.txt"), + "to": .string("tmp:b.txt"), + "overwrite": .bool(true), + ], context: context) + let replaced = try fs.read(arguments: ["path": .string("tmp:b.txt")], context: context) + #expect(try requireObject(replaced).string("text") == "source") +} + +@Test func fileSystemMoveOntoADirectoryAlsoRequiresRecursive() throws { + let fs = FileSystemBridge() + let (context, sandbox) = try makeInvocationContext() + defer { cleanup(sandbox) } + + _ = try fs.write(arguments: ["path": .string("tmp:a.txt"), "data": .string("source")], context: context) + _ = try fs.mkdir(arguments: ["path": .string("tmp:dir")], context: context) + _ = try fs.write(arguments: ["path": .string("tmp:dir/kept.txt"), "data": .string("kept")], context: context) + + do { + _ = try fs.move(arguments: [ + "from": .string("tmp:a.txt"), + "to": .string("tmp:dir"), + "overwrite": .bool(true), + ], context: context) + Issue.record("Expected fs.move onto a directory to require recursive=true") + } catch { + #expect(requireBridgeErrorCode(error) == "INVALID_ARGUMENTS") + } + #expect(try fs.exists(arguments: ["path": .string("tmp:dir/kept.txt")], context: context).boolValue == true) + + _ = try fs.move(arguments: [ + "from": .string("tmp:a.txt"), + "to": .string("tmp:dir"), + "overwrite": .bool(true), + "recursive": .bool(true), + ], context: context) + #expect(try fs.exists(arguments: ["path": .string("tmp:dir/kept.txt")], context: context).boolValue == false) +} + +// MARK: - Size and decoding limits + +@Test func fileSystemReadRefusesFilesOverTheLimit() throws { + let fs = FileSystemBridge(limits: FileSystemLimits(maxReadBytes: 64, maxWriteBytes: 1_024)) + let (context, sandbox) = try makeInvocationContext() + defer { cleanup(sandbox) } + + _ = try fs.write(arguments: [ + "path": .string("tmp:big.txt"), + "data": .string(String(repeating: "x", count: 512)), + ], context: context) + + do { + _ = try fs.read(arguments: ["path": .string("tmp:big.txt")], context: context) + Issue.record("Expected fs.read to refuse a file over the read limit") + } catch { + #expect(requireBridgeErrorCode(error) == "INVALID_ARGUMENTS") + } +} + +@Test func fileSystemWriteRefusesPayloadsOverTheLimit() throws { + let fs = FileSystemBridge(limits: FileSystemLimits(maxReadBytes: 1_024, maxWriteBytes: 16)) + let (context, sandbox) = try makeInvocationContext() + defer { cleanup(sandbox) } + + do { + _ = try fs.write(arguments: [ + "path": .string("tmp:nested/big.txt"), + "data": .string(String(repeating: "x", count: 512)), + ], context: context) + Issue.record("Expected fs.write to refuse a payload over the write limit") + } catch { + #expect(requireBridgeErrorCode(error) == "INVALID_ARGUMENTS") + } + + // A refused write leaves no directory behind. + #expect(try fs.exists(arguments: ["path": .string("tmp:nested")], context: context).boolValue == false) +} + +@Test func fileSystemReadReportsUndecodableUTF8InsteadOfEmptyText() throws { + let fs = FileSystemBridge() + let (context, sandbox) = try makeInvocationContext() + defer { cleanup(sandbox) } + + // 0xFF is never valid UTF-8. Returning "" here told the script the file was + // empty; it must be an error the script can repair with base64. + let binary = Data([0xFF, 0xFE, 0x00, 0x01]) + _ = try fs.write(arguments: [ + "path": .string("tmp:binary.bin"), + "data": .string(binary.base64EncodedString()), + "encoding": .string("base64"), + ], context: context) + + do { + _ = try fs.read(arguments: ["path": .string("tmp:binary.bin")], context: context) + Issue.record("Expected fs.read to fail on undecodable UTF-8") + } catch { + #expect(requireBridgeErrorCode(error) == "INVALID_ARGUMENTS") + } + + let base64 = try fs.read(arguments: [ + "path": .string("tmp:binary.bin"), + "encoding": .string("base64"), + ], context: context) + #expect(try requireObject(base64).string("base64") == binary.base64EncodedString()) +} + +// MARK: - Source-side guards and non-destructive replacement + +@Test func fileSystemMoveRefusesASandboxRootAsSource() throws { + let fs = FileSystemBridge() + let (context, sandbox) = try makeInvocationContext() + defer { cleanup(sandbox) } + _ = try fs.write(arguments: ["path": .string("documents:precious.txt"), "data": .string("keep")], context: context) + + // The destination-side guard alone let this move Documents into tmp:, which + // the OS may purge. + do { + _ = try fs.move(arguments: ["from": .string("documents:"), "to": .string("tmp:stolen")], context: context) + Issue.record("Expected fs.move to refuse a sandbox root as its source") + } catch { + #expect(requireBridgeErrorCode(error) == "PATH_POLICY_VIOLATION") + } + #expect(FileManager.default.fileExists(atPath: sandbox.documents.appendingPathComponent("precious.txt").path)) +} + +@Test func fileSystemCopyRefusesASandboxRootAsSource() throws { + let fs = FileSystemBridge() + let (context, sandbox) = try makeInvocationContext() + defer { cleanup(sandbox) } + + do { + _ = try fs.copy(arguments: ["from": .string("documents:"), "to": .string("tmp:copy")], context: context) + Issue.record("Expected fs.copy to refuse a sandbox root as its source") + } catch { + #expect(requireBridgeErrorCode(error) == "PATH_POLICY_VIOLATION") + } +} + +@Test func fileSystemOverwriteWithAMissingSourceKeepsTheDestination() throws { + let fs = FileSystemBridge() + let (context, sandbox) = try makeInvocationContext() + defer { cleanup(sandbox) } + _ = try fs.write(arguments: ["path": .string("documents:important.txt"), "data": .string("keep")], context: context) + + // Previously: destination removed, then the move failed on the missing source. + #expect(throws: (any Error).self) { + _ = try fs.move(arguments: [ + "from": .string("tmp:does-not-exist"), + "to": .string("documents:important.txt"), + "overwrite": .bool(true), + ], context: context) + } + let survived = try fs.read(arguments: ["path": .string("documents:important.txt")], context: context) + #expect(try requireObject(survived).string("text") == "keep") +} + +@Test func fileSystemOverwriteOntoItselfIsRefusedAndKeepsTheFile() throws { + let fs = FileSystemBridge() + let (context, sandbox) = try makeInvocationContext() + defer { cleanup(sandbox) } + _ = try fs.write(arguments: ["path": .string("tmp:x.txt"), "data": .string("keep")], context: context) + _ = try fs.mkdir(arguments: ["path": .string("tmp:dir")], context: context) + + // from == to with overwrite: true used to delete the source. + #expect(throws: (any Error).self) { + _ = try fs.move(arguments: ["from": .string("tmp:x.txt"), "to": .string("tmp:x.txt"), "overwrite": .bool(true)], context: context) + } + #expect(try fs.exists(arguments: ["path": .string("tmp:x.txt")], context: context).boolValue == true) + + // A destination inside the source can never succeed either. + #expect(throws: (any Error).self) { + _ = try fs.move(arguments: ["from": .string("tmp:dir"), "to": .string("tmp:dir/inner")], context: context) + } +} + +/// Fails the copy itself, after the destination has been moved aside — the case +/// the old remove-then-act order turned into data loss. +private final class FailingCopyFileSystem: CodeModeFileSystem, @unchecked Sendable { + private let base = LocalCodeModeFileSystem() + func listDirectory(at url: URL) throws -> [CodeModeFileSystemEntry] { try base.listDirectory(at: url) } + func readData(at url: URL) throws -> Data { try base.readData(at: url) } + func writeData(_ data: Data, to url: URL) throws { try base.writeData(data, to: url) } + func moveItem(at sourceURL: URL, to destinationURL: URL) throws { try base.moveItem(at: sourceURL, to: destinationURL) } + func copyItem(at sourceURL: URL, to destinationURL: URL) throws { + throw CocoaError(.fileWriteOutOfSpace) + } + func removeItem(at url: URL) throws { try base.removeItem(at: url) } + func createDirectory(at url: URL, recursive: Bool) throws { try base.createDirectory(at: url, recursive: recursive) } + func attributesOfItem(at url: URL) throws -> CodeModeFileSystemAttributes { try base.attributesOfItem(at: url) } + func itemExists(at url: URL) -> Bool { base.itemExists(at: url) } + func access(at url: URL) -> CodeModeFileSystemAccess { base.access(at: url) } +} + +@Test func fileSystemFailedReplacementRestoresTheDestination() throws { + let fs = FileSystemBridge(fileSystem: FailingCopyFileSystem()) + let (context, sandbox) = try makeInvocationContext() + defer { cleanup(sandbox) } + _ = try fs.write(arguments: ["path": .string("tmp:src.txt"), "data": .string("new")], context: context) + _ = try fs.write(arguments: ["path": .string("tmp:dst.txt"), "data": .string("original")], context: context) + + #expect(throws: (any Error).self) { + _ = try fs.copy(arguments: ["from": .string("tmp:src.txt"), "to": .string("tmp:dst.txt"), "overwrite": .bool(true)], context: context) + } + + let restored = try fs.read(arguments: ["path": .string("tmp:dst.txt")], context: context) + #expect(try requireObject(restored).string("text") == "original") + // And nothing left behind from the aside move. + let leftovers = try FileManager.default.contentsOfDirectory(atPath: sandbox.tmp.path).filter { $0.contains("codemode-replacing") } + #expect(leftovers.isEmpty) +} + +@Test func fileSystemSuccessfulReplacementLeavesNoAsideFile() throws { + let fs = FileSystemBridge() + let (context, sandbox) = try makeInvocationContext() + defer { cleanup(sandbox) } + _ = try fs.write(arguments: ["path": .string("tmp:src.txt"), "data": .string("new")], context: context) + _ = try fs.write(arguments: ["path": .string("tmp:dst.txt"), "data": .string("original")], context: context) + + _ = try fs.move(arguments: ["from": .string("tmp:src.txt"), "to": .string("tmp:dst.txt"), "overwrite": .bool(true)], context: context) + + let replaced = try fs.read(arguments: ["path": .string("tmp:dst.txt")], context: context) + #expect(try requireObject(replaced).string("text") == "new") + let leftovers = try FileManager.default.contentsOfDirectory(atPath: sandbox.tmp.path).filter { $0.contains("codemode-replacing") } + #expect(leftovers.isEmpty) +} + +@Test func fileSystemCopyAnswersToTheWriteLimit() throws { + let fs = FileSystemBridge(limits: FileSystemLimits(maxReadBytes: 1_024, maxWriteBytes: 100)) + let (context, sandbox) = try makeInvocationContext() + defer { cleanup(sandbox) } + _ = try fs.mkdir(arguments: ["path": .string("tmp:tree/nested")], context: context) + _ = try fs.write(arguments: ["path": .string("tmp:tree/a.txt"), "data": .string(String(repeating: "x", count: 60))], context: context) + _ = try fs.write(arguments: ["path": .string("tmp:tree/nested/b.txt"), "data": .string(String(repeating: "y", count: 60))], context: context) + + // Each file is under the limit; the tree is not. + do { + _ = try fs.copy(arguments: ["from": .string("tmp:tree"), "to": .string("tmp:tree-copy")], context: context) + Issue.record("Expected fs.copy to refuse a tree over the write limit") + } catch { + #expect(requireBridgeErrorCode(error) == "INVALID_ARGUMENTS") + } + #expect(try fs.exists(arguments: ["path": .string("tmp:tree-copy")], context: context).boolValue == false) +} + +/// A host's own policy — here, one that narrows the default to two roots. Since +/// `allowedRoots` has no default, it must state its roots, and the bridge's root +/// guards apply to it exactly as to `DefaultPathPolicy`. +private struct TwoRootPolicy: PathPolicy { + let base: DefaultPathPolicy + let roots: [URL] + + func resolve(path: String) throws -> URL { + let url = try base.resolve(path: path) + guard roots.contains(where: { url.path.hasPrefix($0.resolvingSymlinksInPath().path) }) else { + throw BridgeError.pathViolation("outside this host's roots") + } + return url + } + + var allowedRoots: [URL] { roots } +} + +@Test func customPathPoliciesGetTheSameRootGuards() throws { + let sandbox = try makeTestSandbox() + defer { cleanup(sandbox) } + let policy = TwoRootPolicy( + base: DefaultPathPolicy(config: PathPolicyConfig(tmpRoot: sandbox.tmp, cachesRoot: sandbox.caches, documentsRoot: sandbox.documents)), + roots: [sandbox.tmp, sandbox.documents] + ) + let context = BridgeInvocationContext( + executionContext: .init(), + allowedCapabilities: Set(CapabilityID.allCases), + pathPolicy: policy, + artifactStore: InMemoryArtifactStore(), + permissionBroker: NoopPermissionBroker(), + auditLogger: SyncAuditLogger(), + transcript: ExecutionTranscript(), + cancellationController: ExecutionCancellationController() + ) + let fs = FileSystemBridge() + _ = try fs.write(arguments: ["path": .string("documents:keep.txt"), "data": .string("x")], context: context) + + for arguments: [String: JSONValue] in [ + ["from": .string("documents:"), "to": .string("tmp:stolen")], + ["from": .string("tmp:keep.txt"), "to": .string("documents:"), "overwrite": .bool(true), "recursive": .bool(true)], + ] { + do { + _ = try fs.move(arguments: arguments, context: context) + Issue.record("Expected the custom policy's root to be refused for \(arguments)") + } catch { + #expect(requireBridgeErrorCode(error) == "PATH_POLICY_VIOLATION") + } + } + #expect(FileManager.default.fileExists(atPath: sandbox.documents.appendingPathComponent("keep.txt").path)) +} diff --git a/Tests/CodeModeTests/HostConfigurationValidatorTests.swift b/Tests/CodeModeTests/HostConfigurationValidatorTests.swift index 671aa27..f35cf2f 100644 --- a/Tests/CodeModeTests/HostConfigurationValidatorTests.swift +++ b/Tests/CodeModeTests/HostConfigurationValidatorTests.swift @@ -201,3 +201,21 @@ import Testing #expect(issues.contains(where: { $0.key == "Apple Pay merchant configuration" && $0.severity == .warning })) #expect(issues.contains(where: { $0.key == "StoreKit configuration" && $0.severity == .warning })) } + +// MARK: - Configuration validation + +@Test func validatorWarnsWhenTheCapabilityGrantIsUnrestricted() { + // `.unrestricted` is the default for source compatibility, but it means the + // model-authored allowlist is the only one in force. A host should be told. + let issues = HostConfigurationValidator.validate(configuration: CodeModeConfiguration()) + let issue = issues.first { $0.key == "capabilityGrant" } + #expect(issue?.severity == .warning) + #expect(issue?.message.contains(".unrestricted") == true) +} + +@Test func validatorIsQuietWhenAGrantIsSet() { + for grant in [CapabilityGrant.only([.fsRead]), .none] { + let issues = HostConfigurationValidator.validate(configuration: CodeModeConfiguration(capabilityGrant: grant)) + #expect(issues.isEmpty, "unexpected issues for grant \(grant): \(issues)") + } +} diff --git a/Tests/CodeModeTests/HostRuntimeTests.swift b/Tests/CodeModeTests/HostRuntimeTests.swift index 158798a..489db5d 100644 --- a/Tests/CodeModeTests/HostRuntimeTests.swift +++ b/Tests/CodeModeTests/HostRuntimeTests.swift @@ -602,7 +602,11 @@ import Testing ) #expect(observed.result == nil) - #expect(observed.error?.code == "EXECUTION_TIMEOUT") + // A promise with no resolve path and no pending timer is provably + // unsettleable under this runtime, so it is reported immediately instead of + // sleeping a thread until the deadline for a result that can never arrive. + #expect(observed.error?.code == "JS_RUNTIME_ERROR") + #expect(observed.error?.message.contains("can never settle") == true) } @Test func executeRecoversWithFreshContextAfterTimeout() async throws { @@ -617,7 +621,7 @@ import Testing timeoutMs: 30 ) ) - #expect(timedOut.error?.code == "EXECUTION_TIMEOUT") + #expect(timedOut.error?.code == "JS_RUNTIME_ERROR") let recovered = try await execute( tools, @@ -700,16 +704,33 @@ import Testing let (tools, sandbox) = try makeTools() defer { cleanup(sandbox) } + // A long timer, not an unsettleable promise: the latter now fails fast with + // JS_RUNTIME_ERROR, which would win the race against the cancellation this + // test is about. let call = try await tools.executeJavaScript( JavaScriptExecutionRequest( code: """ - await new Promise(() => {}); + console.log('running'); + await new Promise(resolve => setTimeout(resolve, 20000)); return { never: true }; """, - allowedCapabilities: [] + allowedCapabilities: [], + timeoutMs: 60_000 ) ) + // Cancel only once the script is observed to be running. A `Task.yield()` + // before cancelling is not a guarantee under load: on a stalled runner the + // test task can be descheduled long enough for the 20s timer to elapse, and + // then a cancel of an already-finished execution is a no-op that this test + // would misreport as a runtime bug. + var running = false + for await event in call.events { + if case let .log(entry) = event, entry.message == "running" { running = true; break } + if case .finished = event { break } + } + #expect(running, "the script never reported starting") + let waiter = Task, Never> { do { return .success(try await call.result) @@ -720,7 +741,6 @@ import Testing } } - await Task.yield() waiter.cancel() let waiterResult = await waiter.value diff --git a/Tests/CodeModeTests/KeychainBridgeTests.swift b/Tests/CodeModeTests/KeychainBridgeTests.swift index 46b6946..722467f 100644 --- a/Tests/CodeModeTests/KeychainBridgeTests.swift +++ b/Tests/CodeModeTests/KeychainBridgeTests.swift @@ -1,8 +1,9 @@ import Foundation import Testing @testable import CodeMode +import Security -@Test func keychainRoundTripWriteReadDelete() throws { +@Test(.enabled(if: keychainIsAvailable, "no keychain entitlement in this test host (errSecMissingEntitlement)")) func keychainRoundTripWriteReadDelete() throws { let bridge = KeychainBridge(service: "CodeModeTests.\(UUID().uuidString)") let key = "token" @@ -20,7 +21,7 @@ import Testing #expect(postDelete == .null) } -@Test func keychainReadMissingValueReturnsNull() throws { +@Test(.enabled(if: keychainIsAvailable, "no keychain entitlement in this test host (errSecMissingEntitlement)")) func keychainReadMissingValueReturnsNull() throws { let bridge = KeychainBridge(service: "CodeModeTests.\(UUID().uuidString)") let readValue = try bridge.read(arguments: ["key": .string("missing")]) #expect(readValue == .null) @@ -37,7 +38,7 @@ import Testing } } -@Test func executeUsesKeychainBridge() async throws { +@Test(.enabled(if: keychainIsAvailable, "no keychain entitlement in this test host (errSecMissingEntitlement)")) func executeUsesKeychainBridge() async throws { let (tools, sandbox) = try makeTools() defer { cleanup(sandbox) } @@ -58,3 +59,15 @@ import Testing let payload = try requireJSONObject(from: try #require(observed.result)) #expect(payload["value"] as? String == "value-from-execute") } + +@Test func keychainFailuresDescribeTheStatusInsteadOfPrintingIt() { + // -34018 is the one host developers hit in their own unsigned test targets, + // and the model must learn it is not retryable. + let missing = KeychainBridge.failure("keychain.write", status: errSecMissingEntitlement) + #expect(missing.localizedDescription.contains("no keychain entitlement")) + #expect(missing.localizedDescription.contains("retrying will not help")) + + let other = KeychainBridge.failure("keychain.read", status: errSecDuplicateItem) + #expect(other.localizedDescription.contains("keychain.read failed:")) + #expect(other.localizedDescription.contains("(\(errSecDuplicateItem))")) +} diff --git a/Tests/CodeModeTests/NetworkBridgeTests.swift b/Tests/CodeModeTests/NetworkBridgeTests.swift index 5e444ae..ec72f4e 100644 --- a/Tests/CodeModeTests/NetworkBridgeTests.swift +++ b/Tests/CodeModeTests/NetworkBridgeTests.swift @@ -227,3 +227,74 @@ private final class StubHTTPURLProtocol: URLProtocol { #expect(result.string("stringValue") == "https://example.com/path?q=1") #expect(result.string("methodValue") == "https://example.com/path?q=1") } + +// MARK: - Ambient authority + +@Test func networkBridgeDefaultSessionCarriesNoAmbientCredentials() { + // The shipping default used to be `URLSession.shared`, which reads + // HTTPCookieStorage.shared and URLCredentialStorage.shared — so a script got + // authenticated session-riding against every origin the app is logged in to, + // and a Set-Cookie in a script-fetched response poisoned the app's jar. + let configuration = NetworkBridge.isolatedSession.configuration + #expect(configuration.httpCookieStorage == nil) + #expect(configuration.urlCredentialStorage == nil) + #expect(configuration.urlCache == nil) + #expect(configuration.httpShouldSetCookies == false) + #expect(configuration.httpCookieAcceptPolicy == .never) + #expect(NetworkBridge.isolatedSession !== URLSession.shared) +} + +@Test func networkFetchRefusesCredentialHeadersByDefault() throws { + let configuration = URLSessionConfiguration.ephemeral + configuration.protocolClasses = [StubHTTPURLProtocol.self] + let session = URLSession(configuration: configuration) + + let bridge = NetworkBridge(session: session) + let (context, sandbox) = try makeInvocationContext() + defer { + cleanup(sandbox) + session.invalidateAndCancel() + } + + for header in ["Cookie", "cookie", "Authorization", "Proxy-Authorization", "proxy-authenticate"] { + do { + _ = try bridge.fetch(arguments: [ + "url": .string("https://unit.test/endpoint"), + "options": .object(["headers": .object([header: .string("secret")])]), + ], context: context) + Issue.record("Expected \(header) to be refused") + } catch { + #expect(requireBridgeErrorCode(error) == "NETWORK_POLICY_VIOLATION") + } + } + + // Ordinary headers are unaffected. + let ok = try bridge.fetch(arguments: [ + "url": .string("https://unit.test/endpoint"), + "options": .object(["headers": .object(["X-Unit": .string("plain")])]), + ], context: context) + #expect(try requireObject(ok).bool("ok") == true) +} + +@Test func networkFetchAllowsCredentialHeadersWhenHostOptsIn() throws { + let configuration = URLSessionConfiguration.ephemeral + configuration.protocolClasses = [StubHTTPURLProtocol.self] + let session = URLSession(configuration: configuration) + + let bridge = NetworkBridge( + session: session, + policy: NetworkAccessPolicy(allowsCredentialHeaders: true) + ) + let (context, sandbox) = try makeInvocationContext() + defer { + cleanup(sandbox) + session.invalidateAndCancel() + } + + let result = try bridge.fetch(arguments: [ + "url": .string("https://unit.test/endpoint"), + "options": .object(["headers": .object(["Authorization": .string("Bearer t")])]), + ], context: context) + #expect(try requireObject(result).bool("ok") == true) + #expect(NetworkAccessPolicy.permissive.allowsCredentialHeaders) +} diff --git a/Tests/CodeModeTests/PolishFindingsTests.swift b/Tests/CodeModeTests/PolishFindingsTests.swift new file mode 100644 index 0000000..ac3f7cb --- /dev/null +++ b/Tests/CodeModeTests/PolishFindingsTests.swift @@ -0,0 +1,143 @@ +import Foundation +import Testing +@testable import CodeMode + +// MARK: - One ISO 8601 parser + +@Test func iso8601ParsingAcceptsFractionalSeconds() { + // `Date.toISOString()` in JavaScript always emits milliseconds, so this is + // the spelling a model produces by default. A bare ISO8601DateFormatter + // rejects it, and the bridges that used one silently substituted "now". + #expect(CodeModeDate.parse("2026-07-24T10:00:00.000Z") != nil) + #expect(CodeModeDate.parse("2026-07-24T10:00:00Z") != nil) + #expect(CodeModeDate.parse("2026-07-24T10:00:00+01:00") != nil) + #expect(CodeModeDate.parse(" 2026-07-24T10:00:00Z ") != nil) + #expect(CodeModeDate.parse("not a date") == nil) + #expect(CodeModeDate.parse("") == nil) + #expect(CodeModeDate.parse(nil) == nil) +} + +@Test func requiredDatesFailInsteadOfSubstitutingADefault() { + #expect(throws: (any Error).self) { + _ = try CodeModeDate.require("yesterday", argument: "start", capability: "calendar.read") + } + #expect(throws: (any Error).self) { + _ = try CodeModeDate.require(nil, argument: "start", capability: "calendar.read") + } + // A present-but-unparseable optional is still an error; only absence defaults. + #expect(throws: (any Error).self) { + _ = try CodeModeDate.optional("soon", argument: "end", capability: "calendar.read") + } +} + +@Test func absentOptionalDatesStillFallBackToTheCallersDefault() throws { + let absent = try CodeModeDate.optional(nil, argument: "end", capability: "calendar.read") + #expect(absent == nil) +} + +// MARK: - Network policy + +@Test func standardPolicyBlocksBareSingleLabelHosts() { + let policy = NetworkAccessPolicy.standard + // These resolve onto the LAN through the DHCP search domain, exactly like + // *.local, but previously fell through to "allowed". + for host in ["router", "intranet", "wpad", "nas", "printer"] { + let url = URL(string: "http://\(host)/admin")! + #expect(policy.violationReason(for: url) != nil, "expected \(host) to be refused") + } + + // Ordinary public hosts are unaffected. + #expect(policy.violationReason(for: URL(string: "https://example.com/x")!) == nil) + #expect(policy.violationReason(for: URL(string: "https://api.example.co.uk/x")!) == nil) +} + +@Test func hostsCanStillAllowlistASingleLabelHost() { + let policy = NetworkAccessPolicy(allowedHosts: ["router"]) + #expect(policy.violationReason(for: URL(string: "http://router/admin")!) == nil) +} + +// MARK: - System UI destinations + +@Test func webPresentationRespectsTheNetworkAccessPolicy() throws { + let bridge = SystemUIBridge(networkAccessPolicy: .standard) + let (context, sandbox) = try makeInvocationContext() + defer { cleanup(sandbox) } + + // Opening a page in a browser was exempt from the policy governing fetch, so + // a blocked origin could still be put in front of the user — or be the target + // of a script-started OAuth flow. + for capability in ["web", "auth"] { + let arguments: [String: JSONValue] = ["url": .string("http://169.254.169.254/latest/meta-data")] + do { + if capability == "web" { + _ = try bridge.presentWeb(arguments: arguments, context: context) + } else { + _ = try bridge.authenticateWeb(arguments: arguments, context: context) + } + Issue.record("Expected \(capability) presentation to be refused by the network policy") + } catch { + #expect(requireBridgeErrorCode(error) == "NETWORK_POLICY_VIOLATION") + } + } +} + +// MARK: - Path containment + +@Test func danglingSymlinksDoNotPassContainment() throws { + let sandbox = try makeTestSandbox() + defer { cleanup(sandbox) } + + let outsideRoot = sandbox.root.appendingPathComponent("outside", isDirectory: true) + let policy = DefaultPathPolicy( + config: PathPolicyConfig(tmpRoot: sandbox.tmp, cachesRoot: sandbox.caches, documentsRoot: sandbox.documents) + ) + + // The target deliberately does not exist: `fileExists` follows symlinks and + // so reported the link as a *missing component*, which was re-appended to the + // resolved ancestor and admitted instead of being resolved out of the root. + let link = sandbox.tmp.appendingPathComponent("escape") + try FileManager.default.createSymbolicLink( + at: link, + withDestinationURL: outsideRoot.appendingPathComponent("secret.txt") + ) + + #expect(throws: (any Error).self) { + _ = try policy.resolve(path: "tmp:escape") + } +} + +@Test func allowedRootsAreReportedByTheDefaultPolicy() throws { + let sandbox = try makeTestSandbox() + defer { cleanup(sandbox) } + + let policy = DefaultPathPolicy( + config: PathPolicyConfig(tmpRoot: sandbox.tmp, cachesRoot: sandbox.caches, documentsRoot: sandbox.documents) + ) + let documentsRoot = try policy.resolve(path: "documents:") + let tmpRoot = try policy.resolve(path: "tmp:") + let insideTmp = try policy.resolve(path: "tmp:file.txt") + + #expect(policy.allowedRoots.count == 3) + #expect(policy.isAllowedRoot(documentsRoot)) + #expect(policy.isAllowedRoot(tmpRoot)) + #expect(policy.isAllowedRoot(insideTmp) == false) +} + +// MARK: - Diagnostics reach the stream + +@Test func errorSeverityDiagnosticsAreEmittedToTheEventStream() { + let received = LockedBox<[ToolDiagnostic]>([]) + let transcript = ExecutionTranscript { event in + if case let .diagnostic(diagnostic) = event { + received.set(received.get() + [diagnostic]) + } + } + + transcript.record(diagnostic: ToolDiagnostic(severity: .info, code: "I", message: "info")) + transcript.record(diagnostic: ToolDiagnostic(severity: .warning, code: "W", message: "warning")) + // Previously filtered out, so a host watching the stream saw every diagnostic + // except the ones that mattered most. + transcript.record(diagnostic: ToolDiagnostic(severity: .error, code: "E", message: "error")) + + #expect(received.get().map(\.code) == ["I", "W", "E"]) +} diff --git a/Tests/CodeModeTests/TestSupport.swift b/Tests/CodeModeTests/TestSupport.swift index 3e2a23a..4dc756b 100644 --- a/Tests/CodeModeTests/TestSupport.swift +++ b/Tests/CodeModeTests/TestSupport.swift @@ -1,6 +1,26 @@ import Foundation import Testing @testable import CodeMode +import Security + +/// Whether this process can use the keychain at all. An unsigned SwiftPM test +/// bundle on the iOS simulator has no keychain access group, so every keychain +/// call fails with `errSecMissingEntitlement` (-34018). Tests whose subject *is* +/// keychain storage skip there, with that reason; tests that only touch the +/// keychain incidentally assert their real subject instead. +let keychainIsAvailable: Bool = { + let query: [String: Any] = [ + kSecClass as String: kSecClassGenericPassword, + kSecAttrService as String: "CodeModeTests.keychain-probe", + kSecAttrAccount as String: UUID().uuidString, + kSecValueData as String: Data("probe".utf8), + ] + let status = SecItemAdd(query as CFDictionary, nil) + if status == errSecSuccess { + SecItemDelete(query as CFDictionary) + } + return status != errSecMissingEntitlement +}() struct TestSandbox { let root: URL @@ -35,6 +55,8 @@ func cleanup(_ sandbox: TestSandbox) { func makeTools( permissionBroker: any PermissionBroker = NoopPermissionBroker(), + capabilityGrant: CapabilityGrant = .unrestricted, + executionLimits: ExecutionLimits = .standard, fileSystem: any CodeModeFileSystem = LocalCodeModeFileSystem(), systemUIPresenter: any SystemUIPresenter = UnavailableSystemUIPresenter(), eventInbox: any CodeModeEventInbox = UnavailableCodeModeEventInbox(), @@ -49,7 +71,8 @@ func makeTools( passKitClient: any PassKitClient = UnavailablePassKitClient(), storeKitClient: any StoreKitClient = UnavailableStoreKitClient(), codeModeProviders: [any CodeModeProvider] = [], - hostPlatform: HostPlatform = .current + hostPlatform: HostPlatform = .current, + clock: any RuntimeClock = RealClock() ) throws -> (CodeModeAgentTools, TestSandbox) { let sandbox = try makeTestSandbox() @@ -59,7 +82,9 @@ func makeTools( let configuration = CodeModeConfiguration( pathPolicy: pathPolicy, + capabilityGrant: capabilityGrant, fileSystem: fileSystem, + executionLimits: executionLimits, artifactStore: InMemoryArtifactStore(), permissionBroker: permissionBroker, auditLogger: SyncAuditLogger(), @@ -79,7 +104,7 @@ func makeTools( hostPlatform: hostPlatform ) - let tools = CodeModeAgentTools(config: configuration) + let tools = CodeModeAgentTools(config: configuration, clock: clock) return (tools, sandbox) } diff --git a/Tests/CodeModeTests/TypeScriptDeclarationsTests.swift b/Tests/CodeModeTests/TypeScriptDeclarationsTests.swift new file mode 100644 index 0000000..e7e9f9a --- /dev/null +++ b/Tests/CodeModeTests/TypeScriptDeclarationsTests.swift @@ -0,0 +1,324 @@ +import Foundation +import Testing +@testable import CodeMode + +// Cloudflare's central Code Mode insight is that models write markedly better +// code against real types. Before this the model got a flat `argumentTypes` map +// of six coarse cases plus a prose `resultSummary`; these tests pin the shape of +// what it gets now. + +private func reference( + capability: String = "fs.read", + jsNames: [String] = ["apple.fs.read", "fs.promises.readFile"], + required: [String] = ["path"], + optional: [String] = ["encoding"], + types: [String: CapabilityArgumentType] = ["path": .string, "encoding": .string], + hints: [String: String] = ["path": "File path using allowed root prefix."], + constraints: CapabilityArgumentConstraints = .init(allowedStringValues: ["encoding": ["utf8", "base64"]]) +) -> JavaScriptAPIReference { + JavaScriptAPIReference( + capability: capability, + jsNames: jsNames, + summary: "Read a file.", + tags: ["fs"], + example: "await apple.fs.read({ path: 'tmp:x.txt' })", + requiredArguments: required, + optionalArguments: optional, + argumentTypes: types, + argumentHints: hints, + argumentConstraints: constraints, + resultSummary: "Object with path plus text or base64." + ) +} + +@Test func declarationsTypeArgumentsAndCarryHintsAsDocComments() { + let dts = TypeScriptDeclarations.declaration(for: reference()) + + #expect(dts.contains("path: string;")) + #expect(dts.contains("/** File path using allowed root prefix. */")) + // Required vs optional is expressed in the type, not left to prose. + #expect(dts.contains("encoding?:")) + #expect(dts.contains("path?:") == false) + #expect(dts.contains("Capability: fs.read")) + #expect(dts.contains("Returns: Object with path plus text or base64.")) +} + +@Test func constrainedArgumentsBecomeStringLiteralUnions() { + let dts = TypeScriptDeclarations.declaration(for: reference()) + // The constraint IS the type; a union beats `string` plus a prose hint the + // model has to notice and obey. + #expect(dts.contains(#"encoding?: "utf8" | "base64";"#)) +} + +@Test func nodeStyleAliasesAreDocumentedRatherThanMistypedAsObjectCalls() { + let dts = TypeScriptDeclarations.declaration(for: reference()) + // `fs.promises.readFile` is positional, which the catalog cannot express, so + // the generated signature must describe the namespaced helper and only + // mention the alias. + #expect(dts.contains("Aliases: fs.promises.readFile")) + #expect(dts.contains("positional")) +} + +@Test func dottedArgumentPathsBecomeNestedObjectTypes() { + let nested = reference( + capability: "network.fetch", + jsNames: ["fetch"], + required: ["url"], + optional: ["options.method", "options.timeoutMs"], + types: ["url": .string, "options.method": .string, "options.timeoutMs": .number], + hints: ["options.method": "HTTP method; defaults to GET."], + constraints: .none + ) + let dts = TypeScriptDeclarations.declaration(for: nested) + + #expect(dts.contains("options?: {")) + #expect(dts.contains("method?: string;")) + #expect(dts.contains("timeoutMs?: number;")) +} + +@Test func argumentlessHelpersTakeNoArgument() { + let none = reference( + capability: "location.permission.request", + jsNames: ["apple.location.requestPermission"], + required: [], + optional: [], + types: [:], + hints: [:], + constraints: .none + ) + #expect(TypeScriptDeclarations.declaration(for: none).contains("(): Promise")) +} + +@Test func theWholeSurfaceIsNamespacedAndPlatformFiltered() async throws { + let (tools, sandbox) = try makeTools(hostPlatform: .macOS) + defer { cleanup(sandbox) } + + let surface = tools.typeDeclarations() + + #expect(surface.contains("declare const apple: {")) + #expect(surface.contains("fs: {")) + #expect(surface.contains("read(args: {")) + // The Node-compat globals live in the hand-authored preamble, because their + // positional convention is not something the catalog can describe. + #expect(surface.contains("declare function fetch(")) + #expect(surface.contains("type CodeModeValue =")) + // iOS-only helpers must not be declared for a macOS host. + #expect(surface.contains("alarm: {") == false) +} + +@Test func iOSOnlyHelpersAppearForAniOSHost() async throws { + let (tools, sandbox) = try makeTools(hostPlatform: .iOS) + defer { cleanup(sandbox) } + #expect(tools.typeDeclarations().contains("alarm: {")) +} + +@Test func everyCapabilityCarriesItsOwnDeclarationForSearchResults() async throws { + let (tools, sandbox) = try makeTools() + defer { cleanup(sandbox) } + + let references = tools.capabilities() + #expect(references.isEmpty == false) + #expect(references.allSatisfy { $0.dts.isEmpty == false }) + + let read = try #require(references.first { $0.capability == "fs.read" }) + #expect(read.dts.contains("path: string;")) +} + +@Test func searchCanReturnDeclarationsAsItsPayload() async throws { + let (tools, sandbox) = try makeTools() + defer { cleanup(sandbox) } + + // Code-driven search stays the filter; the payload is now TypeScript. + let response = try await tools.searchJavaScriptAPI( + JavaScriptAPISearchRequest( + code: """ + async () => { + return api.references + .filter(ref => ref.tags.includes("filesystem")) + .map(ref => ref.dts) + .join("\\n\\n"); + } + """ + ) + ) + let payload = try #require(response.result?.stringValue) + #expect(payload.contains("declare const apple: {")) + #expect(payload.contains("read(args: {")) + #expect(payload.contains("path: string;")) +} + +@Test func nodeAliasesCanPassTheOverwriteFlagTheDeclarationPromises() async throws { + let (tools, sandbox) = try makeTools() + defer { cleanup(sandbox) } + + // fs.move/fs.copy now refuse an existing destination without an explicit + // overwrite, so the Node-style aliases had to gain a way to pass one — the + // preamble declares it, and this proves the runtime honours it. + let observed = try await execute( + tools, + request: JavaScriptExecutionRequest( + code: """ + await apple.fs.write({ path: 'tmp:src.txt', data: 'source' }); + await apple.fs.write({ path: 'tmp:dst.txt', data: 'destination' }); + await fs.promises.copyFile('tmp:src.txt', 'tmp:dst.txt', { overwrite: true }); + return await fs.promises.readFile('tmp:dst.txt', 'utf8'); + """, + allowedCapabilities: [.fsWrite, .fsRead, .fsCopy] + ) + ) + + #expect(observed.error == nil) + #expect(observed.result?.output == .string("source")) +} + +@Test(.enabled(if: keychainIsAvailable, "no keychain entitlement in this test host (errSecMissingEntitlement)")) func keychainHelpersAcceptTheObjectFormTheCatalogAdvertises() async throws { + let (tools, sandbox) = try makeTools() + defer { cleanup(sandbox) } + + // The catalog declared `key: string` as an object argument while the wrapper + // took a positional string, and its own example showed the positional form — + // so code written from the metadata sent "[object Object]" as the key. Both + // spellings now work, which is what makes the generated declaration true. + let observed = try await execute( + tools, + request: JavaScriptExecutionRequest( + code: """ + await apple.keychain.set({ key: 'codemode-dts-test', value: 'object-form' }); + const viaObject = await apple.keychain.get({ key: 'codemode-dts-test' }); + const viaPositional = await apple.keychain.get('codemode-dts-test'); + await apple.keychain.delete({ key: 'codemode-dts-test' }); + return { viaObject: viaObject && viaObject.value, viaPositional: viaPositional && viaPositional.value }; + """, + allowedCapabilities: [.keychainRead, .keychainWrite, .keychainDelete] + ) + ) + + #expect(observed.error == nil) + let output = try #require(observed.result?.output?.objectValue) + #expect(output.string("viaObject") == "object-form") + #expect(output.string("viaPositional") == "object-form") +} + +@Test func reservedWordHelpersRenderAsValidMemberNames() async throws { + let (tools, sandbox) = try makeTools(hostPlatform: .iOS) + defer { cleanup(sandbox) } + + // `apple.keychain.delete` and the `export` helpers used to render as + // `function delete(` / `function export(` — invalid TypeScript. As object + // type members they are legal. + let surface = tools.typeDeclarations() + let reserved = ["break", "case", "catch", "class", "const", "continue", "debugger", "default", "delete", + "do", "else", "enum", "export", "extends", "false", "finally", "for", "function", "if", + "import", "in", "instanceof", "new", "null", "return", "super", "switch", "this", + "throw", "true", "try", "typeof", "var", "void", "while", "with"] + for word in reserved { + #expect(surface.contains("function \(word)(") == false, "reserved word '\(word)' declared as a function") + } + #expect(surface.contains("delete(args: {")) + #expect(surface.contains("declare namespace") == false) +} + +// MARK: - Declarations follow the grant + +@Test func typeDeclarationsFollowTheCapabilityGrant() async throws { + let (granted, grantedSandbox) = try makeTools(capabilityGrant: .only([.fsRead, .fsList]), hostPlatform: .iOS) + defer { cleanup(grantedSandbox) } + let (open, openSandbox) = try makeTools(hostPlatform: .iOS) + defer { cleanup(openSandbox) } + + let narrow = granted.typeDeclarations() + let full = open.typeDeclarations() + + // Only what the host grants is advertised: never a helper the model would be + // denied, and never 21k tokens of surface when two helpers are permitted. + #expect(narrow.contains("read(args: {")) + #expect(narrow.contains("list(args: {")) + #expect(narrow.contains("keychain: {") == false) + #expect(narrow.contains("write(args: {") == false) + #expect(narrow.contains("declare function fetch("), "the preamble is always included") + #expect(narrow.count * 10 < full.count) +} + +@Test func typeDeclarationsCanBeNarrowedToAnExplicitSubset() async throws { + let (tools, sandbox) = try makeTools(hostPlatform: .iOS) + defer { cleanup(sandbox) } + + let subset = tools.typeDeclarations(for: [.keychainRead]) + #expect(subset.contains("keychain: {")) + #expect(subset.contains("get(args: {")) + // Not `fs: {` — the always-included preamble declares the Node `fs` shim. + #expect(subset.contains("read(args: {") == false) +} + +@Test func anUnrestrictedGrantDeclaresTheWholeSurface() async throws { + let (tools, sandbox) = try makeTools(hostPlatform: .iOS) + defer { cleanup(sandbox) } + let declared = tools.typeDeclarations() + for reference in tools.capabilities() { + let leaf = reference.jsNames.first { $0.contains(".") && !$0.hasPrefix("fs.promises.") }? + .split(separator: ".").last.map(String.init) + if let leaf { + #expect(declared.contains("\(leaf)("), "missing \(reference.capability)") + } + } +} + +// MARK: - Codable compatibility + +@Test func referencesEncodedBeforeDtsExistedStillDecode() throws { + // The shape a host cached before this branch: no `dts` key. The synthesized + // decoder threw keyNotFound on it. + let legacy = #"{"capability":"fs.read","capabilityKey":"fs.read","builtInCapability":"fs.read","jsNames":["apple.fs.read"],"summary":"Read a file.","tags":[],"example":"","requiredArguments":["path"],"optionalArguments":[],"argumentTypes":{"path":"string"},"argumentHints":{},"argumentConstraints":{"allowedStringValues":{}},"resultSummary":"Object."}"# + let decoded = try JSONDecoder().decode(JavaScriptAPIReference.self, from: Data(legacy.utf8)) + #expect(decoded.capability == "fs.read") + // Regenerated rather than left empty. + #expect(decoded.dts.contains("read(args: {")) + #expect(decoded.dts.contains("path: string;")) +} + +@Test func referencesRoundTripThroughCodableUnchanged() async throws { + let (tools, sandbox) = try makeTools() + defer { cleanup(sandbox) } + let original = tools.capabilities() + let decoded = try JSONDecoder().decode([JavaScriptAPIReference].self, from: JSONEncoder().encode(original)) + #expect(decoded == original) +} + +// MARK: - Artifacts for the compiler check + +/// Writes every generated declaration to `$CODEMODE_DTS_OUTPUT` for a real +/// `tsc --strict` pass (`scripts/check-typescript-declarations.sh`, run in CI). +/// The unit tests above check known failure shapes; only a compiler checks +/// that the output is TypeScript at all. Inert unless the variable is set. +@Test(.enabled(if: ProcessInfo.processInfo.environment["CODEMODE_DTS_OUTPUT"] != nil, "set CODEMODE_DTS_OUTPUT to write declarations for tsc")) +func writeTypeDeclarationsForCompilerCheck() async throws { + let output = URL(fileURLWithPath: try #require(ProcessInfo.processInfo.environment["CODEMODE_DTS_OUTPUT"])) + let surfaces = output.appendingPathComponent("surfaces", isDirectory: true) + let fragments = output.appendingPathComponent("fragments", isDirectory: true) + try FileManager.default.createDirectory(at: surfaces, withIntermediateDirectories: true) + try FileManager.default.createDirectory(at: fragments, withIntermediateDirectories: true) + + try TypeScriptDeclarations.preamble.write( + to: output.appendingPathComponent("preamble.d.ts"), atomically: true, encoding: .utf8 + ) + + // Every platform's surface, unrestricted, so no capability goes unchecked. + for platform in [HostPlatform.iOS, .macOS, .visionOS] { + let (tools, sandbox) = try makeTools(hostPlatform: platform) + defer { cleanup(sandbox) } + try tools.typeDeclarations().write( + to: surfaces.appendingPathComponent("\(platform).d.ts"), atomically: true, encoding: .utf8 + ) + } + + // Fragments from the widest platform surface. + let (tools, sandbox) = try makeTools(hostPlatform: .iOS) + defer { cleanup(sandbox) } + let references = tools.capabilities() + for reference in references { + try reference.dts.write( + to: fragments.appendingPathComponent("\(reference.capability).d.ts"), atomically: true, encoding: .utf8 + ) + } + #expect(references.count > 100, "expected the full iOS catalog") +} diff --git a/Tests/CodeModeTests/capability-metadata-golden.json b/Tests/CodeModeTests/capability-metadata-golden.json index 8e51716..d28571b 100644 --- a/Tests/CodeModeTests/capability-metadata-golden.json +++ b/Tests/CodeModeTests/capability-metadata-golden.json @@ -532,7 +532,7 @@ }, "argumentHints" : { "callbackURLScheme" : "Optional custom URL scheme that completes the session.", - "prefersEphemeralSession" : "Whether to prefer a private browser session; default false.", + "prefersEphemeralSession" : "Whether to use a private browser session; default true. Set false only when the flow must reuse the user's existing browser session.", "timeoutMs" : "Optional timeout for waiting on callback\/cancellation.", "url" : "Absolute HTTP(S) authentication URL." }, @@ -559,7 +559,7 @@ ], "resultSummary" : "Object with action callback\/cancelled and callbackURL when available.", - "summary" : "Start an ASWebAuthenticationSession for OAuth-style browser authentication.", + "summary" : "Start an ASWebAuthenticationSession for OAuth-style browser authentication. Runs in a private browser session by default and the URL must satisfy the host's network access policy.", "tags" : [ "auth", "oauth", @@ -1852,10 +1852,14 @@ }, "argumentHints" : { "from" : "Source sandbox path.", - "to" : "Destination sandbox path." + "overwrite" : "Required as true to replace an existing destination; default false.", + "recursive" : "Required as true alongside overwrite when the existing destination is a directory.", + "to" : "Destination sandbox path. Must name a path inside a root, not the root itself." }, "argumentTypes" : { "from" : "string", + "overwrite" : "bool", + "recursive" : "bool", "to" : "string" }, "example" : "await apple.fs.copy({ from: 'tmp:a.txt', to: 'tmp:b.txt' })", @@ -1865,7 +1869,8 @@ "fs.promises.copyFile" ], "optionalArguments" : [ - + "overwrite", + "recursive" ], "requiredArguments" : [ "from", @@ -1875,7 +1880,7 @@ ], "resultSummary" : "Object with from\/to resolved paths.", - "summary" : "Copy file\/directory within allowed sandbox roots.", + "summary" : "Copy file\/directory within allowed sandbox roots. Fails rather than clobbering an existing destination unless overwrite is set, and never accepts a sandbox root as the destination.", "tags" : [ "filesystem", "io", @@ -2028,10 +2033,14 @@ }, "argumentHints" : { "from" : "Source sandbox path.", - "to" : "Destination sandbox path." + "overwrite" : "Required as true to replace an existing destination; default false.", + "recursive" : "Required as true alongside overwrite when the existing destination is a directory.", + "to" : "Destination sandbox path. Must name a path inside a root, not the root itself." }, "argumentTypes" : { "from" : "string", + "overwrite" : "bool", + "recursive" : "bool", "to" : "string" }, "example" : "await apple.fs.move({ from: 'tmp:a.txt', to: 'tmp:b.txt' })", @@ -2041,7 +2050,8 @@ "fs.promises.rename" ], "optionalArguments" : [ - + "overwrite", + "recursive" ], "requiredArguments" : [ "from", @@ -2051,7 +2061,7 @@ ], "resultSummary" : "Object with from\/to resolved paths.", - "summary" : "Move file\/directory within allowed sandbox roots.", + "summary" : "Move file\/directory within allowed sandbox roots. Fails rather than clobbering an existing destination unless overwrite is set, and never accepts a sandbox root as the destination.", "tags" : [ "filesystem", "io", @@ -2061,10 +2071,14 @@ }, { "allowedStringValues" : { - + "encoding" : [ + "utf8", + "base64", + "utf-8" + ] }, "argumentHints" : { - "encoding" : "utf8 (default) or base64.", + "encoding" : "utf8 (default) or base64. Binary files must use base64; a utf8 read of undecodable bytes fails rather than returning empty text.", "path" : "File path using allowed root prefix." }, "argumentTypes" : { @@ -2131,7 +2145,11 @@ }, { "allowedStringValues" : { - + "encoding" : [ + "utf8", + "base64", + "utf-8" + ] }, "argumentHints" : { "data" : "UTF-8 text or base64 string depending on encoding.", @@ -2195,7 +2213,7 @@ "requiredPermissions" : [ ], - "resultSummary" : "Object with status\/granted fields and requested type arrays.", + "resultSummary" : "Object with status ('completed'), per-type writeAuthorization, and the requested type arrays. Read authorization is never disclosed by HealthKit, so an empty health.read may mean denial.", "summary" : "Request HealthKit authorization for requested read\/write types.", "tags" : [ "healthkit", @@ -2380,7 +2398,7 @@ "argumentTypes" : { "key" : "string" }, - "example" : "await apple.keychain.delete('auth_token')", + "example" : "await apple.keychain.delete({ key: 'auth_token' })", "id" : "keychain.delete", "jsNames" : [ "apple.keychain.delete" @@ -2413,7 +2431,7 @@ "argumentTypes" : { "key" : "string" }, - "example" : "await apple.keychain.get('auth_token')", + "example" : "await apple.keychain.get({ key: 'auth_token' })", "id" : "keychain.read", "jsNames" : [ "apple.keychain.get" @@ -2448,7 +2466,7 @@ "key" : "string", "value" : "string" }, - "example" : "await apple.keychain.set('auth_token', token)", + "example" : "await apple.keychain.set({ key: 'auth_token', value: token })", "id" : "keychain.write", "jsNames" : [ "apple.keychain.set" @@ -3265,7 +3283,7 @@ "argumentHints" : { "options.body" : "UTF-8 request body string.", "options.bodyBase64" : "Base64-encoded request body. Mutually exclusive with options.body.", - "options.headers" : "Object of header key\/value string pairs.", + "options.headers" : "Object of header key\/value string pairs. Cookie, Authorization, and Proxy-* are refused unless the host app permits credential headers.", "options.method" : "HTTP method; defaults to GET.", "options.responseEncoding" : "text (default) or base64.", "options.timeoutMs" : "Request and bridge wait timeout in milliseconds; defaults to 30000.", @@ -3300,7 +3318,7 @@ ], "resultSummary" : "Object with ok\/status\/statusText\/headers\/bodyText or bodyBase64.", - "summary" : "Perform HTTP(S) requests through URLSession via a fetch-compatible API.", + "summary" : "Perform HTTP(S) requests through an isolated URLSession via a fetch-compatible API. Requests carry no app cookies or stored credentials, and credential headers (Cookie, Authorization, Proxy-*) are refused unless the host app permits them.", "tags" : [ "network", "http", diff --git a/scripts/check-typescript-declarations.sh b/scripts/check-typescript-declarations.sh new file mode 100755 index 0000000..d27518d --- /dev/null +++ b/scripts/check-typescript-declarations.sh @@ -0,0 +1,61 @@ +#!/usr/bin/env bash +# Compiles every TypeScript declaration CodeMode generates with `tsc --strict`. +# +# The Swift tests check known failure shapes (reserved-word function names, for +# one); only a real compiler checks that the output is valid TypeScript at all. +# +# scripts/check-typescript-declarations.sh [output-dir] +# +# TSC overrides the compiler command, e.g. `TSC=tsc` to use a local install. +# Default: TypeScript 7.0.2 via npx. +set -euo pipefail + +cd "$(dirname "$0")/.." +out="${1:-$(mktemp -d)}" +rm -rf "$out" +mkdir -p "$out" +tsc_cmd="${TSC:-npx --yes --package typescript@7.0.2 tsc}" + +echo "Writing declarations to $out" +CODEMODE_DTS_OUTPUT="$out" swift test --filter writeTypeDeclarationsForCompilerCheck + +surfaces=$(find "$out/surfaces" -name '*.d.ts' | wc -l | tr -d ' ') +fragments=$(find "$out/fragments" -name '*.d.ts' | wc -l | tr -d ' ') +# Guard against a vacuous pass: a filter typo or a skipped test would leave +# nothing for tsc to check, and an empty compilation succeeds. +if [ "$surfaces" -ne 3 ] || [ "$fragments" -lt 100 ]; then + echo "error: expected 3 surfaces and 100+ fragments, found $surfaces and $fragments" >&2 + exit 1 +fi + +# One compilation, not 124. The preamble stays a global script so fragments can +# use `CodeModeValue`, as they would in a host prompt. Each surface and fragment +# becomes its own module, so their top-level `declare const apple` stay local +# instead of colliding — each is checked exactly as it would be used alone. +for file in "$out"/surfaces/*.d.ts "$out"/fragments/*.d.ts; do + printf '\nexport {};\n' >> "$file" +done + +cat > "$out/tsconfig.json" <<'JSON' +{ + "compilerOptions": { + "strict": true, + "noEmit": true, + "lib": ["es2022"], + "types": [], + "skipLibCheck": false + }, + "include": ["preamble.d.ts", "surfaces/*.d.ts", "fragments/*.d.ts"] +} +JSON + +echo "Compiling $surfaces surfaces and $fragments fragments with: $tsc_cmd" +listed=$($tsc_cmd -p "$out/tsconfig.json" --listFilesOnly | grep -c "$out" || true) +expected=$((surfaces + fragments + 1)) +if [ "$listed" -lt "$expected" ]; then + echo "error: tsc would check $listed of $expected declaration files" >&2 + exit 1 +fi + +$tsc_cmd -p "$out/tsconfig.json" +echo "OK: all $expected declaration files compile under --strict"